We've noticed high CPU usage when creating simple dashboards displaying a few series. I have a simple test case of a Python application generating random data and a Grafana dashboard which goes to 100% CPU usage on the browser within 15 minutes (M1 Macbook Pro).
The generator emits a websocket message once per second. This message consists of 100 time series, broken down into 5 "assets" of 20 time series each. This is quite small compared to our expected production workloads. The Grafana dashboard has a single time series panel, with 4 series displayed. A dashboard variable allows the dashboard to switch between viewing each of the 5 assets.
We are using Python 3.11.2 to generate the sample data with the websockets 11.0.3 package. Grafana is version 10.0.3 with v1.0.2 of grafana-websocket-plugin.
From looking at Chrome's profiler, it looks like the bulk of the time is spent in JSONPath.prototype._trace
from the first entry and in JSON parsing in the second entry (in transformFrames
).
There also appears to be a lot of garbage generated. Also from the Chrome profiler (but a different run) shows the heap going from ~46MB to ~200MB on a consistent, regular basis.
I haven't delved any deeper into the code yet, but is it possible that the entire set of messages is being parsed and JSONPath queries being executed every time a new message is received over the web socket? I wouldn't expect so much CPU and memory usage if the JSON parsing and JSONPathEvaluation were being performed incrementally, (i.e. once per received message).
{
"data": {
"asset1": {
"series1": {
"time": "2023-07-31T02:39:52.581897+00:00",
"value": 31.111732219573604
},
"series2": {
"time": "2023-07-31T02:39:52.581926+00:00",
"value": 63.5330379688653
},
"series3": {
"time": "2023-07-31T02:39:52.581933+00:00",
"value": 65.02248736137514
},
"series4": {
"time": "2023-07-31T02:39:52.581939+00:00",
"value": 73.41998843575468
},
"series5": {
"time": "2023-07-31T02:39:52.581944+00:00",
"value": 86.03779154886973
},
"series6": {
"time": "2023-07-31T02:39:52.581949+00:00",
"value": 65.6194646523102
},
"series7": {
"time": "2023-07-31T02:39:52.581954+00:00",
"value": 30.096497707717766
},
"series8": {
"time": "2023-07-31T02:39:52.581958+00:00",
"value": 53.743947951958866
},
"series9": {
"time": "2023-07-31T02:39:52.582293+00:00",
"value": 84.41985850757489
},
"series10": {
"time": "2023-07-31T02:39:52.582299+00:00",
"value": 37.355429126763866
},
"series11": {
"time": "2023-07-31T02:39:52.582304+00:00",
"value": 15.151001705763685
},
"series12": {
"time": "2023-07-31T02:39:52.582308+00:00",
"value": 33.86689169026734
},
"series13": {
"time": "2023-07-31T02:39:52.582314+00:00",
"value": 57.0090106838116
},
"series14": {
"time": "2023-07-31T02:39:52.582318+00:00",
"value": 32.31957583025593
},
"series15": {
"time": "2023-07-31T02:39:52.582323+00:00",
"value": 8.482657848733249
},
"series16": {
"time": "2023-07-31T02:39:52.582327+00:00",
"value": 37.309829376398504
},
"series17": {
"time": "2023-07-31T02:39:52.582332+00:00",
"value": 40.999623896430194
},
"series18": {
"time": "2023-07-31T02:39:52.582336+00:00",
"value": 30.91474376553277
},
"series19": {
"time": "2023-07-31T02:39:52.582341+00:00",
"value": 87.49841135797917
},
"series20": {
"time": "2023-07-31T02:39:52.582345+00:00",
"value": 46.632828535539396
}
},
"asset2": {
"series1": {
"time": "2023-07-31T02:39:52.582351+00:00",
"value": 18.585603557939766
},
"series2": {
"time": "2023-07-31T02:39:52.582355+00:00",
"value": 57.018406669961585
},
"series3": {
"time": "2023-07-31T02:39:52.582360+00:00",
"value": 43.32716313978087
},
"series4": {
"time": "2023-07-31T02:39:52.582364+00:00",
"value": 43.40153492740899
},
"series5": {
"time": "2023-07-31T02:39:52.582369+00:00",
"value": 25.593522251024506
},
"series6": {
"time": "2023-07-31T02:39:52.582373+00:00",
"value": 61.02313092462206
},
"series7": {
"time": "2023-07-31T02:39:52.582378+00:00",
"value": 73.72712691600128
},
"series8": {
"time": "2023-07-31T02:39:52.582382+00:00",
"value": 75.96536238426866
},
"series9": {
"time": "2023-07-31T02:39:52.582387+00:00",
"value": 44.34975427328715
},
"series10": {
"time": "2023-07-31T02:39:52.582391+00:00",
"value": 15.529848518015163
},
"series11": {
"time": "2023-07-31T02:39:52.582396+00:00",
"value": 40.098751969565576
},
"series12": {
"time": "2023-07-31T02:39:52.582400+00:00",
"value": 93.26790103484674
},
"series13": {
"time": "2023-07-31T02:39:52.582404+00:00",
"value": 83.70344481539355
},
"series14": {
"time": "2023-07-31T02:39:52.582408+00:00",
"value": 28.833837743651735
},
"series15": {
"time": "2023-07-31T02:39:52.582413+00:00",
"value": 4.369127386869462
},
"series16": {
"time": "2023-07-31T02:39:52.582417+00:00",
"value": 10.898625861487838
},
"series17": {
"time": "2023-07-31T02:39:52.582422+00:00",
"value": 50.41686668070026
},
"series18": {
"time": "2023-07-31T02:39:52.582426+00:00",
"value": 30.748993487654584
},
"series19": {
"time": "2023-07-31T02:39:52.582431+00:00",
"value": 71.35657017629057
},
"series20": {
"time": "2023-07-31T02:39:52.582436+00:00",
"value": 38.3809495694423
}
},
"asset3": {
"series1": {
"time": "2023-07-31T02:39:52.582442+00:00",
"value": 30.643587799025973
},
"series2": {
"time": "2023-07-31T02:39:52.582448+00:00",
"value": 71.2267776721094
},
"series3": {
"time": "2023-07-31T02:39:52.582453+00:00",
"value": 10.132010493040038
},
"series4": {
"time": "2023-07-31T02:39:52.582457+00:00",
"value": 24.929407598279543
},
"series5": {
"time": "2023-07-31T02:39:52.582461+00:00",
"value": 31.29253286922883
},
"series6": {
"time": "2023-07-31T02:39:52.582465+00:00",
"value": 19.522077283317635
},
"series7": {
"time": "2023-07-31T02:39:52.582470+00:00",
"value": 43.247530005114974
},
"series8": {
"time": "2023-07-31T02:39:52.582474+00:00",
"value": 30.228105838641323
},
"series9": {
"time": "2023-07-31T02:39:52.582479+00:00",
"value": 35.094002748243184
},
"series10": {
"time": "2023-07-31T02:39:52.582482+00:00",
"value": 74.04013414483143
},
"series11": {
"time": "2023-07-31T02:39:52.582487+00:00",
"value": 33.195993342594385
},
"series12": {
"time": "2023-07-31T02:39:52.582491+00:00",
"value": 22.480432120910248
},
"series13": {
"time": "2023-07-31T02:39:52.582495+00:00",
"value": 73.15460606287232
},
"series14": {
"time": "2023-07-31T02:39:52.582499+00:00",
"value": 0.9106067094319794
},
"series15": {
"time": "2023-07-31T02:39:52.582504+00:00",
"value": 45.80079656886726
},
"series16": {
"time": "2023-07-31T02:39:52.582508+00:00",
"value": 35.065531470380726
},
"series17": {
"time": "2023-07-31T02:39:52.582513+00:00",
"value": 6.408510358262506
},
"series18": {
"time": "2023-07-31T02:39:52.582517+00:00",
"value": 38.870698418860655
},
"series19": {
"time": "2023-07-31T02:39:52.582522+00:00",
"value": 22.374657540795003
},
"series20": {
"time": "2023-07-31T02:39:52.582527+00:00",
"value": 78.4203093413092
}
},
"asset4": {
"series1": {
"time": "2023-07-31T02:39:52.582532+00:00",
"value": 9.290502633907668
},
"series2": {
"time": "2023-07-31T02:39:52.582537+00:00",
"value": 17.08408388217194
},
"series3": {
"time": "2023-07-31T02:39:52.582582+00:00",
"value": 90.56203124331456
},
"series4": {
"time": "2023-07-31T02:39:52.582588+00:00",
"value": 58.213352971868446
},
"series5": {
"time": "2023-07-31T02:39:52.582594+00:00",
"value": 25.454428488661005
},
"series6": {
"time": "2023-07-31T02:39:52.582598+00:00",
"value": 11.898313311680464
},
"series7": {
"time": "2023-07-31T02:39:52.582603+00:00",
"value": 88.12776074535932
},
"series8": {
"time": "2023-07-31T02:39:52.582608+00:00",
"value": 83.34455717866732
},
"series9": {
"time": "2023-07-31T02:39:52.582613+00:00",
"value": 22.020829234563593
},
"series10": {
"time": "2023-07-31T02:39:52.582617+00:00",
"value": 22.46444061535176
},
"series11": {
"time": "2023-07-31T02:39:52.582622+00:00",
"value": 33.52192429914582
},
"series12": {
"time": "2023-07-31T02:39:52.582625+00:00",
"value": 22.283904092739306
},
"series13": {
"time": "2023-07-31T02:39:52.582630+00:00",
"value": 7.130853822969623
},
"series14": {
"time": "2023-07-31T02:39:52.582634+00:00",
"value": 7.841786700193415
},
"series15": {
"time": "2023-07-31T02:39:52.582639+00:00",
"value": 74.36816619053545
},
"series16": {
"time": "2023-07-31T02:39:52.582643+00:00",
"value": 78.20524202860932
},
"series17": {
"time": "2023-07-31T02:39:52.582648+00:00",
"value": 28.866178099494128
},
"series18": {
"time": "2023-07-31T02:39:52.582652+00:00",
"value": 17.84580917826871
},
"series19": {
"time": "2023-07-31T02:39:52.582657+00:00",
"value": 17.49455173548536
},
"series20": {
"time": "2023-07-31T02:39:52.582661+00:00",
"value": 94.47549261021013
}
},
"asset5": {
"series1": {
"time": "2023-07-31T02:39:52.582666+00:00",
"value": 12.13133879261409
},
"series2": {
"time": "2023-07-31T02:39:52.582671+00:00",
"value": 30.34177441291777
},
"series3": {
"time": "2023-07-31T02:39:52.582675+00:00",
"value": 82.16341311081666
},
"series4": {
"time": "2023-07-31T02:39:52.582679+00:00",
"value": 36.868762150764894
},
"series5": {
"time": "2023-07-31T02:39:52.582684+00:00",
"value": 71.72535233091388
},
"series6": {
"time": "2023-07-31T02:39:52.582688+00:00",
"value": 87.63281455780572
},
"series7": {
"time": "2023-07-31T02:39:52.582753+00:00",
"value": 77.30002072214205
},
"series8": {
"time": "2023-07-31T02:39:52.582766+00:00",
"value": 9.744614484234571
},
"series9": {
"time": "2023-07-31T02:39:52.582775+00:00",
"value": 25.463353723595315
},
"series10": {
"time": "2023-07-31T02:39:52.582780+00:00",
"value": 66.79971934295749
},
"series11": {
"time": "2023-07-31T02:39:52.582786+00:00",
"value": 25.330746179842286
},
"series12": {
"time": "2023-07-31T02:39:52.582791+00:00",
"value": 64.86910370930916
},
"series13": {
"time": "2023-07-31T02:39:52.582797+00:00",
"value": 76.45466840348693
},
"series14": {
"time": "2023-07-31T02:39:52.582802+00:00",
"value": 48.696411457635804
},
"series15": {
"time": "2023-07-31T02:39:52.582807+00:00",
"value": 68.58407514421128
},
"series16": {
"time": "2023-07-31T02:39:52.582811+00:00",
"value": 69.57896860374916
},
"series17": {
"time": "2023-07-31T02:39:52.582816+00:00",
"value": 64.57902167189174
},
"series18": {
"time": "2023-07-31T02:39:52.582821+00:00",
"value": 17.019976648576847
},
"series19": {
"time": "2023-07-31T02:39:52.582827+00:00",
"value": 62.962668869823915
},
"series20": {
"time": "2023-07-31T02:39:52.582831+00:00",
"value": 57.01574232843745
}
}
}
}