Debug WIP: Added non cachegrind run to cachegrind benchmark

luau-lang · Jun 9, 2022 · 54aaad2 · 54aaad2
1 parent 34e6c73
commit 54aaad2
Show file tree

Hide file tree

Showing 7 changed files with 31 additions and 96 deletions.
diff --git a/.github/workflows/benchmark-cachegrind.yml b/.github/workflows/benchmark-cachegrind.yml
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - master
+      - feat/add-cachegrind
+
     paths-ignore:
       - "docs/**"
       - "papers/**"
@@ -27,11 +29,11 @@ jobs:
               cachegrindIterCount: 20,
             }
         benchResultsRepo:
-          - { name: "luau-lang/benchmark-data", branch: "main" }
+          - { name: "AllanJeremy/luau-benchmark-results", branch: "main" }
 
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Checkout Luau repository
+      - name: Checkout Luau
         uses: actions/checkout@v3
 
       - name: Build Luau
@@ -47,10 +49,20 @@ jobs:
           python -m pip install requests
           python -m pip install --user numpy scipy matplotlib ipython jupyter pandas sympy nose
 
+      - name: Install valgrind
+        run: |
+          sudo apt-get install valgrind
+
       - name: Run benchmark
         run: |
           python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt
 
+      - name: Run ${{ matrix.bench.title }} (Cold Cachegrind)
+        run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt
+
+      - name: Run ${{ matrix.bench.title }} (Warm Cachegrind)
+        run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt
+
       - name: Checkout Benchmark Results repository
         uses: actions/checkout@v3
         with:
@@ -67,13 +79,26 @@ jobs:
           output-file-path: ./${{ matrix.bench.script }}-output.txt
           external-data-json-path: ./gh-pages/dev/bench/data.json
           alert-threshold: 150%
-          fail-threshold: 200%
-          fail-on-alert: true
+          fail-threshold: 1000%
+          fail-on-alert: false
+          comment-on-alert: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Store ${{ matrix.bench.title }} result
+        uses: Roblox/rhysd-github-action-benchmark@v-luau
+        with:
+          name: ${{ matrix.bench.title }} (CacheGrind)
+          tool: "roblox"
+          output-file-path: ./${{ matrix.bench.script }}-output.txt
+          external-data-json-path: ./gh-pages/dev/bench/data.json
+          alert-threshold: 150%
+          fail-threshold: 1000%
+          fail-on-alert: false
           comment-on-alert: true
-          comment-always: true
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Push benchmark results
+
         run: |
           echo "Pushing benchmark results..."
           cd gh-pages

diff --git a/bench.json b/bench.json
@@ -0,0 +1 @@
+[[["base64.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "base64", [15.067666652612388, 15.099500014912337, 15.193625004030764, 15.220624976791441, 15.246208291500807, 15.253000019583851, 15.269000025000423, 15.273416647687554, 15.27958334190771, 15.396249946206808, 15.520916669629514, 15.54091670550406, 15.560166677460074, 15.588125039357692, 15.600375016219914, 15.623041661456227, 15.634291688911617, 15.648250002413988, 15.653166628908366, 15.773249964695424], 20]], [["chess.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "chess", [70.52754168398678, 70.88770833797753, 70.90950000565499, 71.17445836775005, 71.62204169435427, 71.67375000426546, 71.8345416826196, 71.91033335402608, 71.96612498955801, 71.97883329354227, 72.02420837711543, 72.05783331301063, 72.14320835191756, 72.16283335583284, 72.20737502211705, 72.22208334133029, 72.23900000099093, 72.25858338642865, 72.26749998517334, 72.27050000801682], 20]], [["life.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "life", [86.65845834184438, 86.67854167288169, 86.87208330957219, 87.0120000327006, 87.02495833858848, 87.16141665354371, 87.35450002131984, 87.37220836337656, 87.52162498421967, 87.71291666198522, 87.75512501597404, 87.77099999133497, 87.78554166201502, 87.81979168998078, 87.8200416918844, 87.86758332280442, 87.86912501091138, 87.87512499839067, 87.98524999292567, 88.11562502523884], 20]], [["qsort.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "qsort", [63.523625023663044, 63.706375018227845, 63.74829163542017, 63.78783332183957, 63.815124973189086, 64.02708328096196, 64.19545831158757, 64.20508329756558, 64.24045830499381, 64.2517083324492, 64.34191664448008, 64.36833331827074, 64.36912500066683, 64.37799998093396, 64.38566668657586, 64.47354168631136, 64.5666666678153, 64.64583333581686, 64.65783336898312, 64.69949998427182], 20]], [["sha256.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "sha256", [13.122166623361409, 13.131041661836207, 13.156000000890344, 13.157083303667605, 13.173833373002708, 13.193416642025113, 13.197791646234691, 13.202874979469925, 13.225166650954634, 13.225291622802615, 13.239333347883075, 13.241500011645257, 13.244791654869914, 13.269166636746377, 13.278250000439584, 13.309500005561858, 13.329416688065976, 13.416083296760917, 13.419916678685695, 13.427833328023553], 20]], [["ack.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "ack", [52.46454162988812, 52.527666673995554, 52.55633330671117, 52.75187501683831, 52.78174998238683, 52.826333383563906, 52.88858333369717, 52.933583327103406, 52.94500000309199, 52.97970829997212, 53.001666674390435, 53.14616666873917, 53.17374999867752, 53.22512501152232, 53.23195835808292, 53.25874994741753, 53.363916638772935, 53.37966664228588, 53.476291650440544, 53.48533333744854], 20]], [["binary-trees.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "binary-trees", [19.517458335030824, 19.553416699636728, 19.679541641380638, 19.698583288118243, 19.712416629772633, 19.719416624866426, 19.81029164744541, 19.83258337713778, 19.850416691042483, 19.876083359122276, 19.915083306841552, 19.92074999725446, 19.99795832671225, 20.036958332639188, 20.055916684214026, 20.056749985087663, 20.116250030696392, 20.14045836403966, 20.243458333425224, 20.29650000622496], 20]], [["fannkuch-redux.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "fannkuchen-redux", [12.419791659340262, 12.698083301074803, 12.741708371322602, 12.758583296090364, 12.801083328668028, 12.809750041924417, 12.845124991144985, 12.858083355240524, 12.908375007100403, 12.926416704431176, 12.931375007610768, 12.937749968841672, 12.943166657350957, 12.995458324439824, 13.045749976299703, 13.138249982148409, 13.157374982256442, 13.22791661368683, 13.52625002618879, 13.745750009547919], 20]], [["fixpoint-fact.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "fixpoint-fact", [42.03200002666563, 42.20620833802968, 42.23950003506616, 42.47854166897014, 42.51970833865926, 42.611374985426664, 42.63087501749396, 42.64025000156835, 42.65987500548363, 42.72745834896341, 42.79729165136814, 42.80958330491558, 42.82366664847359, 42.842541646678, 42.90708334883675, 43.00500004319474, 43.26516663422808, 43.31020830431953, 43.32812497159466, 43.44104172196239], 20]], [["heapsort.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "heapsort", [31.963000015821308, 31.984750006813556, 32.032458344474435, 32.16045832959935, 32.2135416790843, 32.26958331651986, 32.28120837593451, 32.34991669887677, 32.37345832167193, 32.41149999666959, 32.48845832422376, 32.5043749762699, 32.54062501946464, 32.627208332996815, 32.65549999196082, 32.669458305463195, 32.69812505459413, 32.793083344586194, 32.80495834769681, 32.81875001266599], 20]], [["mandel.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "mandel", [49.69370836624876, 49.908375018276274, 49.9978750012815, 50.26308336528018, 50.28408335056156, 50.34704168792814, 50.347625045105815, 50.36137503338978, 50.43837503762916, 50.445333356037736, 50.4743333440274, 50.49724999116734, 50.568124977871776, 50.613583356607705, 50.6137500051409, 50.65633333288133, 50.6677083321847, 50.67425000015646, 50.69620831636712, 50.698208331596106], 20]], [["n-body.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "n-body", [22.44608337059617, 22.47108332812786, 22.489875031169504, 22.5219166604802, 22.619333351030946, 22.627583297435194, 22.64491666574031, 22.645916673354805, 22.66033337218687, 22.661500028334558, 22.66687504015863, 22.693333332426846, 22.706416668370366, 22.71020831540227, 22.716999985277653, 22.731916687916964, 22.773958335164934, 22.77541666990146, 22.78762497007847, 22.79062499292195], 20]], [["qt.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "qt", [64.12537500727922, 64.41350001841784, 64.42187499487773, 64.86483331536874, 64.91604167968035, 64.99966664705426, 65.00508333556354, 65.0737916585058, 65.09579165140167, 65.24995830841362, 65.31358335632831, 65.43233332922682, 65.4332916601561, 65.45712501974776, 65.59733330504969, 65.61175000388175, 65.64608338521793, 65.67816663300619, 65.67824998637661, 65.76945830602199], 20]], [["queen.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "queen", [2.5195416528731585, 2.5283750146627426, 2.5407083448953927, 2.5465000071562827, 2.5687083252705634, 2.5962083018384874, 2.6028333231806755, 2.616083307657391, 2.6195416576229036, 2.6360000483691692, 2.6557500241324306, 2.6643750024959445, 2.6737916632555425, 2.6831250288523734, 2.6846666587516665, 2.6861666701734066, 2.692833309993148, 2.6956666843034327, 2.703041711356491, 2.707375038880855], 20]], [["scimark.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "scimark", [48.746791668236256, 48.90233330661431, 49.196333333384246, 49.22058334341273, 49.324041639920324, 49.66908332426101, 49.74995832890272, 49.88783336011693, 49.967916624154896, 49.97066664509475, 49.97662495588884, 49.98349997913465, 50.01345835626125, 50.02487497404218, 50.031499995384365, 50.10358331492171, 50.125000008847564, 50.14649999793619, 50.15662498772144, 50.18020834540948], 20]], [["spectral-norm.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "spectral-norm", [11.351291672326624, 11.361583368852735, 11.378791648894548, 11.424833326600492, 11.497541680000722, 11.52679166989401, 11.534041666891426, 11.568958288989961, 11.573916650377214, 11.585166677832603, 11.586708365939558, 11.5904999547638, 11.59370836103335, 11.60100003471598, 11.605666659306735, 11.606416665017605, 11.704875039868057, 11.72229170333594, 11.72775001032278, 11.746791657060385], 20]], [["sieve.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "sieve", [113.75387501902878, 114.15237496839836, 114.21591666294262, 114.55620831111446, 114.71291666384786, 114.79191662510857, 114.94066671002656, 115.13987497892231, 115.18345831427723, 115.64141669077799, 115.64741662004963, 115.69929169490933, 115.79524999251589, 115.80891668563709, 115.85295834811404, 115.87750003673136, 116.10787501558661, 116.27624998800457, 116.61220830865204, 116.74174998188391], 20]], [["3d-cube.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "3d-cube", [7.363374985288829, 7.378333306405693, 7.447833369951695, 7.455166662111878, 7.494333374779671, 7.530041679274291, 7.546374981757253, 7.553333300165832, 7.620041666086763, 7.650041661690921, 7.661458279471844, 7.6782083488069475, 7.7064583310857415, 7.714624982327223, 7.719874964095652, 7.725583331193775, 7.736166648101062, 7.7621666714549065, 7.778583327308297, 7.836958335246891], 20]], [["3d-morph.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "3d-morph", [5.9269999619573355, 6.098333338741213, 6.147375039290637, 6.2752499943599105, 6.280791654717177, 6.313375022727996, 6.322875036858022, 6.35020830668509, 6.3643333269283175, 6.372625008225441, 6.379083322826773, 6.38558337232098, 6.420624966267496, 6.4258333295583725, 6.460916658397764, 6.523624993860722, 6.545791693497449, 6.558208318892866, 6.586500036064535, 6.709458364639431], 20]], [["3d-raytrace.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "3d-raytrace", [7.2648749919608235, 7.375916698947549, 7.386750017758459, 7.392708328552544, 7.44479161221534, 7.457833329681307, 7.469833362847567, 7.4704583385027945, 7.477791688870639, 7.496083329897374, 7.500083360355347, 7.511291652917862, 7.5225416221655905, 7.53529166104272, 7.539749960415065, 7.5465000118128955, 7.572999980766326, 7.583458325825632, 7.589041662868112, 7.603041653055698], 20]], [["controlflow-recursive.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "controlflow-recursive", [4.20233333716169, 4.248375014867634, 4.279291664715856, 4.292708355933428, 4.29562502540648, 4.311166645493358, 4.323249973822385, 4.331833333708346, 4.334041674155742, 4.345791705418378, 4.358708334621042, 4.373041680082679, 4.37691668048501, 4.399958357680589, 4.426583298482001, 4.449958330951631, 4.459874995518476, 4.47729165898636, 4.4908333802595735, 4.515958309639245], 20]], [["crypto-aes.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "crypto-aes", [8.538541675079614, 8.609124983195215, 8.640625048428774, 8.694958291016519, 8.725500025320798, 8.749249973334372, 8.753666654229164, 8.771666674874723, 8.774249989073724, 8.829250000417233, 8.832749968860298, 8.83587502175942, 8.851291669998318, 8.892416663002223, 8.913625031709671, 8.931999967899173, 8.952041680458933, 8.957833342719823, 8.964583335909992, 8.970499970018864], 20]], [["fannkuch.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "fannkuch", [19.5814166800119, 19.708583364263177, 19.73800000268966, 19.74370831158012, 19.746208330616355, 19.755958346650004, 19.942874961998314, 19.974916649516672, 19.99408332630992, 20.00095829134807, 20.01670835306868, 20.025375008117408, 20.02595830708742, 20.051833358593285, 20.05779166938737, 20.09562501916662, 20.10541665367782, 20.144041685853153, 20.149833348114043, 20.163999986834824], 20]], [["math-cordic.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "math-cordic", [12.76850001886487, 12.785083323251456, 12.832666689064354, 12.845249962992966, 12.85300002200529, 12.893583334516734, 12.930583325214684, 12.954833335243165, 12.95841665705666, 12.962749984581023, 12.973958335351199, 12.97891663853079, 13.008666632231325, 13.03016667952761, 13.036666612606496, 13.04141664877534, 13.060958357527852, 13.07766669197008, 13.106916681863368, 13.107458362355828], 20]], [["math-partial-sums.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "math-partial-sums", [2.54025001777336, 2.570750017184764, 2.574333338998258, 2.5753750232979655, 2.576541679445654, 2.5984167004935443, 2.6029582950286567, 2.603833330795169, 2.6114583597518504, 2.628458372782916, 2.6353333378210664, 2.6410833816044033, 2.6411250000819564, 2.652499999385327, 2.664625004399568, 2.677166659850627, 2.6852083392441273, 2.699583361390978, 2.713083289563656, 2.72116664564237], 20]], [["n-body-oop.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "n-body-oop", [32.14116668095812, 32.16999996220693, 32.36300003482029, 32.562000036705285, 32.57362497970462, 32.60299999965355, 32.70300000440329, 32.72770834155381, 32.78412501094863, 32.8052916447632, 32.81008335761726, 32.81570831313729, 32.86733332788572, 32.88175002671778, 32.96041663270444, 32.96279167989269, 32.98041666857898, 32.98654162790626, 33.08433335041627, 33.095624996349216], 20]], [["tictactoe.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "tictactoe", [165.59583332855254, 165.75037501752377, 165.97825003555045, 166.0334583139047, 166.19912500027567, 166.27229168079793, 166.31524998228997, 166.8782916967757, 167.0542083447799, 167.12108335923404, 167.2098750132136, 167.28154162410647, 167.42200002772734, 167.61208337265998, 167.83062502508983, 167.8466250305064, 167.84666664898396, 168.02091663703322, 168.18170831538737, 168.3900416828692], 20]], [["trig.lua", "/Users/kingmaker/Dev/ThisDot/Roblox/luau/luau", "luau", "trig", [13.244458299595863, 13.25633330270648, 13.274208293296397, 13.308958325069398, 13.33708333550021, 13.346791674848646, 13.359041709918529, 13.36250000167638, 13.395041693001986, 13.449583319015801, 13.450708298478276, 13.467041659168899, 13.513333280570805, 13.520708307623863, 13.548416667617857, 13.605958316475153, 13.658416632097214, 13.692333304788917, 13.701208343263716, 13.737208326347172], 20]]]
diff --git a/bench.png b/bench.png
diff --git a/bench/__pycache__/color.cpython-39.pyc b/bench/__pycache__/color.cpython-39.pyc
diff --git a/bench/__pycache__/influxbench.cpython-39.pyc b/bench/__pycache__/influxbench.cpython-39.pyc
diff --git a/bench/__pycache__/tabulate.cpython-39.pyc b/bench/__pycache__/tabulate.cpython-39.pyc