Merge pull request #7786 from plotly/cam/7785/check-regl-ci

camdecoster · web-flow · commit 8965c3ded8d8 · 2026-06-03T08:34:54.000-06:00
ci: Add CI step to check if precompiled regl shaders need to be updated
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -23,6 +23,7 @@ jobs:
     outputs:
       stackgl_modules: ${{ steps.check.outputs.stackgl_modules }}
       topojson: ${{ steps.check.outputs.topojson }}
+      regl_codegen: ${{ steps.check.outputs.regl_codegen }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
         with:
@@ -42,6 +43,12 @@ jobs:
           if git diff --name-only "$BASE"...HEAD -- stackgl_modules/ | grep -q .; then
             echo "stackgl_modules=true" >> "$GITHUB_OUTPUT"
           fi
+          # If any of the directories listed below have changed, we need to re-run the regl-codegen step
+          # stackgl_modules/ is listed here too because the regl-* shader libs live there;
+          # changes can alter shader output and require regenerating the precompiled shaders.
+          if git diff --name-only "$BASE"...HEAD -- src/traces/scattergl/ src/traces/scatterpolargl/ src/traces/splom/ src/traces/parcoords/ src/lib/prepare_regl.js stackgl_modules/ devtools/regl_codegen/ | grep -q .; then
+            echo "regl_codegen=true" >> "$GITHUB_OUTPUT"
+          fi
 
   # ============================================================
   # Root build job - all dependent jobs fan out from here
@@ -636,3 +643,54 @@ jobs:
           name: topojson-dist
           retention-days: 7
           path: topojson/dist/
+
+  check-regl-codegen:
+    needs: [detect-changes, install-and-cibuild]
+    if: >-
+      (github.event_name == 'push' && github.ref_name == github.event.repository.default_branch) ||
+      needs.detect-changes.outputs.regl_codegen == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      - uses: ./.github/actions/setup-workspace
+      - uses: ./.github/actions/setup-chrome
+
+      - name: Run regl codegen
+        run: |
+          # Wipe the codegen output dir so orphaned shader files surface as deletions in the diff below.
+          rm -rf src/generated/regl-codegen/*
+          node devtools/regl_codegen/server.mjs &
+          SERVER_PID=$!
+          until curl -sf -o /dev/null http://localhost:3000/build/regl_codegen-bundle.js 2>/dev/null; do
+            sleep 1
+          done
+          $CHROME_BIN --headless=new --no-sandbox --enable-unsafe-swiftshader --ignore-gpu-blocklist http://localhost:3000/devtools/regl_codegen/index.html &
+          wait $SERVER_PID
+
+      - name: Check regl precompiled shaders are up to date
+        run: |
+          # git add -N so newly-generated (untracked) shader files show up in git diff
+          git add -N src/generated/regl-codegen/
+          if ! git diff --exit-code \
+              src/generated/regl-codegen/ \
+              src/traces/scattergl/regl_precompiled.js \
+              src/traces/scatterpolargl/regl_precompiled.js \
+              src/traces/splom/regl_precompiled.js \
+              src/traces/parcoords/regl_precompiled.js; then
+            echo "::error::Regl precompiled shaders are out of date. Download the 'regl-codegen' artifact from this workflow run, extract it into src/, and commit the updated files to this pull request."
+            exit 1
+          fi
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        name: Upload updated regl codegen files
+        if: failure()
+        with:
+          name: regl-codegen
+          retention-days: 7
+          path: |
+            src/generated/regl-codegen/
+            src/traces/scattergl/regl_precompiled.js
+            src/traces/scatterpolargl/regl_precompiled.js
+            src/traces/splom/regl_precompiled.js
+            src/traces/parcoords/regl_precompiled.js
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -163,19 +163,54 @@ npm run schema
 #### Step 9: REGL - Review & commit potential changes to precompiled regl shaders
 
 If you are implementing a new feature that involves regl shaders, or if you are
-making changes that affect the usage of regl shaders, you would need to run
+making changes that affect the usage of regl shaders, you will need to regenerate the precompiled regl shader code. In practice, that means edits under:
 
-```bash
-npm run regl-codegen
-```
+- `src/traces/{scattergl,scatterpolargl,splom,parcoords}/`
+- `src/lib/prepare_regl.js`
+- `stackgl_modules/` (where the `regl-*` shader libs live)
+- `devtools/regl_codegen/`
 
-to regenerate the regl code. This will prompt you to open a browser window. This will then run through all
-traces with 'regl' in the tags, and store the captured code into 
-[src/generated/regl-codegen](https://github.com/plotly/plotly.js/blob/master/src/generated/regl-codegen). If no updates are necessary, it will be a no-op, but if there are changes, you will need to commit them.
+CI's `check-regl-codegen` job uses this same list as its trigger — see [`.github/workflows/ci.yml`](.github/workflows/ci.yml) for the authoritative version.
 
 This is needed because regl performs codegen in runtime which breaks CSP
 compliance, and so for strict builds we pre-generate regl shader code here.
 
+The CI pipeline will automatically detect when regl-related files have changed and
+run the codegen process. If the precompiled shaders are out of date, the
+`check-regl-codegen` job will fail and upload a `regl-codegen` artifact containing
+the updated files. The artifact represents the full desired state of the codegen
+output (CI wipes the output directory before regenerating, so any orphaned shader
+files are pruned). To fix this:
+
+1. Download the `regl-codegen` artifact from the failed workflow run
+2. Delete `src/generated/regl-codegen/` in your working tree, then unzip the
+   artifact into `src/` so it replaces (rather than merges into) the existing
+   directory. Note that `actions/upload-artifact` strips the longest common
+   parent path from the uploaded paths, so the zip's root contains
+   `generated/regl-codegen/` and `traces/{scattergl,scatterpolargl,splom,parcoords}/regl_precompiled.js`
+   directly — extracting into `src/` restores the correct layout and
+   overwrites the four `regl_precompiled.js` files in one step.
+3. Commit and push the changes to your pull request
+
+Alternatively, you can regenerate the code locally:
+
+```bash
+rm -rf src/generated/regl-codegen/*
+npm run regl-codegen
+```
+
+The `rm -rf` step is needed to clean up any orphaned shader files left over from
+previous changes. `npm run regl-codegen` will prompt you to open
+a browser window, run through the mocks for each regl-using trace
+(`parcoords`, `scattergl`, `scatterpolargl`, `splom`), and store the captured
+shader code into
+[src/generated/regl-codegen](https://github.com/plotly/plotly.js/blob/master/src/generated/regl-codegen).
+The four `src/traces/{parcoords,scattergl,scatterpolargl,splom}/regl_precompiled.js`
+files are rewritten in the same pass so their imports point at the freshly
+generated shader files. Commit any changes that result — both the
+`src/generated/regl-codegen/` updates and any modified `regl_precompiled.js`
+files.
+
 #### Other npm scripts that may be of interest in development
 
 - `npm run preprocess`: pre-processes the css and svg source file in js. This
diff --git a/devtools/regl_codegen/server.mjs b/devtools/regl_codegen/server.mjs
@@ -85,16 +85,20 @@ function handleCodegen(data) {
     var exports = ['', '/* eslint-disable quote-props */', 'module.exports = {', ''].join('\n');
     var varId = 0;
 
-    Object.entries(generated).forEach(function (kv) {
-        var key = kv[0];
-        var value = kv[1];
-        var filePath = path.join(pathToReglCodegenSrc, key);
-        fs.writeFileSync(filePath, 'module.exports = ' + value);
-
-        imports += 'var v' + varId + " = require('../../" + path.join(constants.reglCodegenSubdir, key) + "');\n";
-        exports += "    '" + key + "': v" + varId + ',\n';
-        varId++;
-    });
+    // Sort by shader-hash key so the emitted import/export order is deterministic
+    // across machines (fs.readdir mock order varies by OS/filesystem, which would
+    // otherwise leak into regl_precompiled.js).
+    Object.keys(generated)
+        .sort()
+        .forEach((key) => {
+            var value = generated[key];
+            var filePath = path.join(pathToReglCodegenSrc, key);
+            fs.writeFileSync(filePath, 'module.exports = ' + value);
+
+            imports += 'var v' + varId + " = require('../../" + path.join(constants.reglCodegenSubdir, key) + "');\n";
+            exports += "    '" + key + "': v" + varId + ',\n';
+            varId++;
+        });
 
     if (varId > 0) {
         exports = exports.slice(0, -2) + '\n};\n';
diff --git a/src/traces/parcoords/regl_precompiled.js b/src/traces/parcoords/regl_precompiled.js
@@ -1,13 +1,13 @@
 'use strict';
-var v0 = require('../../generated/regl-codegen/453a70fefa48db31713162aeb1ac438cb8579f54504f3b23acf32128df3dfd45');
-var v1 = require('../../generated/regl-codegen/30680f8f6712ef1af5cf7547e0af35b036fb300c67b07967cf448492ff4de4d0');
-var v2 = require('../../generated/regl-codegen/a3970baf1d8cac9305ee830c7026550387343d4dde2353dd86a4d082c97d3470');
-var v3 = require('../../generated/regl-codegen/3fd666968f3ce90d1c048b7a9aab515f3ce387a5401a10f8b66121c9469d1c0d');
+var v0 = require('../../generated/regl-codegen/30680f8f6712ef1af5cf7547e0af35b036fb300c67b07967cf448492ff4de4d0');
+var v1 = require('../../generated/regl-codegen/3fd666968f3ce90d1c048b7a9aab515f3ce387a5401a10f8b66121c9469d1c0d');
+var v2 = require('../../generated/regl-codegen/453a70fefa48db31713162aeb1ac438cb8579f54504f3b23acf32128df3dfd45');
+var v3 = require('../../generated/regl-codegen/a3970baf1d8cac9305ee830c7026550387343d4dde2353dd86a4d082c97d3470');
 
 /* eslint-disable quote-props */
 module.exports = {
-    '453a70fefa48db31713162aeb1ac438cb8579f54504f3b23acf32128df3dfd45': v0,
-    '30680f8f6712ef1af5cf7547e0af35b036fb300c67b07967cf448492ff4de4d0': v1,
-    'a3970baf1d8cac9305ee830c7026550387343d4dde2353dd86a4d082c97d3470': v2,
-    '3fd666968f3ce90d1c048b7a9aab515f3ce387a5401a10f8b66121c9469d1c0d': v3
+    '30680f8f6712ef1af5cf7547e0af35b036fb300c67b07967cf448492ff4de4d0': v0,
+    '3fd666968f3ce90d1c048b7a9aab515f3ce387a5401a10f8b66121c9469d1c0d': v1,
+    '453a70fefa48db31713162aeb1ac438cb8579f54504f3b23acf32128df3dfd45': v2,
+    'a3970baf1d8cac9305ee830c7026550387343d4dde2353dd86a4d082c97d3470': v3
 };
diff --git a/src/traces/scattergl/regl_precompiled.js b/src/traces/scattergl/regl_precompiled.js
@@ -1,25 +1,25 @@
 'use strict';
 var v0 = require('../../generated/regl-codegen/3e771157d23b4793771f65d83e6387262ed73d488209157f19a7fa027bddd71b');
-var v1 = require('../../generated/regl-codegen/cbf700f001fff25b649fba9c37fa0dc6631c1cdee318ad49473d28ec10dcee81');
-var v2 = require('../../generated/regl-codegen/8fad2284703471df7c0e0d0a7b96d983e8c53f6d707dd55d5921c1eab71f6623');
-var v3 = require('../../generated/regl-codegen/fe5b6844077cde1bdd7273f4495969fad93500c26a69b62e74ec2664c447bcc7');
-var v4 = require('../../generated/regl-codegen/db1b82c68771e7f5012fad1fbdae7ff23b526e58d2995bf6dd2cf30024e0f41d');
-var v5 = require('../../generated/regl-codegen/49e82bba439f1d9d441c17ba252d05640bc63fefdf22d1219993633af7730210');
-var v6 = require('../../generated/regl-codegen/dbd1cc9126a137a605df67dc0706e55116f04e33b4545a80042031752de5aef5');
-var v7 = require('../../generated/regl-codegen/bfc540da96a87fcc039073cb37b45e6b81ef5ee6ef3529d726ceed8336354019');
-var v8 = require('../../generated/regl-codegen/6a5d6bd29c15cf7614221b94c3f384df47c2c46fbe4456e8c57b5cd14c84d923');
-var v9 = require('../../generated/regl-codegen/8902aff2b23b600f8103bcc84a8af2999d28795208aedadc2db06f921f9c7034');
+var v1 = require('../../generated/regl-codegen/49e82bba439f1d9d441c17ba252d05640bc63fefdf22d1219993633af7730210');
+var v2 = require('../../generated/regl-codegen/6a5d6bd29c15cf7614221b94c3f384df47c2c46fbe4456e8c57b5cd14c84d923');
+var v3 = require('../../generated/regl-codegen/8902aff2b23b600f8103bcc84a8af2999d28795208aedadc2db06f921f9c7034');
+var v4 = require('../../generated/regl-codegen/8fad2284703471df7c0e0d0a7b96d983e8c53f6d707dd55d5921c1eab71f6623');
+var v5 = require('../../generated/regl-codegen/bfc540da96a87fcc039073cb37b45e6b81ef5ee6ef3529d726ceed8336354019');
+var v6 = require('../../generated/regl-codegen/cbf700f001fff25b649fba9c37fa0dc6631c1cdee318ad49473d28ec10dcee81');
+var v7 = require('../../generated/regl-codegen/db1b82c68771e7f5012fad1fbdae7ff23b526e58d2995bf6dd2cf30024e0f41d');
+var v8 = require('../../generated/regl-codegen/dbd1cc9126a137a605df67dc0706e55116f04e33b4545a80042031752de5aef5');
+var v9 = require('../../generated/regl-codegen/fe5b6844077cde1bdd7273f4495969fad93500c26a69b62e74ec2664c447bcc7');
 
 /* eslint-disable quote-props */
 module.exports = {
     '3e771157d23b4793771f65d83e6387262ed73d488209157f19a7fa027bddd71b': v0,
-    'cbf700f001fff25b649fba9c37fa0dc6631c1cdee318ad49473d28ec10dcee81': v1,
-    '8fad2284703471df7c0e0d0a7b96d983e8c53f6d707dd55d5921c1eab71f6623': v2,
-    'fe5b6844077cde1bdd7273f4495969fad93500c26a69b62e74ec2664c447bcc7': v3,
-    'db1b82c68771e7f5012fad1fbdae7ff23b526e58d2995bf6dd2cf30024e0f41d': v4,
-    '49e82bba439f1d9d441c17ba252d05640bc63fefdf22d1219993633af7730210': v5,
-    'dbd1cc9126a137a605df67dc0706e55116f04e33b4545a80042031752de5aef5': v6,
-    'bfc540da96a87fcc039073cb37b45e6b81ef5ee6ef3529d726ceed8336354019': v7,
-    '6a5d6bd29c15cf7614221b94c3f384df47c2c46fbe4456e8c57b5cd14c84d923': v8,
-    '8902aff2b23b600f8103bcc84a8af2999d28795208aedadc2db06f921f9c7034': v9
+    '49e82bba439f1d9d441c17ba252d05640bc63fefdf22d1219993633af7730210': v1,
+    '6a5d6bd29c15cf7614221b94c3f384df47c2c46fbe4456e8c57b5cd14c84d923': v2,
+    '8902aff2b23b600f8103bcc84a8af2999d28795208aedadc2db06f921f9c7034': v3,
+    '8fad2284703471df7c0e0d0a7b96d983e8c53f6d707dd55d5921c1eab71f6623': v4,
+    'bfc540da96a87fcc039073cb37b45e6b81ef5ee6ef3529d726ceed8336354019': v5,
+    'cbf700f001fff25b649fba9c37fa0dc6631c1cdee318ad49473d28ec10dcee81': v6,
+    'db1b82c68771e7f5012fad1fbdae7ff23b526e58d2995bf6dd2cf30024e0f41d': v7,
+    'dbd1cc9126a137a605df67dc0706e55116f04e33b4545a80042031752de5aef5': v8,
+    'fe5b6844077cde1bdd7273f4495969fad93500c26a69b62e74ec2664c447bcc7': v9
 };
diff --git a/src/traces/scatterpolargl/regl_precompiled.js b/src/traces/scatterpolargl/regl_precompiled.js
@@ -1,25 +1,25 @@
 'use strict';
 var v0 = require('../../generated/regl-codegen/3e771157d23b4793771f65d83e6387262ed73d488209157f19a7fa027bddd71b');
-var v1 = require('../../generated/regl-codegen/cbf700f001fff25b649fba9c37fa0dc6631c1cdee318ad49473d28ec10dcee81');
-var v2 = require('../../generated/regl-codegen/8fad2284703471df7c0e0d0a7b96d983e8c53f6d707dd55d5921c1eab71f6623');
-var v3 = require('../../generated/regl-codegen/fe5b6844077cde1bdd7273f4495969fad93500c26a69b62e74ec2664c447bcc7');
-var v4 = require('../../generated/regl-codegen/db1b82c68771e7f5012fad1fbdae7ff23b526e58d2995bf6dd2cf30024e0f41d');
-var v5 = require('../../generated/regl-codegen/49e82bba439f1d9d441c17ba252d05640bc63fefdf22d1219993633af7730210');
-var v6 = require('../../generated/regl-codegen/dbd1cc9126a137a605df67dc0706e55116f04e33b4545a80042031752de5aef5');
-var v7 = require('../../generated/regl-codegen/bfc540da96a87fcc039073cb37b45e6b81ef5ee6ef3529d726ceed8336354019');
-var v8 = require('../../generated/regl-codegen/6a5d6bd29c15cf7614221b94c3f384df47c2c46fbe4456e8c57b5cd14c84d923');
-var v9 = require('../../generated/regl-codegen/8902aff2b23b600f8103bcc84a8af2999d28795208aedadc2db06f921f9c7034');
+var v1 = require('../../generated/regl-codegen/49e82bba439f1d9d441c17ba252d05640bc63fefdf22d1219993633af7730210');
+var v2 = require('../../generated/regl-codegen/6a5d6bd29c15cf7614221b94c3f384df47c2c46fbe4456e8c57b5cd14c84d923');
+var v3 = require('../../generated/regl-codegen/8902aff2b23b600f8103bcc84a8af2999d28795208aedadc2db06f921f9c7034');
+var v4 = require('../../generated/regl-codegen/8fad2284703471df7c0e0d0a7b96d983e8c53f6d707dd55d5921c1eab71f6623');
+var v5 = require('../../generated/regl-codegen/bfc540da96a87fcc039073cb37b45e6b81ef5ee6ef3529d726ceed8336354019');
+var v6 = require('../../generated/regl-codegen/cbf700f001fff25b649fba9c37fa0dc6631c1cdee318ad49473d28ec10dcee81');
+var v7 = require('../../generated/regl-codegen/db1b82c68771e7f5012fad1fbdae7ff23b526e58d2995bf6dd2cf30024e0f41d');
+var v8 = require('../../generated/regl-codegen/dbd1cc9126a137a605df67dc0706e55116f04e33b4545a80042031752de5aef5');
+var v9 = require('../../generated/regl-codegen/fe5b6844077cde1bdd7273f4495969fad93500c26a69b62e74ec2664c447bcc7');
 
 /* eslint-disable quote-props */
 module.exports = {
     '3e771157d23b4793771f65d83e6387262ed73d488209157f19a7fa027bddd71b': v0,
-    'cbf700f001fff25b649fba9c37fa0dc6631c1cdee318ad49473d28ec10dcee81': v1,
-    '8fad2284703471df7c0e0d0a7b96d983e8c53f6d707dd55d5921c1eab71f6623': v2,
-    'fe5b6844077cde1bdd7273f4495969fad93500c26a69b62e74ec2664c447bcc7': v3,
-    'db1b82c68771e7f5012fad1fbdae7ff23b526e58d2995bf6dd2cf30024e0f41d': v4,
-    '49e82bba439f1d9d441c17ba252d05640bc63fefdf22d1219993633af7730210': v5,
-    'dbd1cc9126a137a605df67dc0706e55116f04e33b4545a80042031752de5aef5': v6,
-    'bfc540da96a87fcc039073cb37b45e6b81ef5ee6ef3529d726ceed8336354019': v7,
-    '6a5d6bd29c15cf7614221b94c3f384df47c2c46fbe4456e8c57b5cd14c84d923': v8,
-    '8902aff2b23b600f8103bcc84a8af2999d28795208aedadc2db06f921f9c7034': v9
+    '49e82bba439f1d9d441c17ba252d05640bc63fefdf22d1219993633af7730210': v1,
+    '6a5d6bd29c15cf7614221b94c3f384df47c2c46fbe4456e8c57b5cd14c84d923': v2,
+    '8902aff2b23b600f8103bcc84a8af2999d28795208aedadc2db06f921f9c7034': v3,
+    '8fad2284703471df7c0e0d0a7b96d983e8c53f6d707dd55d5921c1eab71f6623': v4,
+    'bfc540da96a87fcc039073cb37b45e6b81ef5ee6ef3529d726ceed8336354019': v5,
+    'cbf700f001fff25b649fba9c37fa0dc6631c1cdee318ad49473d28ec10dcee81': v6,
+    'db1b82c68771e7f5012fad1fbdae7ff23b526e58d2995bf6dd2cf30024e0f41d': v7,
+    'dbd1cc9126a137a605df67dc0706e55116f04e33b4545a80042031752de5aef5': v8,
+    'fe5b6844077cde1bdd7273f4495969fad93500c26a69b62e74ec2664c447bcc7': v9
 };
diff --git a/src/traces/splom/regl_precompiled.js b/src/traces/splom/regl_precompiled.js
