Skip to content

Commit b9d348f

Browse files
timothypratleytimothypratley
committed
add separate incremental build via cache-repo
1 parent 47d04bd commit b9d348f

File tree

2 files changed

+157
-1
lines changed

2 files changed

+157
-1
lines changed

‎.github/workflows/render-and-publish-incremental.yml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
on:
2+
push:
3+
branches:
4+
- main
5+
workflow_dispatch:
6+
inputs:
7+
publish:
8+
description: "Deploy to Pages (manual runs only)"
9+
type: boolean
10+
default: false
11+
12+
name: Incremental Render and Publish
13+
14+
permissions:
15+
contents: read
16+
17+
env:
18+
SITE_CACHE_REPO: ${{ secrets.SITE_CACHE_REPO }}
19+
SITE_CACHE_TOKEN: ${{ secrets.SITE_CACHE_TOKEN }}
20+
21+
jobs:
22+
build:
23+
runs-on: ubuntu-latest
24+
25+
steps:
26+
- name: Check out repository
27+
uses: actions/checkout@v4
28+
with:
29+
fetch-depth: 0 # need history to get mtimes
30+
31+
- name: Normalize source repo file mtimes to last commit
32+
run: |
33+
set -euo pipefail
34+
git ls-files -z | xargs -0 -I{} sh -c 'ts=$(git log -1 --format=%ct -- "{}" || true); if [ -n "$ts" ]; then touch -d "@$ts" "{}"; fi'
35+
36+
- name: Restore `site` from cache repo without overwriting source controlled files
37+
run: |
38+
set -euo pipefail
39+
git clone "https://x-access-token:${SITE_CACHE_TOKEN}@github.com/${SITE_CACHE_REPO}.git" /tmp/site-cache
40+
cd /tmp/site-cache
41+
git ls-files -z -- site/ | xargs -0 -I{} sh -c 'ts=$(git log -1 --format=%ct -- "{}" || true); if [ -n "$ts" ]; then touch -d "@$ts" "{}"; fi'
42+
cd -
43+
rsync -a --ignore-existing /tmp/site-cache/site/ site/
44+
echo "Contents of site/ after restore:"
45+
ls -lah site/ | head -50
46+
47+
- name: Set up Python
48+
uses: actions/setup-python@v5
49+
with:
50+
python-version: '3.10'
51+
- name: Install dependencies
52+
run: |
53+
pip install plotly kaleido
54+
55+
- name: Set up Java
56+
uses: actions/setup-java@v4
57+
with:
58+
distribution: 'temurin'
59+
java-version: '21'
60+
61+
- name: Set up Clojure
62+
uses: DeLaGuardo/setup-clojure@main
63+
with:
64+
cli: 'latest'
65+
66+
- name: Cache clojure dependencies
67+
uses: actions/cache@v3
68+
with:
69+
path: |
70+
~/.m2/repository
71+
~/.gitlibs
72+
~/.deps.clj
73+
key: cljdeps-${{ runner.os }}
74+
75+
- name: Detect changed Clojure sources
76+
id: changed_clj
77+
run: |
78+
set -euo pipefail
79+
changed=""
80+
if [ -f /tmp/site-cache/SOURCE_COMMIT ]; then
81+
source_commit=$(cat /tmp/site-cache/SOURCE_COMMIT)
82+
echo "Using source commit recorded in cache: $source_commit"
83+
changed=$(git diff --name-only --diff-filter=ACMR "$source_commit" HEAD -- 'src/**/*.clj' | tr '\n' ' ')
84+
fi
85+
if [ -z "$changed" ]; then
86+
echo "No changed Clojure sources detected"
87+
exit 0
88+
fi
89+
echo "Changed Clojure source files:" $changed
90+
echo "files=$changed" >> "$GITHUB_OUTPUT"
91+
92+
- name: Build notebooks
93+
run: clojure -M:clay -A:markdown ${{ steps.changed_clj.outputs.files }}
94+
95+
- name: Set up Quarto
96+
uses: quarto-dev/quarto-actions/setup@v2
97+
with:
98+
tinytex: true
99+
100+
- name: Render Quarto
101+
uses: quarto-dev/quarto-actions/render@v2
102+
with:
103+
path: site
104+
105+
- name: Upload artifact for Pages
106+
uses: actions/upload-pages-artifact@v3
107+
with:
108+
path: site/_site
109+
110+
- name: Push site to cache repo
111+
run: |
112+
set -euo pipefail
113+
source_commit=$(git rev-parse HEAD)
114+
rm -rf /tmp/site-cache/site
115+
cp -r site /tmp/site-cache/
116+
echo "$source_commit" > /tmp/site-cache/SOURCE_COMMIT
117+
cd /tmp/site-cache
118+
git config user.email "github-actions@github.com"
119+
git config user.name "GitHub Actions"
120+
git add -A
121+
if ! git diff --cached --quiet; then
122+
git commit -m "Cache update from build ${source_commit}"
123+
git checkout -b temp
124+
git push --force origin main
125+
else
126+
echo "No changes to cache."
127+
fi
128+
129+
deploy:
130+
needs: build
131+
if: (github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' && github.ref == 'refs/heads/main')
132+
permissions:
133+
pages: write
134+
id-token: write
135+
environment:
136+
name: github-pages
137+
url: ${{ steps.deployment.outputs.page_url }}
138+
runs-on: ubuntu-latest
139+
steps:
140+
- name: Deploy to GitHub Pages
141+
id: deployment
142+
uses: actions/deploy-pages@v4

‎README.md‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,21 @@ Goal: Minimize friction in authoring while ensuring publishable reproducibility.
308308

309309
## Deployment
310310

311-
See [.github/workflows/render-and-publish.yml](.github/workflows/render-and-publish.yml)
311+
The site is built and deployed using GitHub Actions with two workflows:
312+
313+
- **Full Build and Publish**: Triggered on pushes to `main`.
314+
Rebuilds all notebooks with Clay, renders the entire site with Quarto, and deploys to GitHub Pages.
315+
See [.github/workflows/render-and-publish.yml](.github/workflows/render-and-publish.yml).
316+
317+
- **Incremental Build**: Triggered on pushes to `main`.
318+
Uses a separate cache repository to enable partial rebuilds.
319+
Restores the cached site (containing rendered HTML and generated files) without overwriting source-controlled files in `site/`.
320+
Sets file mtimes of source and cache files based on the last commit so that Quarto will know if the cached file is newer than the source.
321+
Compares the current source commit to the last build source commit (stored in the cache) to detect changes in Clojure sources.
322+
Rebuilds affected notebooks with Clay (rebuilds all if no changes detected).
323+
Renders the site incrementally with Quarto.
324+
Replaces the cache with the new site, and writes the source commit in a file in the cache.
325+
See [.github/workflows/render-and-publish-incremental.yml](.github/workflows/render-and-publish-incremental.yml).
312326

313327
## License
314328

0 commit comments

Comments
 (0)