Skip to content

Make semaphore pipeline run less frequently, reduce machine sizes, avoid clearing cache #362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Sep 25, 2025
Merged

Make semaphore pipeline run less frequently, reduce machine sizes, avoid clearing cache #362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3c687eb
Make semaphore pipeline run less frequently, reduce machine sizes, av…
milindl Sep 2, 2025
3528b03
Fix change_in
milindl Sep 2, 2025
3c1c48a
Add pipeline ignore temporarily to change in
milindl Sep 2, 2025
ce4657c
Test changes for change_in pipeline
milindl Sep 2, 2025
d39013b
Test changes for change_in pipeline - 2
milindl Sep 2, 2025
c57409c
Remove /deps - change in deps will be change in util.js also, so it's…
milindl Sep 2, 2025
9d9e06d
Don't make pipelines run repeatedly on master when they're running on…
milindl Sep 2, 2025
8fbe5d3
Remove testing changes
milindl Sep 2, 2025
f494729
Add cache store/restore
milindl Sep 15, 2025
97077a5
Add cache store/restore - fix
milindl Sep 15, 2025
109edec
Add cache store/restore - if for public
milindl Sep 15, 2025
5c1ddbd
Remove unrequired workflows and use npm ci for alpine
milindl Sep 16, 2025
cce0e63
Use commit hash rather than version to avoid cloning
milindl Sep 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 0 additions & 25 deletions .github/workflows/npm-publish.yml
View file
Open in desktop

This file was deleted.

43 changes: 0 additions & 43 deletions .github/workflows/test.yml
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion .semaphore/build-docker-alpine.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,5 @@
apk add -U ca-certificates openssl ncurses coreutils python3 make gcc g++ libgcc linux-headers grep util-linux binutils findutils perl patch musl-dev bash
# /v is the volume mount point for the project root
cd /v
npm install
npm ci
npx node-pre-gyp package
42 changes: 29 additions & 13 deletions .semaphore/semaphore.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,22 @@ global_job_config:
prologue:
commands:
- checkout
- export LIBRDKAFKA_COMMIT_HASH="librdkafka_$(git submodule status deps/librdkafka | cut -d' ' -f1)"
- if [[ $(command cache) ]]; then cache restore "$LIBRDKAFKA_COMMIT_HASH" ; fi
- git submodule update --init --recursive
- cd deps/librdkafka
- git fetch origin
- git checkout v2.11.1
- cd ../../
- cache clear
epilogue:
commands:
- if [[ $(command cache) ]]; then cache store "$LIBRDKAFKA_COMMIT_HASH" deps/librdkafka; done

blocks:
- name: "Linux amd64 (musl): Build and test"
run:
when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'"
dependencies: [ ]
task:
agent:
machine:
type: s1-prod-ubuntu24-04-amd64-2
type: s1-prod-ubuntu24-04-amd64-1
prologue:
commands:
- '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
Expand All @@ -48,6 +50,8 @@ blocks:
- docker run -v "$(pwd):/v" node:18-alpine /v/.semaphore/build-docker-alpine.sh

- name: "Linux arm64 (musl): Build and test"
run:
when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'"
dependencies: [ ]
task:
agent:
Expand All @@ -61,12 +65,14 @@ blocks:
commands:
- docker run -v "$(pwd):/v" node:18-alpine /v/.semaphore/build-docker-alpine.sh

- name: "Linux arm64: Build and test"
- name: "Linux amd64: Build and test"
run:
when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'"
dependencies: [ ]
task:
agent:
machine:
type: s1-prod-ubuntu24-04-arm64-1
type: s1-prod-ubuntu24-04-amd64-1
jobs:
- name: "Build from source and test"
commands:
Expand All @@ -75,6 +81,8 @@ blocks:
- make test

- name: 'macOS arm64/m1: Build and test'
run:
when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'"
dependencies: []
task:
agent:
Expand All @@ -87,16 +95,20 @@ blocks:
- npm run install-from-source
- make test

- name: "Linux amd64: Build, test, lint, docs"
- name: "Linux arm64: Build, test, lint, docs"
skip:
when: "tag =~ '^v[0-9]\\.'"
dependencies: [ ]
task:
agent:
machine:
type: s1-prod-ubuntu24-04-amd64-2
type: s1-prod-ubuntu24-04-arm64-1
prologue:
commands:
- export CKJS_LINKING=dynamic
- if [[ $(command cache) ]]; then cache restore; fi
- npm run install-from-source
- if [[ $(command cache) ]]; then cache store; fi
jobs:
- name: "Test"
commands:
Expand Down Expand Up @@ -125,11 +137,13 @@ blocks:
- make docs

- name: "(SR) Linux amd64: test, e2e"
run:
when: "change_in(['/package.json', '/schemaregistry', '/schemaregistry-examples', '/proto']) AND tag !~ '^v[0-9]\\.'"
dependencies: [ ]
task:
agent:
machine:
type: s1-prod-ubuntu24-04-amd64-2
type: s1-prod-ubuntu24-04-amd64-1
prologue:
commands:
- npm install
Expand All @@ -146,6 +160,8 @@ blocks:
- make integtest

- name: "Linux amd64: Performance"
run:
when: "change_in(['/package.json', '/lib', '/src', '/ci/tests']) AND tag !~ '^v[0-9]\\.'"
dependencies: [ ]
task:
agent:
Expand Down Expand Up @@ -188,7 +204,7 @@ blocks:
task:
agent:
machine:
type: s1-prod-ubuntu24-04-amd64-2
type: s1-prod-ubuntu24-04-amd64-1
env_vars:
- name: ARCHITECTURE
value: "x64"
Expand Down Expand Up @@ -476,7 +492,7 @@ blocks:
task:
agent:
machine:
type: s1-prod-ubuntu24-04-amd64-2
type: s1-prod-ubuntu24-04-amd64-1
jobs:
- name: "Tarball"
commands:
Expand Down