From 3c687eb32f13343a94fef73d374c4c6c9ce3f960 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 2 Sep 2025 04:16:01 +0000 Subject: [PATCH 01/13] Make semaphore pipeline run less frequently, reduce machine sizes, avoid clearing cache --- .semaphore/semaphore.yml | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index a17da647..9208c286 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -30,15 +30,16 @@ global_job_config: - git fetch origin - git checkout v2.11.1 - cd ../../ - - cache clear blocks: - name: "Linux amd64 (musl): Build and test" + run: + when: "change_in('['/package.json', '/deps', '/lib', '/src', '/test']') AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: machine: - type: s1-prod-ubuntu24-04-amd64-2 + type: s1-prod-ubuntu24-04-amd64-1 prologue: commands: - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY' @@ -48,6 +49,8 @@ blocks: - docker run -v "$(pwd):/v" node:18-alpine /v/.semaphore/build-docker-alpine.sh - name: "Linux arm64 (musl): Build and test" + run: + when: "change_in('['/package.json', '/deps', '/lib', '/src', '/test']') AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -61,12 +64,14 @@ blocks: commands: - docker run -v "$(pwd):/v" node:18-alpine /v/.semaphore/build-docker-alpine.sh - - name: "Linux arm64: Build and test" + - name: "Linux amd64: Build and test" + run: + when: "change_in('['/package.json', '/deps', '/lib', '/src', '/test']') AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: machine: - type: s1-prod-ubuntu24-04-arm64-1 + type: s1-prod-ubuntu24-04-amd64-1 jobs: - name: "Build from source and test" commands: @@ -75,6 +80,8 @@ blocks: - make test - name: 'macOS arm64/m1: Build and test' + run: + when: "change_in('['/package.json', '/deps', '/lib', '/src', '/test']') AND tag !~ '^v[0-9]\\.'" dependencies: [] task: agent: @@ -87,12 +94,14 @@ blocks: - npm run install-from-source - make test - - name: "Linux amd64: Build, test, lint, docs" + - name: "Linux arm64: Build, test, lint, docs" + skip: + when: "tag =~ '^v[0-9]\\.'" dependencies: [ ] task: agent: machine: - type: s1-prod-ubuntu24-04-amd64-2 + type: s1-prod-ubuntu24-04-arm64-1 prologue: commands: - export CKJS_LINKING=dynamic @@ -125,11 +134,13 @@ blocks: - make docs - name: "(SR) Linux amd64: test, e2e" + run: + when: "change_in('['/package.json', '/schemaregistry', '/schemaregistry-examples', '/proto') AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: machine: - type: s1-prod-ubuntu24-04-amd64-2 + type: s1-prod-ubuntu24-04-amd64-1 prologue: commands: - npm install @@ -146,6 +157,8 @@ blocks: - make integtest - name: "Linux amd64: Performance" + run: + when: "change_in('['/package.json', '/deps', '/lib', '/src', '/ci/tests']') AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -188,7 +201,7 @@ blocks: task: agent: machine: - type: s1-prod-ubuntu24-04-amd64-2 + type: s1-prod-ubuntu24-04-amd64-1 env_vars: - name: ARCHITECTURE value: "x64" @@ -476,7 +489,7 @@ blocks: task: agent: machine: - type: s1-prod-ubuntu24-04-amd64-2 + type: s1-prod-ubuntu24-04-amd64-1 jobs: - name: "Tarball" commands: From 3528b03e191dab0cb20ca0d1bad7812c1c67e8c1 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 2 Sep 2025 04:34:02 +0000 Subject: [PATCH 02/13] Fix change_in --- .semaphore/semaphore.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 9208c286..907bd74c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -34,7 +34,7 @@ global_job_config: blocks: - name: "Linux amd64 (musl): Build and test" run: - when: "change_in('['/package.json', '/deps', '/lib', '/src', '/test']') AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -50,7 +50,7 @@ blocks: - name: "Linux arm64 (musl): Build and test" run: - when: "change_in('['/package.json', '/deps', '/lib', '/src', '/test']') AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -66,7 +66,7 @@ blocks: - name: "Linux amd64: Build and test" run: - when: "change_in('['/package.json', '/deps', '/lib', '/src', '/test']') AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -81,7 +81,7 @@ blocks: - name: 'macOS arm64/m1: Build and test' run: - when: "change_in('['/package.json', '/deps', '/lib', '/src', '/test']') AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" dependencies: [] task: agent: @@ -135,7 +135,7 @@ blocks: - name: "(SR) Linux amd64: test, e2e" run: - when: "change_in('['/package.json', '/schemaregistry', '/schemaregistry-examples', '/proto') AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/schemaregistry', '/schemaregistry-examples', '/proto']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -158,7 +158,7 @@ blocks: - name: "Linux amd64: Performance" run: - when: "change_in('['/package.json', '/deps', '/lib', '/src', '/ci/tests']') AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/ci/tests']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: From 3c1c48ac8c9fedc6a94a7a74b5e517e0c71dcfc5 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 2 Sep 2025 04:51:23 +0000 Subject: [PATCH 03/13] Add pipeline ignore temporarily to change in --- .semaphore/semaphore.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 907bd74c..6f1ba018 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -34,7 +34,7 @@ global_job_config: blocks: - name: "Linux amd64 (musl): Build and test" run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -50,7 +50,7 @@ blocks: - name: "Linux arm64 (musl): Build and test" run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -66,7 +66,7 @@ blocks: - name: "Linux amd64: Build and test" run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -81,7 +81,7 @@ blocks: - name: 'macOS arm64/m1: Build and test' run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [] task: agent: @@ -135,7 +135,7 @@ blocks: - name: "(SR) Linux amd64: test, e2e" run: - when: "change_in(['/package.json', '/schemaregistry', '/schemaregistry-examples', '/proto']) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/schemaregistry', '/schemaregistry-examples', '/proto'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -158,7 +158,7 @@ blocks: - name: "Linux amd64: Performance" run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/ci/tests']) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/deps', '/lib', '/src', '/ci/tests'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: From ce4657ce00c858c15920b658d61ab2bd270b7a93 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 2 Sep 2025 05:03:25 +0000 Subject: [PATCH 04/13] Test changes for change_in pipeline --- deps/librdkafka | 2 +- schemaregistry-examples/src/constants.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/librdkafka b/deps/librdkafka index 69b1865e..c56a3e68 160000 --- a/deps/librdkafka +++ b/deps/librdkafka @@ -1 +1 @@ -Subproject commit 69b1865efdc0118cd017760d038d34e52fb3f0d0 +Subproject commit c56a3e68483ae33622901988ab9c4085f0785c3c diff --git a/schemaregistry-examples/src/constants.ts b/schemaregistry-examples/src/constants.ts index a30f5881..86745997 100644 --- a/schemaregistry-examples/src/constants.ts +++ b/schemaregistry-examples/src/constants.ts @@ -22,7 +22,7 @@ const basicAuthCredentials: BasicAuthCredentials = { userInfo: ':', }; -export { - issuerEndpointUrl, oauthClientId, oauthClientSecret, scope, identityPoolId, kafkaLogicalCluster, schemaRegistryLogicalCluster, +export { + issuerEndpointUrl , oauthClientId, oauthClientSecret, scope, identityPoolId, kafkaLogicalCluster, schemaRegistryLogicalCluster, baseUrl, clusterBootstrapUrl, clusterApiKey, clusterApiSecret, basicAuthCredentials, localAuthCredentials }; \ No newline at end of file From d39013bb90c38e5ee9ead25c75fdb78efb1491b7 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 2 Sep 2025 05:07:58 +0000 Subject: [PATCH 05/13] Test changes for change_in pipeline - 2 --- schemaregistry-examples/src/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemaregistry-examples/src/constants.ts b/schemaregistry-examples/src/constants.ts index 86745997..50281a4f 100644 --- a/schemaregistry-examples/src/constants.ts +++ b/schemaregistry-examples/src/constants.ts @@ -23,6 +23,6 @@ const basicAuthCredentials: BasicAuthCredentials = { }; export { - issuerEndpointUrl , oauthClientId, oauthClientSecret, scope, identityPoolId, kafkaLogicalCluster, schemaRegistryLogicalCluster, + issuerEndpointUrl , oauthClientId, oauthClientSecret, scope, identityPoolId, kafkaLogicalCluster, schemaRegistryLogicalCluster, baseUrl, clusterBootstrapUrl, clusterApiKey, clusterApiSecret, basicAuthCredentials, localAuthCredentials }; \ No newline at end of file From c57409c85e8168f7c222803f639c5b632adac5a5 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 2 Sep 2025 05:32:24 +0000 Subject: [PATCH 06/13] Remove /deps - change in deps will be change in util.js also, so it's not needed --- .semaphore/semaphore.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 6f1ba018..66831838 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -34,7 +34,7 @@ global_job_config: blocks: - name: "Linux amd64 (musl): Build and test" run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -50,7 +50,7 @@ blocks: - name: "Linux arm64 (musl): Build and test" run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -66,7 +66,7 @@ blocks: - name: "Linux amd64: Build and test" run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -81,7 +81,7 @@ blocks: - name: 'macOS arm64/m1: Build and test' run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [] task: agent: @@ -158,7 +158,7 @@ blocks: - name: "Linux amd64: Performance" run: - when: "change_in(['/package.json', '/deps', '/lib', '/src', '/ci/tests'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/ci/tests'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: From 9d9e06deff8fbda02abe7d0424ec1d9390895919 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 2 Sep 2025 05:54:52 +0000 Subject: [PATCH 07/13] Don't make pipelines run repeatedly on master when they're running on PR anyway --- .semaphore/semaphore.yml | 10 +++++----- deps/librdkafka | 2 +- schemaregistry-examples/src/constants.ts | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 66831838..99445fca 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -34,7 +34,7 @@ global_job_config: blocks: - name: "Linux amd64 (musl): Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [ ] task: agent: @@ -50,7 +50,7 @@ blocks: - name: "Linux arm64 (musl): Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [ ] task: agent: @@ -66,7 +66,7 @@ blocks: - name: "Linux amd64: Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [ ] task: agent: @@ -81,7 +81,7 @@ blocks: - name: 'macOS arm64/m1: Build and test' run: - when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [] task: agent: @@ -158,7 +158,7 @@ blocks: - name: "Linux amd64: Performance" run: - when: "change_in(['/package.json', '/lib', '/src', '/ci/tests'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/lib', '/src', '/ci/tests'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [ ] task: agent: diff --git a/deps/librdkafka b/deps/librdkafka index c56a3e68..69b1865e 160000 --- a/deps/librdkafka +++ b/deps/librdkafka @@ -1 +1 @@ -Subproject commit c56a3e68483ae33622901988ab9c4085f0785c3c +Subproject commit 69b1865efdc0118cd017760d038d34e52fb3f0d0 diff --git a/schemaregistry-examples/src/constants.ts b/schemaregistry-examples/src/constants.ts index 50281a4f..a30f5881 100644 --- a/schemaregistry-examples/src/constants.ts +++ b/schemaregistry-examples/src/constants.ts @@ -22,7 +22,7 @@ const basicAuthCredentials: BasicAuthCredentials = { userInfo: ':', }; -export { - issuerEndpointUrl , oauthClientId, oauthClientSecret, scope, identityPoolId, kafkaLogicalCluster, schemaRegistryLogicalCluster, +export { + issuerEndpointUrl, oauthClientId, oauthClientSecret, scope, identityPoolId, kafkaLogicalCluster, schemaRegistryLogicalCluster, baseUrl, clusterBootstrapUrl, clusterApiKey, clusterApiSecret, basicAuthCredentials, localAuthCredentials }; \ No newline at end of file From 8fbe5d37162c70287599a318a771569e39c50d81 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 2 Sep 2025 06:28:06 +0000 Subject: [PATCH 08/13] Remove testing changes --- .semaphore/semaphore.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 99445fca..3a619eeb 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -34,7 +34,7 @@ global_job_config: blocks: - name: "Linux amd64 (musl): Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [ ] task: agent: @@ -50,7 +50,7 @@ blocks: - name: "Linux arm64 (musl): Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [ ] task: agent: @@ -66,7 +66,7 @@ blocks: - name: "Linux amd64: Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [ ] task: agent: @@ -81,7 +81,7 @@ blocks: - name: 'macOS arm64/m1: Build and test' run: - when: "change_in(['/package.json', '/lib', '/src', '/test'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [] task: agent: @@ -135,7 +135,7 @@ blocks: - name: "(SR) Linux amd64: test, e2e" run: - when: "change_in(['/package.json', '/schemaregistry', '/schemaregistry-examples', '/proto'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.'" + when: "change_in(['/package.json', '/schemaregistry', '/schemaregistry-examples', '/proto']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -158,7 +158,7 @@ blocks: - name: "Linux amd64: Performance" run: - when: "change_in(['/package.json', '/lib', '/src', '/ci/tests'], {pipeline_file: 'ignore'}) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/ci/tests']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" dependencies: [ ] task: agent: From f4947299ccf7abef5c0ca6d7bee3787a43967585 Mon Sep 17 00:00:00 2001 From: Milind L Date: Mon, 15 Sep 2025 10:31:27 +0000 Subject: [PATCH 09/13] Add cache store/restore --- .semaphore/semaphore.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 3a619eeb..9278741f 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -23,18 +23,25 @@ queue: global_job_config: prologue: + env_vars: + - name: LIBRDKAFKA_VERSION + value: v2.11.1 commands: - checkout + - cache restore librdkafka-dep-${LIBRDKAFKA_VERSION} - git submodule update --init --recursive - cd deps/librdkafka - git fetch origin - - git checkout v2.11.1 + - git checkout $LIBRDKAFKA_VERSION - cd ../../ + epilogue: + commands: + - cache store librdkafka-dep-${LIBRDKAFKA_VERSION} deps/librdkafka blocks: - name: "Linux amd64 (musl): Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -50,7 +57,7 @@ blocks: - name: "Linux arm64 (musl): Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -66,7 +73,7 @@ blocks: - name: "Linux amd64: Build and test" run: - when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: @@ -81,7 +88,7 @@ blocks: - name: 'macOS arm64/m1: Build and test' run: - when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/test']) AND tag !~ '^v[0-9]\\.'" dependencies: [] task: agent: @@ -105,7 +112,9 @@ blocks: prologue: commands: - export CKJS_LINKING=dynamic + - cache restore - npm run install-from-source + - cache store jobs: - name: "Test" commands: @@ -158,7 +167,7 @@ blocks: - name: "Linux amd64: Performance" run: - when: "change_in(['/package.json', '/lib', '/src', '/ci/tests']) AND tag !~ '^v[0-9]\\.' AND branch != 'master'" + when: "change_in(['/package.json', '/lib', '/src', '/ci/tests']) AND tag !~ '^v[0-9]\\.'" dependencies: [ ] task: agent: From 97077a5bfb6d39b182f93a5618315a2e375570e5 Mon Sep 17 00:00:00 2001 From: Milind L Date: Mon, 15 Sep 2025 10:35:03 +0000 Subject: [PATCH 10/13] Add cache store/restore - fix --- .semaphore/semaphore.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 9278741f..d661dd85 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,10 +22,10 @@ queue: processing: parallel global_job_config: + env_vars: + - name: LIBRDKAFKA_VERSION + value: v2.11.1 prologue: - env_vars: - - name: LIBRDKAFKA_VERSION - value: v2.11.1 commands: - checkout - cache restore librdkafka-dep-${LIBRDKAFKA_VERSION} From 109edecf4966bb7a50b4176be3fb2718d22f3270 Mon Sep 17 00:00:00 2001 From: Milind L Date: Mon, 15 Sep 2025 11:26:40 +0000 Subject: [PATCH 11/13] Add cache store/restore - if for public --- .semaphore/semaphore.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d661dd85..88c995cf 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -28,7 +28,7 @@ global_job_config: prologue: commands: - checkout - - cache restore librdkafka-dep-${LIBRDKAFKA_VERSION} + - if [[ $(command cache) ]]; then cache restore librdkafka-dep-${LIBRDKAFKA_VERSION} ; fi - git submodule update --init --recursive - cd deps/librdkafka - git fetch origin @@ -36,7 +36,7 @@ global_job_config: - cd ../../ epilogue: commands: - - cache store librdkafka-dep-${LIBRDKAFKA_VERSION} deps/librdkafka + - if [[ $(command cache) ]]; then cache store librdkafka-dep-${LIBRDKAFKA_VERSION} deps/librdkafka; done blocks: - name: "Linux amd64 (musl): Build and test" @@ -112,9 +112,9 @@ blocks: prologue: commands: - export CKJS_LINKING=dynamic - - cache restore + - if [[ $(command cache) ]]; then cache restore; fi - npm run install-from-source - - cache store + - if [[ $(command cache) ]]; then cache store; fi jobs: - name: "Test" commands: From 5c1ddbdac368529fc08ffc5b98f61ba1a9689b62 Mon Sep 17 00:00:00 2001 From: Milind L Date: Tue, 16 Sep 2025 04:49:27 +0000 Subject: [PATCH 12/13] Remove unrequired workflows and use npm ci for alpine --- .github/workflows/npm-publish.yml | 25 ------------------ .github/workflows/test.yml | 43 ------------------------------- .semaphore/build-docker-alpine.sh | 2 +- 3 files changed, 1 insertion(+), 69 deletions(-) delete mode 100644 .github/workflows/npm-publish.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml deleted file mode 100644 index 0a21a501..00000000 --- a/.github/workflows/npm-publish.yml +++ /dev/null @@ -1,25 +0,0 @@ -# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created -# For more information see: https://help.github.com/actions/language-and-framework-guides/publishing-nodejs-packages - -name: Publish confluent-kafka-javascript - -on: - release: - types: [created] - -jobs: - publish-npm: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - uses: actions/setup-node@v3 - with: - node-version: 18 - registry-url: https://registry.npmjs.org/ - cache: "npm" - - run: npm ci - - run: npm publish - env: - NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 3a531a38..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Build & Test - -on: - push: - branches: ["master"] - pull_request: - branches: ["master"] - -jobs: - build: - strategy: - matrix: - node: [10, 12, 14, 16, 18, 20] - os: [ubuntu-22.04] - include: - # single mac test due to minute multipliers - # https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions - - node: 20 - os: macos-12 - # single windows test due to node.js 14 node-gyp / vs 2022 issues - - node: 20 - os: windows-2022 - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Install Node ${{ matrix.node }} in ${{ runner.os }} - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node }} - cache: "npm" - - name: Update node-gyp - run: npm install --global node-gyp@latest - - name: Install Windows packages - if: runner.os == 'Windows' - run: ./win_install.ps1 - - name: Build - run: npm ci - # skipping on windows for now due to Make / mocha exit code issues - - name: Test - if: runner.os != 'Windows' && matrix.node != 10 - run: npm test diff --git a/.semaphore/build-docker-alpine.sh b/.semaphore/build-docker-alpine.sh index 3f148428..a1b24b89 100755 --- a/.semaphore/build-docker-alpine.sh +++ b/.semaphore/build-docker-alpine.sh @@ -6,5 +6,5 @@ apk add -U ca-certificates openssl ncurses coreutils python3 make gcc g++ libgcc linux-headers grep util-linux binutils findutils perl patch musl-dev bash # /v is the volume mount point for the project root cd /v -npm install +npm ci npx node-pre-gyp package From cce0e63e715be957e5bbe8db90a598798fcf462b Mon Sep 17 00:00:00 2001 From: Milind L Date: Wed, 17 Sep 2025 09:42:22 +0000 Subject: [PATCH 13/13] Use commit hash rather than version to avoid cloning --- .semaphore/semaphore.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 88c995cf..1562952d 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,21 +22,15 @@ queue: processing: parallel global_job_config: - env_vars: - - name: LIBRDKAFKA_VERSION - value: v2.11.1 prologue: commands: - checkout - - if [[ $(command cache) ]]; then cache restore librdkafka-dep-${LIBRDKAFKA_VERSION} ; fi + - export LIBRDKAFKA_COMMIT_HASH="librdkafka_$(git submodule status deps/librdkafka | cut -d' ' -f1)" + - if [[ $(command cache) ]]; then cache restore "$LIBRDKAFKA_COMMIT_HASH" ; fi - git submodule update --init --recursive - - cd deps/librdkafka - - git fetch origin - - git checkout $LIBRDKAFKA_VERSION - - cd ../../ epilogue: commands: - - if [[ $(command cache) ]]; then cache store librdkafka-dep-${LIBRDKAFKA_VERSION} deps/librdkafka; done + - if [[ $(command cache) ]]; then cache store "$LIBRDKAFKA_COMMIT_HASH" deps/librdkafka; done blocks: - name: "Linux amd64 (musl): Build and test"