From 4de0a554cde8abd8a39f8caf98b6bdc25ed2bb79 Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 10:45:09 +0530 Subject: [PATCH 01/11] adds gha --- .github/CODEOWNERS | 5 +- .github/workflows/merge-publish.yml | 42 ++++++++++++++++ .github/workflows/pr-build.yml | 74 +++++++++++++++++++++++++++++ .github/workflows/pr-test.yml | 59 +++++++++++++++++++++++ .github/workflows/publish.yml | 62 ++++++++++++++++++++++++ build.gradle.kts | 4 +- codecov.yml | 2 + 7 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/merge-publish.yml create mode 100644 .github/workflows/pr-build.yml create mode 100644 .github/workflows/pr-test.yml create mode 100644 .github/workflows/publish.yml create mode 100644 codecov.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d3070b4ce..8148f0eb9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,7 @@ # Each line is a file pattern followed by one or more owners. # global -* @hypertrace/trace-pipeline-owners \ No newline at end of file +* @hypertrace/trace-pipeline-owners + +# GH action +.github/ @aaron-steinfeld @jbahire @kotharironak @buchi-busireddy \ No newline at end of file diff --git a/.github/workflows/merge-publish.yml b/.github/workflows/merge-publish.yml new file mode 100644 index 000000000..f4b5e83ad --- /dev/null +++ b/.github/workflows/merge-publish.yml @@ -0,0 +1,42 @@ +name: merge-publish +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + merge-publish: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + uses: actions/cache@v2 + with: + path: ~/.gradle + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_READ_USER }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + + - name: push docker image + uses: hypertrace/github-actions/gradle@main + with: + args: dockerPushImages + env: + DOCKER_USERNAME: ${{ secrets.DOCKERHUB_PUBLISH_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PUBLISH_TOKEN }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml new file mode 100644 index 000000000..1fdd5f8f5 --- /dev/null +++ b/.github/workflows/pr-build.yml @@ -0,0 +1,74 @@ +name: build and validate +on: + push: + branches: + - main + pull_request_target: + branches: + - main + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + fetch-depth: 0 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + uses: actions/cache@v2 + with: + path: ~/.gradle + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_READ_USER }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + + - name: Build with Gradle + uses: hypertrace/github-actions/gradle@main + with: + args: build dockerBuildImages + + validate-helm-charts: + runs-on: ubuntu-20.04 + steps: + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + fetch-depth: 0 + + - name: validate charts + uses: hypertrace/github-actions/validate-charts@main + + snyk-scan: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + fetch-depth: 0 + - name: Setup snyk + uses: snyk/actions/setup@0.3.0 + - name: Snyk test + run: snyk test --all-sub-projects --org=hypertrace --severity-threshold=low --policy-path=.snyk --configuration-matching='^runtimeClasspath$' + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + GRADLE_OPTS: -Dorg.gradle.workers.max=1 diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml new file mode 100644 index 000000000..6077ec4e0 --- /dev/null +++ b/.github/workflows/pr-test.yml @@ -0,0 +1,59 @@ +name: test +on: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + uses: actions/cache@v2 + with: + path: ~/.gradle + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Unit test + uses: hypertrace/github-actions/gradle@main + with: + args: jacocoTestReport + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + name: unit test reports + fail_ci_if_error: true + flags: unit + + - name: copy test reports + uses: hypertrace/github-actions/gradle@main + with: + args: copyAllReports --output-dir=/tmp/test-reports + + - name: Archive test reports + uses: actions/upload-artifact@v1 + with: + name: test-reports + path: /tmp/test-reports + if: always() + + - name: Publish Unit Test Results + uses: docker://ghcr.io/enricomi/publish-unit-test-result-action:v1.6 + if: always() + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + files: ./**/build/test-results/**/*.xml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..0c247297f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,62 @@ +name: Publish artifacts +on: +# Will only run when release is published. + release: + types: + - created + workflow_dispatch: + +jobs: + publish-artifacts: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + uses: actions/cache@v2 + with: + path: ~/.gradle + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_READ_USER }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + + - name: publish docker image + uses: hypertrace/github-actions/gradle@main + with: + args: publish dockerPushImages + env: + DOCKER_USERNAME: ${{ secrets.DOCKERHUB_PUBLISH_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PUBLISH_TOKEN }} + ORG_GRADLE_PROJECT_publishUser: ${{ secrets.BINTRAY_USER }} + ORG_GRADLE_PROJECT_publishApiKey: ${{ secrets.BINTRAY_API_KEY }} + + publish-helm-charts: + needs: publish-artifacts + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Checkout Repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: package and release charts + uses: hypertrace/github-actions/helm-gcs-publish@main + with: + helm-gcs-credentials: ${{ secrets.HELM_GCS_CREDENTIALS }} + helm-gcs-repository: ${{ secrets.HELM_GCS_REPOSITORY }} + diff --git a/build.gradle.kts b/build.gradle.kts index 022c84d33..73a023e1d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,8 @@ plugins { id("org.hypertrace.repository-plugin") version "0.2.1" id("org.hypertrace.ci-utils-plugin") version "0.1.2" - id("org.hypertrace.docker-java-application-plugin") version "0.7.1" apply false - id("org.hypertrace.docker-publish-plugin") version "0.7.1" apply false + id("org.hypertrace.docker-java-application-plugin") version "0.8.1" apply false + id("org.hypertrace.docker-publish-plugin") version "0.8.1" apply false id("org.hypertrace.jacoco-report-plugin") version "0.1.1" apply false id("org.hypertrace.publish-plugin") version "0.3.3" apply false id("org.hypertrace.avro-plugin") version "0.2.4" apply false diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..8a333e58d --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +codecov: + max_report_age: off From 43d5ae84d1601d408ff1365af28db7beeb6a215c Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 10:59:14 +0530 Subject: [PATCH 02/11] uses helm script --- {.circleci => .github/workflows}/helm.sh | 4 ++-- .github/workflows/pr-build.yml | 2 +- .github/workflows/publish.yml | 10 ++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) rename {.circleci => .github/workflows}/helm.sh (93%) diff --git a/.circleci/helm.sh b/.github/workflows/helm.sh similarity index 93% rename from .circleci/helm.sh rename to .github/workflows/helm.sh index b203e7761..af09a2418 100755 --- a/.circleci/helm.sh +++ b/.github/workflows/helm.sh @@ -22,7 +22,7 @@ case "$subcommand" in done ;; package) - CHART_VERSION=$(git describe --abbrev=0) + CHART_VERSION=$(echo ${GITHUB_REF} | cut -d/ -f 3) for SUB_PROJ_DIR in $SUB_PROJECTS_DIRS; do cd $SUB_PROJ_DIR echo "*******" @@ -33,7 +33,7 @@ case "$subcommand" in done ;; publish) - CHART_VERSION=$(git describe --abbrev=0) + CHART_VERSION=$(echo ${GITHUB_REF} | cut -d/ -f 3) export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/helm-gcs-key.json echo ${HELM_GCS_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS} helm repo add helm-gcs ${HELM_GCS_REPOSITORY} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 1fdd5f8f5..2711bf40f 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -53,7 +53,7 @@ jobs: fetch-depth: 0 - name: validate charts - uses: hypertrace/github-actions/validate-charts@main + run: ./.github/workflows/helm.sh validate snyk-scan: runs-on: ubuntu-20.04 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0c247297f..98c9c13a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -55,8 +55,10 @@ jobs: fetch-depth: 0 - name: package and release charts - uses: hypertrace/github-actions/helm-gcs-publish@main - with: - helm-gcs-credentials: ${{ secrets.HELM_GCS_CREDENTIALS }} - helm-gcs-repository: ${{ secrets.HELM_GCS_REPOSITORY }} + run: | + ./.github/workflows/helm.sh package + ./.github/workflows/helm.sh publish + env: + HELM_GCS_CREDENTIALS: ${{ secrets.HELM_GCS_CREDENTIALS }} + HELM_GCS_REPOSITORY: ${{ secrets.HELM_GCS_REPOSITORY }} From 18049cd988f0854e5da7b12146b62bd6beb7e857 Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 10:59:22 +0530 Subject: [PATCH 03/11] removes cci --- .circleci/config.yml | 162 ------------------------------------------- 1 file changed, 162 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 588deb52a..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,162 +0,0 @@ -# run `circleci config validate` to avoid syntax problems upon update -version: 2.1 - -orbs: - codecov: codecov/codecov@1 - snyk: snyk/snyk@0.0.11 - -executors: - gradle_docker: - docker: - - image: cimg/openjdk:11.0 - helm: - docker: - - image: hypertrace/helm-gcs-packager:0.3.0 - -commands: - gradle: - description: 'Run the provided gradle command' - parameters: - args: - type: string - when: - default: "on_success" - type: enum - enum: ["on_fail", "on_success", "always"] - steps: - - run: - name: << parameters.args >> - command: ./gradlew << parameters.args >> --info --max-workers=2 -Dorg.gradle.jvmargs=-Xmx2g -Dorg.gradle.console=plain --continue - when: << parameters.when >> - setup_build_environment: - description: 'Generates cache key from a hash of all gradle files' - steps: - - checkout - - run: - name: Generate cache key - command: find . -type f -name "*.gradle*" -o -name "gradle-wrapper*" -exec shasum {} + | sort > /tmp/checksum.txt && cat /tmp/checksum.txt - - restore_cache: - keys: - - v1-dependencies-{{ checksum "/tmp/checksum.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - populate_and_save_cache: - description: 'Downloads all gradle dependencies and uploads cache for later use' - steps: - - gradle: - args: downloadDependencies - - save_cache: - paths: - - ~/.gradle - key: v1-dependencies-{{ checksum "/tmp/checksum.txt" }} -jobs: - build: - executor: gradle_docker - steps: - - setup_build_environment - - setup_remote_docker: &latest_remote_docker - version: 19.03.12 - - populate_and_save_cache - - gradle: - args: build dockerBuildImages - - gradle: - args: jacocoTestReport - - codecov/upload: - flags: unit - merge-publish: - executor: gradle_docker - steps: - - setup_build_environment - - setup_remote_docker: *latest_remote_docker - - gradle: - args: dockerPushImages - release-publish: - executor: gradle_docker - steps: - - setup_build_environment - - setup_remote_docker: *latest_remote_docker - - gradle: - args: publish dockerPushImages - validate-charts: - executor: helm - working_directory: ~/repo - steps: - - checkout - - run: - name: Helm Charts Lint and Template Render - command: | - .circleci/helm.sh validate - snyk-scan: - executor: - name: gradle_docker - environment: - GRADLE_OPTS: -Dorg.gradle.workers.max=1 # Snyk doesn't handle parallelism well - steps: - - setup_build_environment - - snyk/scan: - additional-arguments: --all-sub-projects --policy-path=.snyk --configuration-matching='^runtimeClasspath$' - release-charts: - executor: helm - working_directory: ~/repo - steps: - - checkout - - run: - name: Add release tag - command: | - git config --global user.email "${CIRCLE_USERNAME}@hypertrace.org" - git config --global user.name "$CIRCLE_USERNAME" - git tag -am "Released by $CIRCLE_USERNAME" $(git describe --abbrev=0 --tags | sed 's/^release-//') - - run: - name: Remove trigger tag - command: git tag -d release-$(git describe --abbrev=0) - - run: - name: Package and Publish Helm Charts - # Read the "name:" from Chart.yaml. The chart version is - - command: | - .circleci/helm.sh package - .circleci/helm.sh publish - - add_ssh_keys: - fingerprints: - # This ssh key gives write permission needed for the following step. - - '76:7a:c8:91:c0:cd:eb:71:96:3c:44:c6:cf:b0:26:83' - - run: - name: Update remote tags - command: git push origin refs/tags/$(git describe --abbrev=0) :refs/tags/release-$(git describe --abbrev=0) -workflows: - version: 2 - build-and-publish: - jobs: - - build - - validate-charts - - snyk-scan: - context: - - hypertrace-vulnerability-scanning - - merge-publish: - context: - - hypertrace-publishing - requires: - - build - - validate-charts - - snyk-scan - filters: - branches: - only: - - main - - release-publish: - context: - - hypertrace-publishing - filters: - branches: - ignore: /.*/ - tags: - only: /^release-.*/ - - release-charts: - context: - - hypertrace-publishing - requires: - - release-publish - filters: - branches: - ignore: /.*/ - tags: - only: /^release-.*/ From d3e9e65d75692e0edb03b013f1d40794c6285ba2 Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 11:13:44 +0530 Subject: [PATCH 04/11] trying to set env github-ref --- .github/workflows/hypertrace-ingester/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hypertrace-ingester/docker-compose.yml b/.github/workflows/hypertrace-ingester/docker-compose.yml index 09833da8b..04698fc98 100644 --- a/.github/workflows/hypertrace-ingester/docker-compose.yml +++ b/.github/workflows/hypertrace-ingester/docker-compose.yml @@ -38,7 +38,7 @@ services: condition: service_healthy sut: - image: hypertrace/hypertrace-ingester:test + image: hypertrace/hypertrace-ingester:main container_name: sut environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 From ea4e9138ca931471638c577d1ec55e9b019c3bc1 Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 11:26:31 +0530 Subject: [PATCH 05/11] trying to set env github-ref --- .github/workflows/docker-tests.yml | 44 +++++++++++++------ .../hypertrace-ingester/docker-compose.yml | 2 +- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 61ec1ed55..2fb2f25cc 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -5,43 +5,61 @@ on: - main paths-ignore: - '**/*.md' - - '**/*.txt' pull_request: + branches: + - main # workflow_dispatch will let us manually trigger the workflow from GitHub actions dashboard. # See https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: jobs: build-on-linux: - # Ubuntu-20.04 runner comes with docker 19.03 and OpenJDK 11 and we are using that here. + # Ubuntu-20.04 runner comes with docker 19.03 and OpenJDK 11 and we are using that here. runs-on: ubuntu-20.04 name: Build and run Docker images steps: - name: Checkout Repository uses: actions/checkout@v2 with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} fetch-depth: 0 submodules: recursive - - name: Cache Gradle - uses: actions/cache@v1 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + id: cache-packages + uses: actions/cache@v2 with: path: ~/.gradle - key: ${{ runner.os }}-gradle-${{ hashFiles('**/build.gradle.kts') }} - restore-keys: ${{ runner.os }}-gradle - - name: Invoke Docker build with Gradle - uses: eskatos/gradle-command-action@v1 - with: - arguments: dockerBuildImages + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Set Production Environment + if: github.ref != 'refs/heads/main' + run: echo "::set-env name=GITHUB_REF::test" + + - name: Build with Gradle + uses: hypertrace/github-actions/gradle@main + with: + args: dockerBuildImages + - name: Verify hypertrace image - working-directory: ./.github/workflows/hypertrace-ingester + working-directory: ./.github/workflows/hypertrace-service # Below tests a docker-compose.yml service named 'sut' with a valid HEALTHCHECK instruction: run: | docker-compose -f docker-compose.yml -f docker-compose-zipkin-example.yml up -d || { ../scripts/inspect.sh ; exit 1 ; } + - name: Waits for some stability - working-directory: ./.github/workflows/hypertrace-ingester + working-directory: ./.github/workflows/hypertrace-service run: | sleep 60 # you can decrease it but never increase it docker-compose -f docker-compose.yml ps + - name: Runs tests - working-directory: ./.github/workflows/hypertrace-ingester/scripts + working-directory: ./.github/workflows/hypertrace-service/scripts run: ./tests.sh diff --git a/.github/workflows/hypertrace-ingester/docker-compose.yml b/.github/workflows/hypertrace-ingester/docker-compose.yml index 04698fc98..09833da8b 100644 --- a/.github/workflows/hypertrace-ingester/docker-compose.yml +++ b/.github/workflows/hypertrace-ingester/docker-compose.yml @@ -38,7 +38,7 @@ services: condition: service_healthy sut: - image: hypertrace/hypertrace-ingester:main + image: hypertrace/hypertrace-ingester:test container_name: sut environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 From 49cf19e8885fc1b2f6c85cc9295fc63d22f85080 Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 11:29:11 +0530 Subject: [PATCH 06/11] runs e2e only on PR with branch tag --- .github/workflows/docker-tests.yml | 9 --------- .github/workflows/hypertrace-ingester/docker-compose.yml | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 2fb2f25cc..ac8abf29f 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -1,10 +1,5 @@ name: e2e test on: - push: - branches: - - main - paths-ignore: - - '**/*.md' pull_request: branches: - main @@ -39,10 +34,6 @@ jobs: gradle-packages-${{ runner.os }}-${{ github.job }} gradle-packages-${{ runner.os }} - - name: Set Production Environment - if: github.ref != 'refs/heads/main' - run: echo "::set-env name=GITHUB_REF::test" - - name: Build with Gradle uses: hypertrace/github-actions/gradle@main with: diff --git a/.github/workflows/hypertrace-ingester/docker-compose.yml b/.github/workflows/hypertrace-ingester/docker-compose.yml index 09833da8b..51e386282 100644 --- a/.github/workflows/hypertrace-ingester/docker-compose.yml +++ b/.github/workflows/hypertrace-ingester/docker-compose.yml @@ -38,7 +38,7 @@ services: condition: service_healthy sut: - image: hypertrace/hypertrace-ingester:test + image: hypertrace/hypertrace-ingester:${GITHUB_REF//\//} container_name: sut environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 From 132801d19a55b381cc12139cc8a2d4d9b8af87fe Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 11:34:47 +0530 Subject: [PATCH 07/11] cleanup --- .github/workflows/docker-tests.yml | 6 +++--- .github/workflows/pr-build.yml | 6 ++++++ .github/workflows/publish.yml | 12 ++++++++---- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index ac8abf29f..e1d17f118 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -40,17 +40,17 @@ jobs: args: dockerBuildImages - name: Verify hypertrace image - working-directory: ./.github/workflows/hypertrace-service + working-directory: ./.github/workflows/hypertrace-ingester # Below tests a docker-compose.yml service named 'sut' with a valid HEALTHCHECK instruction: run: | docker-compose -f docker-compose.yml -f docker-compose-zipkin-example.yml up -d || { ../scripts/inspect.sh ; exit 1 ; } - name: Waits for some stability - working-directory: ./.github/workflows/hypertrace-service + working-directory: ./.github/workflows/hypertrace-ingester run: | sleep 60 # you can decrease it but never increase it docker-compose -f docker-compose.yml ps - name: Runs tests - working-directory: ./.github/workflows/hypertrace-service/scripts + working-directory: ./.github/workflows/hypertrace-ingester/scripts run: ./tests.sh diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 2711bf40f..9f481f3ad 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -44,6 +44,12 @@ jobs: validate-helm-charts: runs-on: ubuntu-20.04 + container: + image: hypertrace/helm-gcs-packager:0.3.0 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation steps: - name: Check out code uses: actions/checkout@v2.3.4 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 98c9c13a2..70e88a994 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -45,8 +45,12 @@ jobs: ORG_GRADLE_PROJECT_publishApiKey: ${{ secrets.BINTRAY_API_KEY }} publish-helm-charts: - needs: publish-artifacts runs-on: ubuntu-20.04 + container: + image: hypertrace/helm-gcs-packager:0.3.0 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} steps: # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation - name: Checkout Repository @@ -55,10 +59,10 @@ jobs: fetch-depth: 0 - name: package and release charts + env: + HELM_GCS_CREDENTIALS: ${{ secrets.HELM_GCS_CREDENTIALS }} + HELM_GCS_REPOSITORY: ${{ secrets.HELM_GCS_REPOSITORY }} run: | ./.github/workflows/helm.sh package ./.github/workflows/helm.sh publish - env: - HELM_GCS_CREDENTIALS: ${{ secrets.HELM_GCS_CREDENTIALS }} - HELM_GCS_REPOSITORY: ${{ secrets.HELM_GCS_REPOSITORY }} From 90ce31c736c28c53af2f3b19a7cfbb6e9bb32fec Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 11:39:53 +0530 Subject: [PATCH 08/11] fix: e2e --- .github/workflows/docker-tests.yml | 7 ++++++- .github/workflows/hypertrace-ingester/docker-compose.yml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index e1d17f118..fb9d60a40 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -1,6 +1,11 @@ name: e2e test on: - pull_request: + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request_target: branches: - main # workflow_dispatch will let us manually trigger the workflow from GitHub actions dashboard. diff --git a/.github/workflows/hypertrace-ingester/docker-compose.yml b/.github/workflows/hypertrace-ingester/docker-compose.yml index 51e386282..04698fc98 100644 --- a/.github/workflows/hypertrace-ingester/docker-compose.yml +++ b/.github/workflows/hypertrace-ingester/docker-compose.yml @@ -38,7 +38,7 @@ services: condition: service_healthy sut: - image: hypertrace/hypertrace-ingester:${GITHUB_REF//\//} + image: hypertrace/hypertrace-ingester:main container_name: sut environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 From 76bab546a51c94f0517519f695746de26e03a607 Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 11:56:34 +0530 Subject: [PATCH 09/11] test: e2e --- .github/workflows/docker-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index fb9d60a40..bce39b1c3 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -5,7 +5,7 @@ on: - main paths-ignore: - '**/*.md' - pull_request_target: + pull_request: branches: - main # workflow_dispatch will let us manually trigger the workflow from GitHub actions dashboard. From 652f34049ab2fbfdde051d14c54d66610820d8eb Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Thu, 7 Jan 2021 11:59:48 +0530 Subject: [PATCH 10/11] fix: e2e --- .github/workflows/docker-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index bce39b1c3..fb9d60a40 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -5,7 +5,7 @@ on: - main paths-ignore: - '**/*.md' - pull_request: + pull_request_target: branches: - main # workflow_dispatch will let us manually trigger the workflow from GitHub actions dashboard. From aff94b9e021611cadb430a5f45cb85cc31a4030c Mon Sep 17 00:00:00 2001 From: Jayesh Bapu Ahire Date: Fri, 8 Jan 2021 09:52:09 +0530 Subject: [PATCH 11/11] Update helm.sh --- .github/workflows/helm.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/helm.sh b/.github/workflows/helm.sh index af09a2418..350895471 100755 --- a/.github/workflows/helm.sh +++ b/.github/workflows/helm.sh @@ -1,11 +1,6 @@ #!/bin/sh set -eu -script=$0 - -SCRIPT_DIR="$( cd "$( dirname "$script" )" >/dev/null 2>&1 && pwd )" -ROOT_PROJECT_DIR="$(dirname "${SCRIPT_DIR}")" -cd $ROOT_PROJECT_DIR SUB_PROJECTS_DIRS=$(find . -iname "helm" | sed 's/\(.*\)\/.*/\1/') subcommand=$1; shift @@ -18,7 +13,7 @@ case "$subcommand" in helm dependency update ./helm/ helm lint --strict ./helm/ helm template ./helm/ - cd $ROOT_PROJECT_DIR + cd .. done ;; package) @@ -29,7 +24,7 @@ case "$subcommand" in echo "building charts for:$(pwd)" helm dependency update ./helm/ helm package --version ${CHART_VERSION} --app-version ${CHART_VERSION} ./helm/ - cd $ROOT_PROJECT_DIR + cd .. done ;; publish) @@ -43,7 +38,7 @@ case "$subcommand" in echo "publishing charts for:$(pwd)" CHART_NAME=$(awk '/^name:/ {print $2}' ./helm/Chart.yaml) helm gcs push ${CHART_NAME}-${CHART_VERSION}.tgz helm-gcs --public --retry - cd $ROOT_PROJECT_DIR + cd .. done ;; *)