diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 588deb52a..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,162 +0,0 @@ -# run `circleci config validate` to avoid syntax problems upon update -version: 2.1 - -orbs: - codecov: codecov/codecov@1 - snyk: snyk/snyk@0.0.11 - -executors: - gradle_docker: - docker: - - image: cimg/openjdk:11.0 - helm: - docker: - - image: hypertrace/helm-gcs-packager:0.3.0 - -commands: - gradle: - description: 'Run the provided gradle command' - parameters: - args: - type: string - when: - default: "on_success" - type: enum - enum: ["on_fail", "on_success", "always"] - steps: - - run: - name: << parameters.args >> - command: ./gradlew << parameters.args >> --info --max-workers=2 -Dorg.gradle.jvmargs=-Xmx2g -Dorg.gradle.console=plain --continue - when: << parameters.when >> - setup_build_environment: - description: 'Generates cache key from a hash of all gradle files' - steps: - - checkout - - run: - name: Generate cache key - command: find . -type f -name "*.gradle*" -o -name "gradle-wrapper*" -exec shasum {} + | sort > /tmp/checksum.txt && cat /tmp/checksum.txt - - restore_cache: - keys: - - v1-dependencies-{{ checksum "/tmp/checksum.txt" }} - # fallback to using the latest cache if no exact match is found - - v1-dependencies- - populate_and_save_cache: - description: 'Downloads all gradle dependencies and uploads cache for later use' - steps: - - gradle: - args: downloadDependencies - - save_cache: - paths: - - ~/.gradle - key: v1-dependencies-{{ checksum "/tmp/checksum.txt" }} -jobs: - build: - executor: gradle_docker - steps: - - setup_build_environment - - setup_remote_docker: &latest_remote_docker - version: 19.03.12 - - populate_and_save_cache - - gradle: - args: build dockerBuildImages - - gradle: - args: jacocoTestReport - - codecov/upload: - flags: unit - merge-publish: - executor: gradle_docker - steps: - - setup_build_environment - - setup_remote_docker: *latest_remote_docker - - gradle: - args: dockerPushImages - release-publish: - executor: gradle_docker - steps: - - setup_build_environment - - setup_remote_docker: *latest_remote_docker - - gradle: - args: publish dockerPushImages - validate-charts: - executor: helm - working_directory: ~/repo - steps: - - checkout - - run: - name: Helm Charts Lint and Template Render - command: | - .circleci/helm.sh validate - snyk-scan: - executor: - name: gradle_docker - environment: - GRADLE_OPTS: -Dorg.gradle.workers.max=1 # Snyk doesn't handle parallelism well - steps: - - setup_build_environment - - snyk/scan: - additional-arguments: --all-sub-projects --policy-path=.snyk --configuration-matching='^runtimeClasspath$' - release-charts: - executor: helm - working_directory: ~/repo - steps: - - checkout - - run: - name: Add release tag - command: | - git config --global user.email "${CIRCLE_USERNAME}@hypertrace.org" - git config --global user.name "$CIRCLE_USERNAME" - git tag -am "Released by $CIRCLE_USERNAME" $(git describe --abbrev=0 --tags | sed 's/^release-//') - - run: - name: Remove trigger tag - command: git tag -d release-$(git describe --abbrev=0) - - run: - name: Package and Publish Helm Charts - # Read the "name:" from Chart.yaml. The chart version is - - command: | - .circleci/helm.sh package - .circleci/helm.sh publish - - add_ssh_keys: - fingerprints: - # This ssh key gives write permission needed for the following step. - - '76:7a:c8:91:c0:cd:eb:71:96:3c:44:c6:cf:b0:26:83' - - run: - name: Update remote tags - command: git push origin refs/tags/$(git describe --abbrev=0) :refs/tags/release-$(git describe --abbrev=0) -workflows: - version: 2 - build-and-publish: - jobs: - - build - - validate-charts - - snyk-scan: - context: - - hypertrace-vulnerability-scanning - - merge-publish: - context: - - hypertrace-publishing - requires: - - build - - validate-charts - - snyk-scan - filters: - branches: - only: - - main - - release-publish: - context: - - hypertrace-publishing - filters: - branches: - ignore: /.*/ - tags: - only: /^release-.*/ - - release-charts: - context: - - hypertrace-publishing - requires: - - release-publish - filters: - branches: - ignore: /.*/ - tags: - only: /^release-.*/ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d3070b4ce..8148f0eb9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,7 @@ # Each line is a file pattern followed by one or more owners. # global -* @hypertrace/trace-pipeline-owners \ No newline at end of file +* @hypertrace/trace-pipeline-owners + +# GH action +.github/ @aaron-steinfeld @jbahire @kotharironak @buchi-busireddy \ No newline at end of file diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 61ec1ed55..fb9d60a40 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -5,43 +5,57 @@ on: - main paths-ignore: - '**/*.md' - - '**/*.txt' - pull_request: + pull_request_target: + branches: + - main # workflow_dispatch will let us manually trigger the workflow from GitHub actions dashboard. # See https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: jobs: build-on-linux: - # Ubuntu-20.04 runner comes with docker 19.03 and OpenJDK 11 and we are using that here. + # Ubuntu-20.04 runner comes with docker 19.03 and OpenJDK 11 and we are using that here. runs-on: ubuntu-20.04 name: Build and run Docker images steps: - name: Checkout Repository uses: actions/checkout@v2 with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} fetch-depth: 0 submodules: recursive - - name: Cache Gradle - uses: actions/cache@v1 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + id: cache-packages + uses: actions/cache@v2 with: path: ~/.gradle - key: ${{ runner.os }}-gradle-${{ hashFiles('**/build.gradle.kts') }} - restore-keys: ${{ runner.os }}-gradle - - name: Invoke Docker build with Gradle - uses: eskatos/gradle-command-action@v1 - with: - arguments: dockerBuildImages + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Build with Gradle + uses: hypertrace/github-actions/gradle@main + with: + args: dockerBuildImages + - name: Verify hypertrace image working-directory: ./.github/workflows/hypertrace-ingester # Below tests a docker-compose.yml service named 'sut' with a valid HEALTHCHECK instruction: run: | docker-compose -f docker-compose.yml -f docker-compose-zipkin-example.yml up -d || { ../scripts/inspect.sh ; exit 1 ; } + - name: Waits for some stability working-directory: ./.github/workflows/hypertrace-ingester run: | sleep 60 # you can decrease it but never increase it docker-compose -f docker-compose.yml ps + - name: Runs tests working-directory: ./.github/workflows/hypertrace-ingester/scripts run: ./tests.sh diff --git a/.circleci/helm.sh b/.github/workflows/helm.sh similarity index 80% rename from .circleci/helm.sh rename to .github/workflows/helm.sh index b203e7761..350895471 100755 --- a/.circleci/helm.sh +++ b/.github/workflows/helm.sh @@ -1,11 +1,6 @@ #!/bin/sh set -eu -script=$0 - -SCRIPT_DIR="$( cd "$( dirname "$script" )" >/dev/null 2>&1 && pwd )" -ROOT_PROJECT_DIR="$(dirname "${SCRIPT_DIR}")" -cd $ROOT_PROJECT_DIR SUB_PROJECTS_DIRS=$(find . -iname "helm" | sed 's/\(.*\)\/.*/\1/') subcommand=$1; shift @@ -18,22 +13,22 @@ case "$subcommand" in helm dependency update ./helm/ helm lint --strict ./helm/ helm template ./helm/ - cd $ROOT_PROJECT_DIR + cd .. done ;; package) - CHART_VERSION=$(git describe --abbrev=0) + CHART_VERSION=$(echo ${GITHUB_REF} | cut -d/ -f 3) for SUB_PROJ_DIR in $SUB_PROJECTS_DIRS; do cd $SUB_PROJ_DIR echo "*******" echo "building charts for:$(pwd)" helm dependency update ./helm/ helm package --version ${CHART_VERSION} --app-version ${CHART_VERSION} ./helm/ - cd $ROOT_PROJECT_DIR + cd .. done ;; publish) - CHART_VERSION=$(git describe --abbrev=0) + CHART_VERSION=$(echo ${GITHUB_REF} | cut -d/ -f 3) export GOOGLE_APPLICATION_CREDENTIALS=${HOME}/helm-gcs-key.json echo ${HELM_GCS_CREDENTIALS} > ${GOOGLE_APPLICATION_CREDENTIALS} helm repo add helm-gcs ${HELM_GCS_REPOSITORY} @@ -43,7 +38,7 @@ case "$subcommand" in echo "publishing charts for:$(pwd)" CHART_NAME=$(awk '/^name:/ {print $2}' ./helm/Chart.yaml) helm gcs push ${CHART_NAME}-${CHART_VERSION}.tgz helm-gcs --public --retry - cd $ROOT_PROJECT_DIR + cd .. done ;; *) diff --git a/.github/workflows/hypertrace-ingester/docker-compose.yml b/.github/workflows/hypertrace-ingester/docker-compose.yml index 09833da8b..04698fc98 100644 --- a/.github/workflows/hypertrace-ingester/docker-compose.yml +++ b/.github/workflows/hypertrace-ingester/docker-compose.yml @@ -38,7 +38,7 @@ services: condition: service_healthy sut: - image: hypertrace/hypertrace-ingester:test + image: hypertrace/hypertrace-ingester:main container_name: sut environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:9092 diff --git a/.github/workflows/merge-publish.yml b/.github/workflows/merge-publish.yml new file mode 100644 index 000000000..f4b5e83ad --- /dev/null +++ b/.github/workflows/merge-publish.yml @@ -0,0 +1,42 @@ +name: merge-publish +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + merge-publish: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + uses: actions/cache@v2 + with: + path: ~/.gradle + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_READ_USER }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + + - name: push docker image + uses: hypertrace/github-actions/gradle@main + with: + args: dockerPushImages + env: + DOCKER_USERNAME: ${{ secrets.DOCKERHUB_PUBLISH_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PUBLISH_TOKEN }} diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml new file mode 100644 index 000000000..9f481f3ad --- /dev/null +++ b/.github/workflows/pr-build.yml @@ -0,0 +1,80 @@ +name: build and validate +on: + push: + branches: + - main + pull_request_target: + branches: + - main + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + fetch-depth: 0 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + uses: actions/cache@v2 + with: + path: ~/.gradle + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_READ_USER }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + + - name: Build with Gradle + uses: hypertrace/github-actions/gradle@main + with: + args: build dockerBuildImages + + validate-helm-charts: + runs-on: ubuntu-20.04 + container: + image: hypertrace/helm-gcs-packager:0.3.0 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + steps: + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + fetch-depth: 0 + + - name: validate charts + run: ./.github/workflows/helm.sh validate + + snyk-scan: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + ref: ${{github.event.pull_request.head.ref}} + repository: ${{github.event.pull_request.head.repo.full_name}} + fetch-depth: 0 + - name: Setup snyk + uses: snyk/actions/setup@0.3.0 + - name: Snyk test + run: snyk test --all-sub-projects --org=hypertrace --severity-threshold=low --policy-path=.snyk --configuration-matching='^runtimeClasspath$' + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + GRADLE_OPTS: -Dorg.gradle.workers.max=1 diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml new file mode 100644 index 000000000..6077ec4e0 --- /dev/null +++ b/.github/workflows/pr-test.yml @@ -0,0 +1,59 @@ +name: test +on: + push: + branches: + - main + pull_request: + +jobs: + test: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + uses: actions/cache@v2 + with: + path: ~/.gradle + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Unit test + uses: hypertrace/github-actions/gradle@main + with: + args: jacocoTestReport + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + name: unit test reports + fail_ci_if_error: true + flags: unit + + - name: copy test reports + uses: hypertrace/github-actions/gradle@main + with: + args: copyAllReports --output-dir=/tmp/test-reports + + - name: Archive test reports + uses: actions/upload-artifact@v1 + with: + name: test-reports + path: /tmp/test-reports + if: always() + + - name: Publish Unit Test Results + uses: docker://ghcr.io/enricomi/publish-unit-test-result-action:v1.6 + if: always() + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + files: ./**/build/test-results/**/*.xml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..70e88a994 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,68 @@ +name: Publish artifacts +on: +# Will only run when release is published. + release: + types: + - created + workflow_dispatch: + +jobs: + publish-artifacts: + runs-on: ubuntu-20.04 + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Check out code + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: create checksum file + uses: hypertrace/github-actions/checksum@main + + - name: Cache packages + uses: actions/cache@v2 + with: + path: ~/.gradle + key: gradle-packages-${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/checksum.txt') }} + restore-keys: | + gradle-packages-${{ runner.os }}-${{ github.job }} + gradle-packages-${{ runner.os }} + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_READ_USER }} + password: ${{ secrets.DOCKERHUB_READ_TOKEN }} + + - name: publish docker image + uses: hypertrace/github-actions/gradle@main + with: + args: publish dockerPushImages + env: + DOCKER_USERNAME: ${{ secrets.DOCKERHUB_PUBLISH_USER }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PUBLISH_TOKEN }} + ORG_GRADLE_PROJECT_publishUser: ${{ secrets.BINTRAY_USER }} + ORG_GRADLE_PROJECT_publishApiKey: ${{ secrets.BINTRAY_API_KEY }} + + publish-helm-charts: + runs-on: ubuntu-20.04 + container: + image: hypertrace/helm-gcs-packager:0.3.0 + credentials: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + steps: + # Set fetch-depth: 0 to fetch commit history and tags for use in version calculation + - name: Checkout Repository + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: package and release charts + env: + HELM_GCS_CREDENTIALS: ${{ secrets.HELM_GCS_CREDENTIALS }} + HELM_GCS_REPOSITORY: ${{ secrets.HELM_GCS_REPOSITORY }} + run: | + ./.github/workflows/helm.sh package + ./.github/workflows/helm.sh publish + diff --git a/build.gradle.kts b/build.gradle.kts index 022c84d33..73a023e1d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,8 +1,8 @@ plugins { id("org.hypertrace.repository-plugin") version "0.2.1" id("org.hypertrace.ci-utils-plugin") version "0.1.2" - id("org.hypertrace.docker-java-application-plugin") version "0.7.1" apply false - id("org.hypertrace.docker-publish-plugin") version "0.7.1" apply false + id("org.hypertrace.docker-java-application-plugin") version "0.8.1" apply false + id("org.hypertrace.docker-publish-plugin") version "0.8.1" apply false id("org.hypertrace.jacoco-report-plugin") version "0.1.1" apply false id("org.hypertrace.publish-plugin") version "0.3.3" apply false id("org.hypertrace.avro-plugin") version "0.2.4" apply false diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000..8a333e58d --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +codecov: + max_report_age: off