ETH-585 docker images build and test in CI

.github/workflows/ci.yaml

@@ -30,18 +30,18 @@ jobs:
             packages/*/node_modules
       - name: Install dependencies
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: |
-          npm ci --no-audit
-          npm i @nomicfoundation/solidity-analyzer-linux-x64-gnu
+        run: npm ci --no-audit
+      - name: Install linux x64 specific package
+        run: npm i @nomicfoundation/solidity-analyzer-linux-x64-gnu solidity-comments-linux-x64-gnu
       - name: Build and lint
         run: npm run build && npm run lint
       - name: Unit tests
         run: npm run test
-  integration-tests:
+  fastchain-docker-image:
     timeout-minutes: 20
+    runs-on: ubuntu-20.04
     needs:
       - lint-and-unit-test
-    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3.0.2
       - uses: actions/setup-node@v3
@@ -59,17 +59,42 @@ jobs:
             packages/*/node_modules
       - name: Install dependencies
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
-        run: |
-          npm ci --no-audit
-          npm i @nomicfoundation/solidity-analyzer-linux-x64-gnu
-      - name: Build packages
-        run: npm run build -w packages/network-contracts -w packages/config
-      - name: Start Streamr Docker Stack
+        run: npm ci --no-audit
+      - name: Install linux x64 specific package
+        run: npm i @nomicfoundation/solidity-analyzer-linux-x64-gnu solidity-comments-linux-x64-gnu
+      - name: Build all packages
+        run: npm run build
+      - name: Build fastchain image
+        run: npm run docker:buildLocalArch -w packages/dev-chain-fast
+      - name: Build subgraph image
+        run: npm run docker:buildLocalArch -w packages/network-subgraphs
+      - name: Start docker stack
         uses: streamr-dev/streamr-docker-dev-action@v1.0.1
         with:
-          services-to-start: 'ens-sync-script deploy-network-subgraphs'
-      - name: Integration Test (with dev-docker environment)
+          services-to-start: 'deploy-network-subgraphs-fastchain'
+      - name: Integration Test (with locally built images)
         env:
           TEST_TIMEOUT: '600000'
           DEBUG: "*,-mocha:*"
         run: npm run integration-test
+
+      # - name: Build & Push
+      #   uses: docker/build-push-action@v4.1.1
+      #   with:
+      #     context: .
+      #     file: ./packages/dev-chain-fast/Dockerfile
+      #     platforms: linux/amd64
+      #     push: true
+      #     load: false
+      #     build-args: |
+      #       NODE_ENV=production
+      #     tags: streamr/broker-node:dev
+      #     labels: |
+      #       org.opencontainers.image.source=
+    # uses: ./.github/workflows/docker-build.yml
+    # with:
+    #   docker_file: ./packages/dev-chain-fast/Dockerfile
+    #   image_name: streamr/broker-node
+    #   test_services_to_start: cassandra init-keyspace dev-chain-fast deploy-network-subgraphs-fastchain broker-node-storage-1
+    #   build_platforms: linux/amd64
+    #   test_script: bash ./.github/healthcheck.sh http://localhost:8891/info streamr-dev-broker-node-storage-1

.github/workflows/docker-build.yaml

@@ -0,0 +1,145 @@
+name: "Docker build"
+
+on:
+  workflow_call:
+    inputs:
+      docker_file:
+        type: string
+        required: true
+      image_name:
+        type: string
+        required: true
+      test_services_to_start:
+        type: string
+      test_script:
+        type: string
+      push_image:
+        type: boolean
+      host_machine_platform:
+        type: string
+        required: false
+        default: ubuntu-20.04
+      build_platforms:
+        type: string
+        required: true
+
+    secrets:
+      dockerhub_username:
+        description: 'Required if the docker image is pushed'
+        required: false
+      dockerhub_token:
+        required: false
+
+jobs:
+  build:
+    runs-on: ${{ inputs.host_machine_platform }}
+    steps:
+      - uses: actions/checkout@v3.5.2
+      - name: Cache Docker layers
+        uses: actions/cache@v3.3.1
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-v1-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-v1-
+      - name: Setup qemu
+        if: contains(inputs.build_platforms, 'arm64')
+        uses: docker/setup-qemu-action@v2.2.0
+        with:
+          platforms: arm64
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2.9.1
+        with:
+          driver-opts: |
+            network=host
+      - uses: docker/login-action@v2.2.0
+        if: ${{inputs.push_image}}
+        with:
+          username: ${{ secrets.dockerhub_username }}
+          password: ${{ secrets.dockerhub_token }}
+
+      - name: Get current Broker git tag (for Docker meta)
+        run: |
+          git fetch --tags origin --force
+          echo "broker_head_tag=$(git tag --points-at HEAD --list 'broker/v*')" >> $GITHUB_ENV
+
+
+      - name: Docker meta
+        id: docker_meta_success
+        uses: docker/metadata-action@v4.6.0
+        if: ${{inputs.push_image}}
+        with:
+          images: ${{inputs.image_name}}
+          flavor: |
+            latest=false
+          tags: |
+            type=match,pattern=broker/(v.*),group=1,value=${{ env.broker_head_tag }}
+            type=raw,value=dev,enable=${{ github.ref == 'refs/heads/main' }}
+
+      - name: Build
+        if: ${{!inputs.push_image}}
+        uses: docker/build-push-action@v4.1.1
+        with:
+          context: .
+          file: ${{inputs.docker_file}}
+          platforms: ${{inputs.build_platforms}}
+          push: false
+          load: true
+          build-args: |
+            NODE_ENV=development
+          tags: ${{inputs.image_name}}:dev
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+      - name: Start Streamr Docker Stack
+        if: ${{inputs.test_services_to_start && inputs.test_script }}
+        uses: streamr-dev/streamr-docker-dev-action@v1.0.1
+        with:
+          services-to-start: ${{inputs.test_services_to_start}}
+
+      - name: Run test script
+        if: ${{inputs.test_services_to_start && inputs.test_script }}
+        run: ${{inputs.test_script}}
+
+      - name: Build & Push
+        if: ${{inputs.push_image}}
+        uses: docker/build-push-action@v4.1.1
+        with:
+          context: .
+          file: ${{inputs.docker_file}}
+          platforms: ${{inputs.build_platforms}}
+          push: true
+          load: false
+          build-args: |
+            NODE_ENV=production
+          tags: ${{ steps.docker_meta_success.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
+
+      - name: Collect docker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2.2.1
+        with:
+          dest: 'logs'
+
+      - name: Upload logs to GitHub
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: docker-logs-node${{ matrix.node-version }}--${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt}}
+          path: 'logs'
+
+      - name: Stop Streamr Docker Stack
+        if: always()
+        run: |
+          docker kill $(docker ps -q)
+          docker rm $(docker ps -a -q)
+
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache