V4 Build Docker Image #39
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: V4 Build Docker Image | |
| on: | |
| #define the workflow that will trigger this workflow to run | |
| workflow_run: | |
| workflows: [Maven Java CI] | |
| types: | |
| - completed | |
| #allow the workflow to run manually with version as input | |
| workflow_dispatch: | |
| inputs: | |
| manual_build_version: | |
| description: 'Crawler Version Number - format: x.x.x or x.x.x-snapshot (Case senstive)' | |
| required: false | |
| type: string | |
| env: | |
| # define the types of committers | |
| committer_types: "solr idol elasticsearch sql amazoncloudsearch neo4j azurecognitivesearch apachekafka" | |
| crawler_types: "web fs" | |
| jobs: | |
| # job to build the base crawler image | |
| build-image: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| #check which version value should be used | |
| - name: check condition - version value | |
| uses: haya14busa/action-cond@v1 | |
| id: version_value_check | |
| with: | |
| cond: ${{ inputs.manual_build_version == '' }} | |
| if_true: $(cat pom.xml | grep -m1 "version" | sed 's/<.*>\(.*\)<.*>/\1/' | sed 's/^[ ]*//g') | |
| if_false: ${{ inputs.manual_build_version }} | |
| # assign version value into GITHUB env variable | |
| - name: Use conditional value | |
| run: | | |
| echo "tmp_version=${{ steps.version_value_check.outputs.value }}" >> $GITHUB_ENV | |
| # convert the string to lower case for SNAPSHOT build | |
| - name: convert string to lowercase | |
| id: string | |
| uses: ASzc/change-string-case-action@v1 | |
| with: | |
| string: ${{ env.tmp_version }} | |
| # assign the lowercase value back to an variable for later use | |
| - run: | | |
| echo "version_lowercase=${{ steps.string.outputs.lowercase }}" >> $GITHUB_ENV | |
| echo "version_uppercase=${{ steps.string.outputs.uppercase }}" >> $GITHUB_ENV | |
| # extract version number from value and assign it to different variables for tagging use | |
| - name: Extract version numbers for tagging use | |
| run: | | |
| # Extract major version | |
| echo "major=$(echo ${{env.version_lowercase}} | cut -d '.' -f 1)" >> $GITHUB_ENV | |
| # Extract major_minor version | |
| echo "major_minor=$(echo ${{env.version_lowercase}} | cut -d '.' -f 1-2)" >> $GITHUB_ENV | |
| # Extract major_minor_revis version | |
| echo "major_minor_revis=$(echo ${{env.version_lowercase}} | cut -d '-' -f 1)" >> $GITHUB_ENV | |
| - name: Setup Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'adopt' | |
| java-version: '17' | |
| # build the crawler base docker image | |
| - name: Build docker base image | |
| run: | | |
| for type in $crawler_types; do | |
| # create folder for the downloaded files | |
| mkdir -p downloaded/$type | |
| #check whether a regular or a snapshot version should be built | |
| if [[ "${{env.version_lowercase}}" != *"-snapshot"* ]]; then | |
| # start building Regular Base Image | |
| echo "Regular image build start" | |
| # download the latest zip file from meven central | |
| mvn dependency:get -DgroupId=com.norconex.crawler \ | |
| -DartifactId=nx-crawler-$type \ | |
| -Dpackaging=zip \ | |
| -Dversion=${{env.version_lowercase}} \ | |
| -Dtransitive=false \ | |
| -DremoteRepositories=https://repo1.maven.org/maven2/com/norconex/ | |
| # copy the zip files over to the downloaded folder where docker has access to | |
| cp /home/runner/.m2/repository/com/norconex/crawler/nx-crawler-$type/${{env.version_lowercase}}/*.zip ./downloaded/$type | |
| #Extract Filename | |
| zip_filename=$(basename /home/runner/.m2/repository/com/norconex/crawler/nx-crawler-$type/${{env.version_lowercase}}/*.zip) | |
| docker build . --file .github/workflows/docker-files/Dockerfile-base --build-arg crawler_type=$type --build-arg build_version=${{env.version_lowercase}} --build-arg dl_filename=$zip_filename --target regular --tag ${{env.version_lowercase}}-crawler-$type-base | |
| else | |
| # start building Snapshot Base Image | |
| echo "Snapshot image build start" | |
| # download the latest zip file from meven central | |
| mvn dependency:get -DgroupId=com.norconex.crawler \ | |
| -DartifactId=nx-crawler-$type \ | |
| -Dpackaging=zip \ | |
| -Dversion=${{env.version_uppercase}} \ | |
| -Dtransitive=false \ | |
| -DremoteRepositories=https://oss.sonatype.org/content/repositories/snapshots/ | |
| # copy the zip files over to the downloaded folder where docker has access to | |
| cp /home/runner/.m2/repository/com/norconex/crawler/nx-crawler-$type/${{env.version_uppercase}}/*.zip ./downloaded/$type | |
| #Extract Filename | |
| zip_filename=$(basename /home/runner/.m2/repository/com/norconex/crawler/nx-crawler-$type/${{ env.version_uppercase}}/*.zip) | |
| docker build . --file .github/workflows/docker-files/Dockerfile-base --build-arg crawler_type=$type --build-arg build_version=${{env.major_minor_revis}} --build-arg dl_filename=$zip_filename --target snapshot --tag ${{env.version_lowercase}}-crawler-$type-base | |
| fi | |
| # update the committer docker file with the right crawler base image | |
| if [[ $type == "web" ]]; then | |
| echo "replace file with web" | |
| cp .github/workflows/docker-files/Dockerfile-committers-template .github/workflows/docker-files/Dockerfile-committers | |
| sed -i 's/replace-this-with-real-version/${{env.version_lowercase}}-crawler-web-base/g' .github/workflows/docker-files/Dockerfile-committers | |
| more .github/workflows/docker-files/Dockerfile-committers | |
| else | |
| echo "replace file with fs" | |
| cp .github/workflows/docker-files/Dockerfile-committers-template .github/workflows/docker-files/Dockerfile-committers | |
| sed -i 's/replace-this-with-real-version/${{env.version_lowercase}}-crawler-fs-base/g' .github/workflows/docker-files/Dockerfile-committers | |
| more .github/workflows/docker-files/Dockerfile-committers | |
| fi | |
| # Start the committer image build process | |
| for com_type in $committer_types; do | |
| # create folder for the downloaded files | |
| mkdir -p downloaded/$com_type | |
| # download the latest zip file from meven central | |
| mvn dependency:get -DgroupId=com.norconex.crawler \ | |
| -DartifactId=nx-committer-$com_type \ | |
| -Dpackaging=zip \ | |
| -Dversion=${{env.version_uppercase}} \ | |
| -Dtransitive=false \ | |
| -DremoteRepositories=https://oss.sonatype.org/content/repositories/snapshots/ | |
| # copy the zip files over to the downloaded folder where docker has access to | |
| cp /home/runner/.m2/repository/com/norconex/crawler/nx-committer-$com_type/${{env.version_uppercase}}/*.zip ./downloaded/$com_type | |
| #Extract Filename | |
| com_zip_filename=$(basename /home/runner/.m2/repository/com/norconex/crawler/nx-committer-$com_type/${{env.version_uppercase}}/*.zip) | |
| # run docker build to build the committer image | |
| docker build . --file .github/workflows/docker-files/Dockerfile-committers --build-arg committer_type=$com_type --build-arg dl_filename=$com_zip_filename --build-arg committer_version=${{ env.version_uppercase}} --build-arg build_version=${{env.major_minor_revis}} --build-arg crawler_type=$type --tag ${{env.version_lowercase}}-crawler-$type-$com_type | |
| done | |
| done | |
| # login to Dockerhub | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| # Tag and push images onto Dockerhub repo | |
| - name: Tag and Push images to Dockerhub | |
| run: | | |
| for type in $crawler_types; do | |
| # check if it is a snapshot build, if not, upload different favor of images | |
| if [[ "${{env.version_lowercase}}" != *"-snapshot"* ]]; then | |
| # tag local base image with online norconex tag | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-base norconex/crawler:$type-latest | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-base norconex/crawler:${{env.major}}-$type | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-base norconex/crawler:${{env.major_minor}}-$type | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-base norconex/crawler:${{env.version_lowercase}}-$type | |
| # upload to dockerhub | |
| docker push norconex/crawler:$type-latest | |
| docker push norconex/crawler:${{env.major}}-$type | |
| docker push norconex/crawler:${{env.major_minor}}-$type | |
| docker push norconex/crawler:${{env.version_lowercase}}-$type | |
| for com_type in $committer_types; do | |
| # tag local committer images with online norconex tag | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-$com_type norconex/crawler:${{env.major}}-$type-$com_type | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-$com_type norconex/crawler:${{env.major_minor}}-$type-$com_type | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-$com_type norconex/crawler:${{env.version_lowercase}}-$type-$com_type | |
| # upload online norconex tagged images onto docker hub | |
| docker push norconex/crawler:${{env.major}}-$type-$com_type | |
| docker push norconex/crawler:${{env.major_minor}}-$type-$com_type | |
| docker push norconex/crawler:${{env.version_lowercase}}-$type-$com_type | |
| done | |
| # check what images are created | |
| docker image ls -a | |
| echo "Base, and committers images uploaded!" | |
| else # snapshot images | |
| # tag the local images with repo inform for upload use | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-base norconex/crawler:${{env.version_lowercase}}-$type | |
| # upload tagged images to dockerhub | |
| docker push norconex/crawler:${{env.version_lowercase}}-$type | |
| # tag local committer images with online norconex tag | |
| for com_type in $committer_types; do | |
| docker image tag ${{env.version_lowercase}}-crawler-$type-$com_type norconex/crawler:${{env.version_lowercase}}-$type-$com_type | |
| # upload committer images | |
| docker push norconex/crawler:${{env.version_lowercase}}-$type-$com_type | |
| done | |
| echo "snapshot uploaded!" | |
| fi | |
| done |