Merge main (#970)

.github/workflows/copyright-check-new.yml

@@ -14,11 +14,19 @@ jobs:
       contents: write
 
     steps:
+      - name: Check if PR is from Dependabot
+        id: check_dependabot
+        run: |
+          echo "check=$(jq -r '.pull_request.user.login' "$GITHUB_EVENT_PATH" | grep -c dependabot)" >> $GITHUB_OUTPUT
+                  
+
       - uses: actions/checkout@v4
+        if: steps.check_dependabot.outputs.check == '0'
         with:
           fetch-depth: 0
 
       - name: Get modified files 
+        if: steps.check_dependabot.outputs.check == '0'
         id: modified-files        
         uses: tj-actions/changed-files@v42.1.0
         with:
@@ -30,6 +38,7 @@ jobs:
             **/assembly/**
 
       - name: Find all changed files don't contain Copyright Info and add the info to it
+        if: steps.check_dependabot.outputs.check == '0'
         run: |
           for file in ${{ steps.modified-files.outputs.all_changed_files }}; do
             if ! grep -qE 'Copyright [0-9]{4} Norconex Inc\.|Copyright [0-9]{4}-[0-9]{4} Norconex Inc\.' "$file"; then
@@ -81,6 +90,7 @@ jobs:
           done
           
       - name: Verify Year on files' Copyright Info is up to date
+        if: steps.check_dependabot.outputs.check == '0'
         run: |
           # make sure the file exist
           touch /tmp/modified-file-list.txt
@@ -150,6 +160,7 @@ jobs:
 
 
       - name: Upload Modified Files
+        if: steps.check_dependabot.outputs.check == '0'
         uses: stefanzweifel/git-auto-commit-action@v5
         with:
           commit_message: Apply Copyright year changes

.github/workflows/docker-files/Dockerfile-base

@@ -0,0 +1,35 @@
+FROM ubuntu:22.04 AS base
+ARG crawler_type
+ARG build_version
+ARG dl_filename
+
+WORKDIR /nxer
+
+RUN apt-get -y update && \
+    apt-get install -y nano && \
+    apt-get install -y curl && \
+    apt-get install -y unzip && \
+    apt-get install -y iputils-ping && \
+    apt install bash-completion && \
+    curl -LO https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.deb && \
+    dpkg -i jdk-17_linux-x64_bin.deb && \    
+    rm *.deb
+
+
+FROM base AS regular
+WORKDIR /nxer
+
+COPY downloaded/$crawler_type/*.zip /nxer/
+RUN unzip ${dl_filename}
+RUN chmod +x nx-crawler-${crawler_type}-${build_version}/crawl-${crawler_type}.*
+RUN rm *.zip
+
+
+FROM base AS snapshot
+WORKDIR /nxer
+
+COPY downloaded/$crawler_type/*.zip /nxer/
+RUN unzip ${dl_filename}
+RUN mv nx-crawler-${crawler_type}-${build_version}-SNAPSHOT nx-crawler-${crawler_type}-${build_version}
+RUN chmod +x nx-crawler-${crawler_type}-${build_version}/crawl-${crawler_type}.*
+RUN rm *.zip

.github/workflows/docker-files/Dockerfile-committers-template

@@ -0,0 +1,13 @@
+FROM replace-this-with-real-version
+ARG committer_type
+ARG dl_filename
+ARG committer_version
+ARG build_version
+ARG crawler_type
+
+WORKDIR /nxer
+
+COPY downloaded/$committer_type/*.zip .
+RUN unzip ${dl_filename}
+RUN cp nx-committer-${committer_type}-${committer_version}/lib/* nx-crawler-${crawler_type}-${build_version}/lib/
+RUN rm *.zip

.github/workflows/merge-notification.yml

@@ -0,0 +1,90 @@
+name: Proactive notification of branch conflict
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  test-merge:
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Install Git
+        run: sudo apt-get install git -y
+
+      - name: setup mail service
+        run: sudo apt install mailutils -y
+
+      # Clone the branch, setup user information, and checkout main branch for later files checking  
+      - name: Fetch all branches and setup user information
+        run: |
+          git clone https://github.com/Norconex/crawlers.git /home/runner/work/crawlers/crawlers
+          cd /home/runner/work/crawlers/crawlers            
+          git config user.name "GitHub Action"
+          git config user.email "${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com"
+          git checkout main
+
+      # check if core files are being modified
+      - name: Get modified files           
+        id: modified-files        
+        uses: tj-actions/changed-files@v42.0.5
+        with:
+          files_ignore: |     
+            **/.github/**
+            **/assembly/**
+            **/**.txt
+            **/**.md
+            .gitignore
+
+      # start testing main branch into each sub-branches, only run if core files are modified
+      - name: Test merge main into branches
+        if: steps.modified-files.outputs.any_changed == 'true'
+        run: |          
+          for branch in $(git branch -r | grep -v 'origin/main' | cut -d '/' -f 2-); do
+            # only test branches that does not contains "snyk, dependabot, 3.x, 2.x, bugfix/unknown-properties in branch name
+            if [[ $branch != *"snyk"* && $branch != *"dependabot"* && $branch != *".x-branch"* ]]; then
+                                                      
+              git checkout -b temp-merge-$branch origin/$branch
+
+              # Extract the branch name from the format xxxx/branch-name
+              branch_name=$(echo "$branch" | cut -d '/' -f 2-)
+
+              # Get the last commit author of all the committers on the branch, only committers who has a valid norconex or gmail email will be emailed. 
+              COMMIT_AUTHORS=$(git log remotes/origin/$branch --format='%ae' | sort -u | grep 'gmail\|norconex')
+                                         
+              # start merging the main branch into sub-branch
+              if git merge --no-commit --no-ff origin/main; then
+                
+                # notify branch committers merge succeeded
+                echo "Merging main into $branch succeeded"
+                                
+                for author in $COMMIT_AUTHORS; do
+                  echo "Commit author: $author"   
+                  # email committers changes in main can be merged into his/her branch without conflicts
+                  echo "There are updates to the main branch, and test merge to your branch succeeded. Please update your branch. Thanks" | mail -s "Test Merge succeeded - Branch: $branch_name" $author                                 
+                done
+                                             
+                git merge --abort
+
+              else
+                # notify branch committers merge failed
+                echo "Merging main into $branch failed"
+
+                # Save the result of the merge command to a file
+                touch failed-report-$branch_name.diff
+
+                git diff > failed-report-$branch_name.diff
+                more failed-report-$branch_name.diff
+                
+                for author in $COMMIT_AUTHORS; do
+                  echo "Commit author: $author"                  
+                  # email committers that changes in main has conflicts with his/her branch, and attached the details for review.
+                  echo "There are updates to the main branch, but test merge failed, attached is the details of the conflicts. Please resolve the conflicts and keep your branch up to date. Thanks" | mail -s "Test Merge FAILED - Branch: $branch_name" -A failed-report-$branch_name.diff $author
+                done
+                
+                git merge --abort
+              fi               
+            fi
+          done