diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 0000000..e18a96e --- /dev/null +++ b/.github/README.md @@ -0,0 +1,23 @@ +# Github Actions + +## Test CI locally + +```bash +act \ + --secret AWS_ACCESS_KEY_ID \ + --secret AWS_SECRET_ACCESS_KEY \ + --secret AWS_REGION \ + --job auto-update +``` + +## Test in forked repository + +### Set secrets before running + +```bash +gh secret set AWS_ACCESS_KEY_ID --body "$AWS_ACCESS_KEY_ID"; +gh secret set AWS_SECRET_ACCESS_KEY --body "$AWS_SECRET_ACCESS_KEY"; +gh secret set AWS_REGION --body "$AWS_REGION"; +gh secret set DOCKER_USERNAME --body "$DOCKER_USERNAME"; +gh secret set DOCKER_PASSWORD --body "$DOCKER_PASSWORD"; +``` diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml new file mode 100644 index 0000000..dc7e0d6 --- /dev/null +++ b/.github/workflows/auto-update.yml @@ -0,0 +1,154 @@ +name: auto-update + +on: + schedule: + - cron: "0 0 * * *" + push: + branches: + - feat/github-actions + +jobs: + auto-update: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install serverless + run: npm install -g serverless + - name: Note docker image digest + id: docker-image-digest + run: | + docker pull public.ecr.aws/lambda/python:latest + SHA256_DIGEST=$(docker inspect public.ecr.aws/lambda/python:latest | jq -r '.[0].RepoDigests[0] | split(":") | .[1]' ) + echo "SHA256_DIGEST=${SHA256_DIGEST}" >> $GITHUB_OUTPUT + - name: Note Chromium versions + id: chromium-versions + run: | + WHOLE_JSON=$(curl https://omahaproxy.appspot.com/all.json) + JSON=$(echo $WHOLE_JSON | jq -r '.[] | select(.os=="linux") | .versions[] | select(.channel=="stable")') + CHANNEL="stable" + POSITION=$(echo $JSON | jq -r '.branch_base_position') + if (( POSITION < 900000 )); then + JSON=$(echo $WHOLE_JSON | jq -r '.[] | select(.os=="linux") | .versions[] | select(.channel=="beta")') + CHANNEL="beta" + POSITION=$(echo $JSON | jq -r '.branch_base_position') + echo "CHANNEL=${CHANNEL}" >> $GITHUB_OUTPUT + fi + + MAJOR_VERSION=$(echo $JSON | jq -r '.version | split(".") | .[0]') + echo "MAJOR_VERSION=${MAJOR_VERSION}" >> $GITHUB_OUTPUT + + for _ in {1..100}; do + DOWNLOAD_URL="https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F${POSITION}%2Fchrome-linux.zip?alt=media" + curl -I $DOWNLOAD_URL | head -1 | grep -q 404 || break + POSITION=$(($POSITION-1)) + sleep 1 + done + + echo "POSITION=${POSITION}" >> $GITHUB_OUTPUT + - name: Note chromedriver version + id: chromedriver-version + run: | + MAJOR_VERSION=${{ steps.chromium-versions.outputs.MAJOR_VERSION }} + DRIVER_VERSION=$(curl "https://chromedriver.storage.googleapis.com/LATEST_RELEASE_${MAJOR_VERSION}") + echo "DRIVER_VERSION=${DRIVER_VERSION}" >> $GITHUB_OUTPUT + - name: Update Dockerfile + run: | + SHA256_DIGEST=${{ steps.docker-image-digest.outputs.SHA256_DIGEST }} + POSITION=${{ steps.chromium-versions.outputs.POSITION }} + DRIVER_VERSION=${{ steps.chromedriver-version.outputs.DRIVER_VERSION }} + sed -r "s/public.ecr.aws\/lambda\/python[:@a-z0-9]+/public.ecr.aws\/lambda\/python\@sha256\:${SHA256_DIGEST}/g; s/chromedriver.storage.googleapis.com\/[0-9.]+/chromedriver.storage.googleapis.com\/${DRIVER_VERSION}/g; s/Linux_x64%2F[0-9]+%2Fchrome-linux.zip/Linux_x64%2F${POSITION}%2Fchrome-linux.zip/g" -i Dockerfile + - name: Deploy + run: sls deploy + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + - name: Note chrome version + id: chrome-version + run: | + CHROME_VERSION=$(docker run --rm --entrypoint '' serverless-docker-selenium-lambda-prod:img /opt/chrome/chrome --version | awk '{print $2}' | sed -e 's/^[[:space:]]*//') + echo "CHROME_VERSION=${CHROME_VERSION}" >> $GITHUB_OUTPUT + - name: Note selenium version + id: selenium-version + run: | + SELENIUM_VERSION=$(docker run --rm --entrypoint '' serverless-docker-selenium-lambda-prod:img pip freeze | grep selenium | awk -F "==" '{print $2}') + echo "SELENIUM_VERSION=${SELENIUM_VERSION}" >> $GITHUB_OUTPUT + - name: Note python version + id: python-version + run: | + PYTHON_VERSION=$(docker run --rm --entrypoint '' serverless-docker-selenium-lambda-prod:img python -V | awk '{print $2}') + echo "PYTHON_VERSION=${PYTHON_VERSION}" >> $GITHUB_OUTPUT + - name: Invoke + id: invoke + run: sls invoke -f demo > /tmp/scraping-result.txt + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + - name: Archive result + uses: actions/upload-artifact@v3 + if: ${{ !env.ACT }} + with: + name: scraping-result + path: /tmp/scraping-result.txt + - name: Test + run: cat /tmp/scraping-result.txt | grep -q "This domain is for use in illustrative examples in documents" + - name: Update README + run: | + CHROME_VERSION=${{ steps.chrome-version.outputs.CHROME_VERSION }} + DRIVER_VERSION=${{ steps.chromedriver-version.outputs.DRIVER_VERSION }} + SELENIUM_VERSION=${{ steps.selenium-version.outputs.SELENIUM_VERSION }} + PYTHON_VERSION=${{ steps.python-version.outputs.PYTHON_VERSION }} + sed -r "s/- chromium [0-9.]+/- chromium ${CHROME_VERSION}/g; s/- chromedriver [0-9.]+/- chromedriver ${DRIVER_VERSION}/g; s/- selenium [0-9.]+/- selenium ${SELENIUM_VERSION}/g; s/- Python [0-9.]+/- Python ${PYTHON_VERSION}/g" -i README.md + - name: Detect changes + id: detect-changes + run: | + DO_RELEASE="yes" + git --no-pager diff --name-only | grep -q "README.md" || DO_RELEASE="no" + git --no-pager diff --name-only | grep -q "Dockerfile" || DO_RELEASE="no" + echo "DO_RELEASE=${DO_RELEASE}" >> $GITHUB_OUTPUT + - name: Setup git config + run: | + # https://qiita.com/thaim/items/3d1a4d09ec4a7d8844ce + git config user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + - name: Release + if: ${{ steps.detect-changes.outputs.DO_RELEASE == 'yes' && !env.ACT }} + run: | + BRANCH=$(date +%Y-%m-%d-%H%M%S) + TITLE="Version Updates $(date)" + git checkout -b $BRANCH + git add Dockerfile README.md + echo "SHA256_DIGEST=${{ steps.docker-image-digest.outputs.SHA256_DIGEST }}" > /tmp/body-text.txt + echo "POSITION=${{ steps.chromium-versions.outputs.POSITION }}" >> /tmp/body-text.txt + echo "DRIVER_VERSION=${{ steps.chromedriver-version.outputs.DRIVER_VERSION }}" >> /tmp/body-text.txt + echo "PYTHON_VERSION=${{ steps.python-version.outputs.PYTHON_VERSION }}" >> /tmp/body-text.txt + echo "SELENIUM_VERSION=${{ steps.selenium-version.outputs.SELENIUM_VERSION }}" >> /tmp/body-text.txt + echo "CHROME_VERSION=${{ steps.chrome-version.outputs.CHROME_VERSION }}" >> /tmp/body-text.txt + echo "\n```diff\n$(git diff --staged)\n```" >> /tmp/body-text.txt + git commit -m "${TITLE}" + git push --set-upstream origin $BRANCH + gh pr create --body-file /tmp/body-text.txt --title "PR:${TITLE}" + gh pr merge --delete-branch --merge $BRANCH + gh release create $BRANCH --notes-file /tmp/body-text.txt --title "${TITLE}" + env: + GH_TOKEN: ${{ github.token }} + - name: Publish image + if: ${{ steps.detect-changes.outputs.DO_RELEASE == 'yes' && !env.ACT }} + run: | + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + PYTHON_VERSION=${{ steps.python-version.outputs.PYTHON_VERSION }} + SELENIUM_VERSION=${{ steps.selenium-version.outputs.SELENIUM_VERSION }} + CHROME_VERSION=${{ steps.chrome-version.outputs.CHROME_VERSION }} + MAJOR_PYTHON_VERSION=$(echo $PYTHON_VERSION | cut -d "." -f 1) + MINOR_PYTHON_VERSION=$(echo $PYTHON_VERSION | cut -d "." -f 2) + docker image tag serverless-docker-selenium-lambda-prod:img umihico/aws-lambda-selenium-python:latest + docker image tag serverless-docker-selenium-lambda-prod:img umihico/aws-lambda-selenium-python:${MAJOR_PYTHON_VERSION} + docker image tag serverless-docker-selenium-lambda-prod:img umihico/aws-lambda-selenium-python:${MAJOR_PYTHON_VERSION}.${MINOR_PYTHON_VERSION} + docker image tag serverless-docker-selenium-lambda-prod:img umihico/aws-lambda-selenium-python:${PYTHON_VERSION} + docker image tag serverless-docker-selenium-lambda-prod:img umihico/aws-lambda-selenium-python:${PYTHON_VERSION}-selenium${SELENIUM_VERSION} + docker image tag serverless-docker-selenium-lambda-prod:img umihico/aws-lambda-selenium-python:${PYTHON_VERSION}-chrome${CHROME_VERSION} + docker image tag serverless-docker-selenium-lambda-prod:img umihico/aws-lambda-selenium-python:${PYTHON_VERSION}-selenium${SELENIUM_VERSION}-chrome${CHROME_VERSION} + docker image push --all-tags umihico/aws-lambda-selenium-python diff --git a/Dockerfile b/Dockerfile index 3d42abf..473ebda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM public.ecr.aws/lambda/python@sha256:ac758b6345b677eb6e72c2cd0f71c182eca773da5db1e17835a4ffef46dbc997 as build +FROM public.ecr.aws/lambda/python@sha256:75fc5649fde271db33378885fdbfd7bf6c153b1d642d895053036b272a8160b4 as build RUN yum install -y unzip && \ - curl -Lo "/tmp/chromedriver.zip" "https://chromedriver.storage.googleapis.com/111.0.5563.64/chromedriver_linux64.zip" && \ - curl -Lo "/tmp/chrome-linux.zip" "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1097615%2Fchrome-linux.zip?alt=media" && \ + curl -Lo "/tmp/chromedriver.zip" "https://chromedriver.storage.googleapis.com/113.0.5672.63/chromedriver_linux64.zip" && \ + curl -Lo "/tmp/chrome-linux.zip" "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1121454%2Fchrome-linux.zip?alt=media" && \ unzip /tmp/chromedriver.zip -d /opt/ && \ unzip /tmp/chrome-linux.zip -d /opt/ -FROM public.ecr.aws/lambda/python@sha256:ac758b6345b677eb6e72c2cd0f71c182eca773da5db1e17835a4ffef46dbc997 +FROM public.ecr.aws/lambda/python@sha256:75fc5649fde271db33378885fdbfd7bf6c153b1d642d895053036b272a8160b4 RUN yum install atk cups-libs gtk3 libXcomposite alsa-lib \ libXcursor libXdamage libXext libXi libXrandr libXScrnSaver \ libXtst pango at-spi2-atk libXt xorg-x11-server-Xvfb \ diff --git a/README.md b/README.md index e3f28ef..e8f32c8 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,15 @@ # docker-selenium-lambda -This is minimum demo of headless chrome and selenium on container image on AWS Lambda +![badge](https://github.com/umihico/docker-selenium-lambda/actions/workflows/auto-update.yml/badge.svg) -This image goes with these versions. [These are automatically updated and tested everyday. ![CircleCI](https://circleci.com/gh/umihico/docker-selenium-lambda/tree/circleci.svg?style=svg)](https://circleci.com/gh/umihico/docker-selenium-lambda/tree/circleci) +This is minimum demo of headless chrome and selenium on container image on AWS Lambda -- Python 3.10.9 -- chromium 111.0.5563.0 -- chromedriver 111.0.5563.64 -- selenium 4.8.2 +This image goes with these versions. [These are automatically updated and tested everyday.](https://github.com/umihico/docker-selenium-lambda/actions) +- Python 3.10.11 +- chromium 113.0.5672.0 +- chromedriver 113.0.5672.63 +- selenium 4.9.1 ## Running the demo