add-rpl (PaddlePaddle#1)

* add-rpl

* add arm

buildspec.yml

@@ -0,0 +1,109 @@
+# Overview
+#
+# This file is used to configure your build environment and run your build commands
+# and scripts.
+#
+# Usage
+#
+#   - Define requirements for your build environment
+#   - Install any required tools or packages
+#   - Run any build commands or scripts to build your project
+#   - Promote your artifact using the supply-chain CLI to the deployment environment(s) defined in TARGET_ENVS
+# 
+# The example that we provide for the name of the artifact in this buildspec.yml file names the file 
+# using the git branch name found in the $CODEBUILD_WEBHOOK_TRIGGER environment variable.
+#
+# At the end you will create a zip file of your deployment workspace and promote it using the supply-chain CLI. This zip file:
+#    - Must include the deployspec.yml
+#    - Must include any deployment scripts used to deploy
+#  
+#  More Information:
+#
+#  - AWS buildspec.yml - https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html
+#  - supply-chain CLI - https://github.concur.com/ktg/supply-chain#readme
+
+version: 0.2
+
+env:
+  #
+  # List of secrets (how to create : https://wiki.concur.com/confluence/display/CSS/Secret+Management+-+Release+Pipeline)
+  #
+  parameter-store:
+    QUAY_ROBOT_USERNAME: "/fapiao/quay/login"
+    QUAY_ROBOT_TOKEN: "/fapiao/quay/token"
+
+    GITHUB_API_TOKEN: "/fapiao/github/APIToken"
+    # Used to interact with Slack
+    SLACK_WEBHOOK_URL: "/fapiao/slack/SLACK_WEBHOOK_URL"
+  # secrets-manager:
+  #   QUAY_USER: fapiao/quay/login
+  #   QUAY_TOKEN: fapiao/quay/token
+
+  #
+  # List of variables
+  #
+  variables:
+    ORG: fapiao
+    REPO: PaddleOCR
+    ROLETYPE: fapiao
+    GIT_BRANCH: "" # to be computed in pre-build phase
+    GITHUB_CLI_VERSION: "2.7.0"
+    GITHUB_REPOSITORY: "fapiao/PaddleOCR"
+    GITHUB_NAMESPACE_REPOS_REPOSITORY: "namespace-repos/efapiao"
+
+    QUAY_HOSTNAME: quay.cnqr.delivery
+    QUAY_REPOSITORY: "fapiao/paddlehub"
+    SERVICE_MESH_VERSION: "0.2888.0"
+
+    # Images used for pipeline
+    KATE_IMAGE: "quay.cnqr.delivery/kraken/kate:v1.0.4"
+    SUPPLY_CHAIN_IMAGE: "quay.cnqr.delivery/ktg/supply-chain@sha256:f7a6b23c87cd45e744bbab4d4911b604a178f8c60da46d92c12f106bfdb9d885" # v0.0.5
+
+    # Will host the absolute path of the current directory
+    WORKDIR: "" # to be computed in pre-build phase
+    # Build status
+    BUILD_SUCCESSFUL: "N"
+
+phases:
+  install:
+    runtime-versions:
+      docker: 19
+    commands:
+      # Install GitHub CLI
+      - rpm -i https://github.com/cli/cli/releases/download/v${GITHUB_CLI_VERSION}/gh_${GITHUB_CLI_VERSION}_linux_386.rpm
+  pre_build:
+    on-failure: ABORT
+    commands:
+      # Connects to GitHub
+      - echo ${GITHUB_API_TOKEN} | gh auth login --hostname github.concur.com --with-token
+      # Pull shared scripts
+      - gh auth setup-git
+      - gh repo clone eden/secure-pipeline-scripts
+      # Allows execution of embedded pipeline scripts and functions
+      - chmod +x -R ./scripts/*
+      - chmod +x -R ./secure-pipeline-scripts/*
+      # Set WORKDIR
+      - WORKDIR=$(pwd)
+      # Identify the current branch name
+      - GIT_BRANCH=$(echo "${CODEBUILD_WEBHOOK_HEAD_REF}" | sed "s@refs/heads/@@") 
+      - GIT_SHORT_COMMIT=$(git log -n 1 --format=%h)
+      - COMMIT_AUTHOR=$(git log -n 1 --format=%an)
+      - echo ">>> ${CODEBUILD_WEBHOOK_HEAD_REF}"
+      - echo ">>> Branch -> ${GIT_BRANCH}"
+  build:
+    commands:
+      - ./scripts/pipeline/pipeline.sh
+      - BUILD_SUCCESSFUL="Y"
+    finally:
+      - |
+        if [ "${BUILD_SUCCESSFUL}" == "N" ]; then
+          "${WORKDIR}"/secure-pipeline-scripts/notification/slack.sh "#fapiao-ci" "Repo: ${GITHUB_REPOSITORY} Branch : ${GIT_BRANCH}" "danger" "${GIT_SHORT_COMMIT} by ${COMMIT_AUTHOR} Build failed" ""
+        else
+          "${WORKDIR}"/secure-pipeline-scripts/notification/slack.sh "#fapiao-ci" "Repo: ${GITHUB_REPOSITORY} Branch : ${GIT_BRANCH}" "good" "${GIT_SHORT_COMMIT} by ${COMMIT_AUTHOR} Build completed" ""
+        fi
+
+# reports:
+#   junit-reports:
+#     files:
+#       - "**/build/test-results/test/*.xml"
+#     file-format: "JUNITXML"

deploy/docker/hubserving/pretrain-hub/Dockerfile

@@ -6,4 +6,4 @@ RUN hub install chinese_ocr_db_crnn_mobile chinese_text_detection_db_mobile
 
 EXPOSE 8866
 
-CMD ["/bin/bash", "-c", "hub serving start -m chinese_ocr_db_crnn_mobile -p 8866"]
+CMD ["/bin/bash", "-c", "hub serving start -m chinese_ocr_db_crnn_mobile -p 8866"]

deploy/docker/hubserving/pretrain-hub/paddle-arm.dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.8
+
+COPY paddle/paddlepaddle-2.3.1-cp38-cp38-linux_aarch64.whl paddlepaddle-2.3.1-cp38-cp38-linux_aarch64.whl
+
+RUN pip install -U paddlepaddle-2.3.1-cp38-cp38-linux_aarch64.whl
+
+RUN rm paddlepaddle-2.3.1-cp38-cp38-linux_aarch64.whl
+
+RUN apt update
+
+RUN apt install cmake protobuf-compiler ffmpeg libsm6 libxext6  -y
+
+RUN pip install opencv-python==4.6.0.66 shapely pyclipper
+
+ENTRYPOINT [ "/bin/bash" ]

deploy/docker/hubserving/pretrain-hub/paddlehub-arm.dockerfile

@@ -0,0 +1,9 @@
+FROM felix637/paddle:2.3.1-aarch64
+
+RUN pip install paddlehub
+
+RUN hub install chinese_ocr_db_crnn_mobile chinese_text_detection_db_mobile
+
+EXPOSE 8866
+
+ENTRYPOINT ["/bin/bash", "-c", "hub serving start -m chinese_ocr_db_crnn_mobile -p 8866"]

scripts/pipeline/functions/notification.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+#########################################################
+### This file gathers functions related to notifications.
+#########################################################
+
+#
+# Main function to send slack message in #raas-ci.
+#
+function send_slack_message {
+  local title="${1}"
+  local color="${2}"
+  local message="${3}"
+  local channel="#fapiao-ci"
+  local footer="Click <https://codebuild.cnqr.delivery|here> and select 'fapiao' to log in AWS build account, then click <${CODEBUILD_BUILD_URL}|here> to access your build logs."
+
+  echo ">>> Send notification in Slack channel ${channel} :
+          message: ${message}
+  "
+
+  "${WORKDIR}"/secure-pipeline-scripts/notification/slack.sh "${channel}" "${title}" "${color}" "${message}" "${footer}"
+}

scripts/pipeline/pipeline.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -e # exit immediately after any error
+
+#######################################################################################################
+### This is the pipeline script that will be run by buildspec.yml
+###
+### It follows the following steps :
+###   1. Initialization : setup environment variables
+###         --> if release/hotfix : create GitHub release
+###   2. Build : build service and run tests
+###   3. Packaging : build and push docker image
+###   4. Promotion : promote third party images, service-mesh images, and docker image from step 3.
+###   5. Deployment : render helm manifests and create PRs against namespace-repos/raas.
+###   6. Post-deployment : additional actions processed for some scenario (auto-merge, E2E tests, ...)
+#######################################################################################################
+
+#
+# 1. Initialization
+#
+. "${WORKDIR}/scripts/pipeline/steps/initialization.sh"
+
+
+if [ "${IS_DEVELOP_BRANCH}" == "Y" ]; then
+  #
+  # 3. Packaging
+  #
+  . "${WORKDIR}/scripts/pipeline/steps/packaging.sh"
+  #
+  # 4. Promotion
+  #
+  . "${WORKDIR}/secure-pipeline-scripts/promotion/promote_container.sh" "${IMAGE_TAG}" "${CODEBUILD_RESOLVED_SOURCE_VERSION}" "${GITHUB_REPOSITORY}" "${ROLETYPE}"
+fi
+#

scripts/pipeline/steps/initialization.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+set -e # exit immediately after any error
+
+echo "
+#########################################################################################
+### STEP : Initialization
+###
+### - Setup environment variables for the pipeline
+### - Conditional tasks :
+###     --> release/hotfix : create GitHub release
+#########################################################################################
+"
+
+#
+# 0. Import functions
+#
+# . "${WORKDIR}/scripts/pipeline/functions/github.sh"
+. "${WORKDIR}/scripts/pipeline/functions/notification.sh"
+
+#
+# 1. Set possible versions variables
+#
+
+# CURRENT_RELEASE=$(gh release -R "${GITHUB_REPOSITORY}" view --json name | jq -r .name)
+# SPLIT_RELEASE=(${CURRENT_RELEASE//./ })
+# MAJOR=${SPLIT_RELEASE[0]}
+# MINOR=${SPLIT_RELEASE[1]}
+# POINT=${SPLIT_RELEASE[2]}
+
+# LONG_CURRENT_VERSION="${CURRENT_RELEASE}-${GIT_SHORT_COMMIT}"
+# LONG_CURRENT_VERSION_WITH_TIMESTAMP="${LONG_CURRENT_VERSION}-$(date +%s)"
+# NEXT_SHORT_RELEASE_VERSION="${MAJOR}.$(expr ${MINOR} + 1).0"
+# NEXT_LONG_RELEASE_VERSION="${NEXT_SHORT_RELEASE_VERSION}-${GIT_SHORT_COMMIT}"
+# NEXT_SHORT_HOTFIX_VERSION="${MAJOR}.${MINOR}.$(expr ${POINT} + 1)"
+
+# echo ">>> The possible new versions are:
+#         feature:          ${LONG_CURRENT_VERSION_WITH_TIMESTAMP}
+#         deploy:           ${LONG_CURRENT_VERSION}
+#         develop:          ${NEXT_LONG_RELEASE_VERSION}
+#         release:          ${NEXT_SHORT_RELEASE_VERSION}
+#         hotfix:           ${NEXT_SHORT_HOTFIX_VERSION}
+# "
+
+#
+# 2. Declare environment variables with default values
+#
+# export NEW_VERSION="${LONG_CURRENT_VERSION_WITH_TIMESTAMP}"
+export IS_DEVELOP_BRANCH="N"
+
+#
+# 3. Updates environment variables based on branch pattern
+#
+case "${GIT_BRANCH}" in
+  "develop")
+    # NEW_VERSION="${NEXT_LONG_RELEASE_VERSION}"
+    IS_DEVELOP_BRANCH="Y"
+    ;;
+  *)
+    ;;
+esac
+
+#
+# 4. Additional environment variables
+#
+export IMAGE_TAG="${QUAY_HOSTNAME}/${QUAY_REPOSITORY}:${GIT_SHORT_COMMIT}"
+
+# export THIRD_PARTY_IMAGE_SERVICE_MESH_INTEGRATION="${QUAY_HOSTNAME}/servicemesh/integration:${SERVICE_MESH_VERSION}"
+# export THIRD_PARTY_IMAGE_SERVICE_MESH_BEIPR1="${QUAY_HOSTNAME}/servicemesh/beipr1:${SERVICE_MESH_VERSION}"
+
+#
+# 5. Initialization output
+#
+echo ">>> Initialization output :"
+env
+
+#
+# 6. Send slack notification
+#
+send_slack_message "Repo: ${GITHUB_REPOSITORY} Branch: ${GIT_BRANCH}" "#87ceeb" "Build started with version ${GIT_SHORT_COMMIT} by ${COMMIT_AUTHOR} (pattern: ${BRANCH_PATTERN})"
+
+#
+# 7. Create GitHub release (release/hotfix only)
+#
+# if [ "${IS_RELEASE_HOTFIX_BRANCH}" == "Y" ]; then
+#   run_create_github_release
+# fi

scripts/pipeline/steps/packaging.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -e # exit immediately after any error
+
+echo "
+#############################################################################################################
+### STEP : Packaging
+###
+### - Build and Push the Docker image
+### - Export the IMAGE_DIGEST to be injected in k8s manifests
+#############################################################################################################
+"
+
+#
+# 1. Prepare Docker --build-arg arguments
+#
+DOCKER_BUILD_ARGS="--build-arg BUILD_VERSION=${GIT_SHORT_COMMIT}"
+
+#
+# 2. Build + Push docker image, then export IMAGE_DIGEST
+#
+. "${WORKDIR}/secure-pipeline-scripts/packaging/docker.sh" "${IMAGE_TAG}" "deploy/docker/hubserving/pretrain-hub/Dockerfile" "./" "${DOCKER_BUILD_ARGS}"
+export IMAGE_DIGEST=${IMAGE_DIGEST}
+