add NLP pkgs (#96) #66
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build-docker-images | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths-ignore: | |
| - "*.md" | |
| pull_request: | |
| branches: [ main ] | |
| paths-ignore: | |
| - "*.md" | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| env: | |
| REGISTRY_URL: "docker.io" # docker.io or other registry URL, DOCKER_REGISTRY_USERNAME/DOCKER_REGISTRY_PASSWORD to be set in CI env. | |
| BUILDKIT_PROGRESS: "plain" # Full logs for CI build. | |
| # DOCKER_REGISTRY_USERNAME and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets. | |
| DOCKER_REGISTRY_USERNAME: ${{ secrets.DOCKER_REGISTRY_USERNAME }} | |
| DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} | |
| # used to sync image to mirror registry | |
| DOCKER_MIRROR_REGISTRY_USERNAME: ${{ secrets.DOCKER_MIRROR_REGISTRY_USERNAME }} | |
| DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }} | |
| jobs: | |
| qpod_base: | |
| name: 'atom,base,python-3.11' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image atom latest docker_atom/Dockerfile && push_image | |
| build_image python-3.11 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.11" && alias_image python-3.11 latest base latest && push_image | |
| qpod_python: | |
| name: 'python-3.7,python-3.12' | |
| needs: 'qpod_base' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image python-3.7 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.7" && push_image | |
| build_image python-3.12 latest docker_base/Dockerfile --build-arg "PYTHON_VERSION=3.12" && push_image | |
| qpod_core: | |
| name: 'core' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image core latest docker_core/Dockerfile \ | |
| --build-arg "ARG_PROFILE_PYTHON=base,datascience,mkl,database,nlp,cv,chem,tf2,torch" \ | |
| --build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \ | |
| --build-arg "ARG_PROFILE_NODEJS=base" \ | |
| --build-arg "ARG_PROFILE_JAVA=base,maven" \ | |
| --build-arg "ARG_PROFILE_LATEX=base,cjk" | |
| push_image core | |
| qpod_py-data: | |
| name: 'py-data' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: source ./tool.sh && build_image py-data latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database" && push_image | |
| qpod_py-chem: | |
| name: 'py-chem' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: source ./tool.sh && build_image py-chem latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,chem" && push_image | |
| qpod_py-std: | |
| name: 'py-std' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: source ./tool.sh && build_image py-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database,nlp,cv,chem" && push_image | |
| qpod_py-jdk: | |
| name: 'py-jdk' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image py-jdk latest docker_core/Dockerfile \ | |
| --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,database,nlp,cv,chem" \ | |
| --build-arg "ARG_PROFILE_JAVA=base" | |
| push_image | |
| qpod_r-base: | |
| name: 'r-base' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: source ./tool.sh && build_image r-base latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base" && push_image | |
| qpod_r-std: | |
| name: 'r-std' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: source ./tool.sh && build_image r-std latest docker_core/Dockerfile --build-arg "ARG_PROFILE_R=base,datascience" --build-arg "ARG_PROFILE_JAVA=base" && push_image | |
| qpod_r-studio: | |
| name: 'r-studio' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image r-studio latest docker_core/Dockerfile \ | |
| --build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \ | |
| --build-arg "ARG_PROFILE_JAVA=base" \ | |
| --build-arg "ARG_PROFILE_LATEX=base,cjk" | |
| push_image | |
| qpod_node: | |
| name: 'node' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: source ./tool.sh && build_image node latest docker_core/Dockerfile --build-arg "ARG_PROFILE_NODEJS=base" && push_image | |
| qpod_jdk: | |
| name: 'jdk,jdk-8,jdk-11,jdk-17' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image jdk-17 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=17" | |
| build_image jdk-11 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=11" | |
| build_image jdk-8 latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JAVA=base" --build-arg "VERSION_JDK=8" | |
| alias_image jdk-11 latest jdk latest | |
| push_image jdk | |
| qpod_julia: | |
| name: 'julia' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: source ./tool.sh && build_image julia latest docker_core/Dockerfile --build-arg "ARG_PROFILE_JULIA=base" && push_image | |
| qpod_go: | |
| name: 'go,go-stack' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image go latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" | |
| build_image go-stack latest docker_core/Dockerfile --build-arg "ARG_PROFILE_GO=base" --build-arg "ARG_PROFILE_NODEJS=base" | |
| push_image go | |
| qpod_rust: | |
| name: 'rust' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: source ./tool.sh && build_image rust latest docker_core/Dockerfile --build-arg "ARG_PROFILE_RUST=base" && push_image | |
| qpod_data-science-stack: | |
| name: 'data-science-stack' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image data-science-stack latest docker_core/Dockerfile \ | |
| --build-arg "ARG_PROFILE_PYTHON=base,database" \ | |
| --build-arg "ARG_PROFILE_R=base,rstudio,rshiny" \ | |
| --build-arg "ARG_PROFILE_NODEJS=base" \ | |
| --build-arg "ARG_PROFILE_JULIA=base" | |
| push_image data-science-stack | |
| qpod_full-stack: | |
| name: 'full-stack' | |
| needs: qpod_base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image full-stack latest docker_core/Dockerfile \ | |
| --build-arg "ARG_PROFILE_PYTHON=base,database" \ | |
| --build-arg "ARG_PROFILE_R=base,rstudio,rshiny" \ | |
| --build-arg "ARG_PROFILE_NODEJS=base" \ | |
| --build-arg "ARG_PROFILE_JAVA=base,maven" \ | |
| --build-arg "ARG_PROFILE_JULIA=base" \ | |
| --build-arg "ARG_PROFILE_GO=base" \ | |
| --build-arg "ARG_PROFILE_RUST=base" | |
| push_image full-stack | |
| # cuda docker image tags: https://hub.docker.com/r/nvidia/cuda/tags | |
| # latest cuda supported by torch: https://pytorch.org/get-started/locally/ | |
| # latest cuda supported by tensorflow: https://tensorflow.google.cn/install/source?hl=en#gpu | |
| # latest cuda supported by paddlepadle: https://www.paddlepaddle.org.cn/ | |
| qpod_cuda_118: | |
| name: 'cuda,cuda_11.8' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image tmp latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04" && clear_images nvidia/cuda | |
| build_image tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=tmp" | |
| build_image cuda_11.8 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=tmp" | |
| alias_image cuda_11.8 latest cuda latest | |
| push_image cuda | |
| qpod_cuda_112: | |
| name: 'cuda_11.2' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image tmp latest docker_atom/Dockerfile --build-arg "BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04" && clear_images nvidia/cuda | |
| build_image tmp latest docker_base/Dockerfile --build-arg "BASE_IMG=tmp" --build-arg "PYTHON_VERSION=3.8" | |
| build_image cuda_11.2 latest docker_cuda/Dockerfile --build-arg "BASE_IMG=tmp" | |
| push_image cuda | |
| qpod_tf2: | |
| name: 'tf2,tf2-cuda112' | |
| needs: qpod_cuda_112 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image tf2-cuda112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=tf2" | |
| alias_image tf2-cuda112 latest tf2 latest | |
| push_image | |
| qpod_torch_cuda118: | |
| name: 'torch,torch-cuda118' | |
| needs: qpod_cuda_118 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image torch-cuda118 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.8" --build-arg "ARG_PROFILE_PYTHON=torch" | |
| alias_image torch-cuda118 latest torch latest | |
| push_image | |
| qpod_torch_cuda112: | |
| name: 'torch-cuda112' | |
| needs: qpod_cuda_112 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image torch-cuda112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=torch" | |
| push_image | |
| qpod_paddle_cuda118: | |
| name: 'paddle-cuda,paddle-cuda118' | |
| needs: qpod_cuda_118 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image paddle-cuda118 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.8" --build-arg "ARG_PROFILE_PYTHON=paddle,mkl" | |
| alias_image paddle-cuda118 latest paddle-cuda latest | |
| push_image | |
| qpod_paddle_cuda112: | |
| name: 'paddle-cuda112' | |
| needs: qpod_cuda_112 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image paddle-cuda112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=paddle,mkl" | |
| push_image | |
| qpod_py-nlp: | |
| name: 'py-nlp,py-nlp-cuda118' | |
| needs: qpod_cuda_118 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image py-nlp-cuda118 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.8" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,nlp" | |
| alias_image py-nlp-cuda118 latest py-nlp latest | |
| push_image | |
| qpod_py-nlp-cuda112: | |
| name: 'py-nlp-cuda112' | |
| needs: qpod_cuda_112 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image py-nlp-cuda112 latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.2" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,nlp" | |
| push_image | |
| qpod_py-cv: | |
| name: 'py-cv' | |
| needs: qpod_cuda_118 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image py-cv latest docker_core/Dockerfile --build-arg "BASE_IMG=cuda_11.8" --build-arg "ARG_PROFILE_PYTHON=datascience,mkl,torch,cv" | |
| push_image | |
| qpod_core-cuda: | |
| name: 'core-cuda,full-cuda-11.8' | |
| needs: qpod_cuda_118 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh && free_diskspace | |
| build_image full-cuda-11.8 latest docker_core/Dockerfile \ | |
| --build-arg "BASE_IMG=cuda_11.8" \ | |
| --build-arg "ARG_PROFILE_PYTHON=base,datascience,mkl,database,nlp,cv,chem,tf2,torch" \ | |
| --build-arg "ARG_PROFILE_R=base,datascience,rstudio,rshiny" \ | |
| --build-arg "ARG_PROFILE_NODEJS=base" \ | |
| --build-arg "ARG_PROFILE_JAVA=base,maven" \ | |
| --build-arg "ARG_PROFILE_LATEX=base,cjk" | |
| alias_image full-cuda-11.8 latest core-cuda latest && push_image cuda | |
| ## Postgres | |
| qpod_postgres: | |
| name: 'postgres-15,postgres-16' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| build_image postgres-15 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:15" | |
| build_image postgres-15 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-15" --build-arg "PYTHON_VERSION=3.11" | |
| build_image postgres-16 latest docker_atom/Dockerfile --build-arg "BASE_IMG=postgres:16" | |
| build_image postgres-16 latest docker_base/Dockerfile --build-arg "BASE_IMG=postgres-16" --build-arg "PYTHON_VERSION=3.11" | |
| push_image postgres- | |
| ## DockerKit - including image-syncer and docker-compsoe | |
| qpod_docker_kit: | |
| name: 'docker-kit' | |
| needs: ["qpod_base"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile && push_image | |
| ## Sync all images in this build (listed by "names") to mirror registry. | |
| sync_images: | |
| needs: ["qpod_docker_kit", "qpod_core-cuda", "qpod_py-cv", "qpod_py-nlp"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| source ./tool.sh | |
| printenv > /tmp/docker.env | |
| docker run --rm \ | |
| --env-file /tmp/docker.env \ | |
| -v $(pwd):/tmp \ | |
| -w /tmp \ | |
| ${IMG_NAMESPACE:-qpod}/docker-kit python /opt/utils/image-syncer/run_jobs.py |