Skip to content

Commit 8c384e0

Browse files
chensuyuechensuyue
authored
Build up docker images CD workflow (#576)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent 3c9e2aa commit 8c384e0

12 files changed

+264
-21
lines changed

‎.github/workflows/_example-workflow.yml

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Example jobs
5+
permissions: read-all
6+
on:
7+
workflow_call:
8+
inputs:
9+
example:
10+
required: true
11+
type: string
12+
tag:
13+
default: "latest"
14+
required: false
15+
type: string
16+
build:
17+
default: true
18+
required: false
19+
type: boolean
20+
scan:
21+
default: true
22+
required: false
23+
type: boolean
24+
test_compose:
25+
default: false
26+
required: false
27+
type: boolean
28+
test_k8s:
29+
default: false
30+
required: false
31+
type: boolean
32+
publish:
33+
default: false
34+
required: false
35+
type: boolean
36+
publish_tags:
37+
default: "latest"
38+
required: false
39+
type: string
40+
jobs:
41+
####################################################################################################
42+
# Image Build
43+
####################################################################################################
44+
build-images:
45+
if: ${{ fromJSON(inputs.build) }}
46+
strategy:
47+
matrix:
48+
node: ["docker-build-xeon", "docker-build-gaudi"]
49+
runs-on: ${{ matrix.node }}
50+
continue-on-error: true
51+
steps:
52+
- name: Clean Up Working Directory
53+
run: |
54+
sudo rm -rf ${{github.workspace}}/*
55+
56+
- name: Checkout out Repo
57+
uses: actions/checkout@v4
58+
59+
- name: Build Image
60+
uses: opea-project/validation/actions/image-build@main
61+
with:
62+
work_dir: ${{ github.workspace }}/${{ inputs.example }}
63+
docker_compose_path: ${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.example }}-compose.yaml
64+
registry: ${OPEA_IMAGE_REPO}opea
65+
tag: ${{ inputs.tag }}
66+
67+
####################################################################################################
68+
# Trivy Scan
69+
####################################################################################################
70+
image-list:
71+
needs: [ build-images ]
72+
if: ${{ fromJSON(inputs.scan) }}
73+
runs-on: ubuntu-latest
74+
outputs:
75+
matrix: ${{ steps.scan-matrix.outputs.matrix }}
76+
steps:
77+
- name: Harden Runner
78+
uses: step-security/harden-runner@v2.8.1
79+
with:
80+
egress-policy: audit
81+
82+
- name: Checkout out Repo
83+
uses: actions/checkout@v4
84+
85+
- name: Set Matrix
86+
id: scan-matrix
87+
run: |
88+
pip install yq
89+
compose_path=${{ github.workspace }}/.github/workflows/docker/compose/${{ inputs.example }}-compose.yaml
90+
echo "matrix=$(cat ${compose_path} | yq -r '.[]' | jq 'keys' | jq -c '.')" >> $GITHUB_OUTPUT
91+
92+
scan-images:
93+
needs: [image-list]
94+
if: ${{ fromJSON(inputs.scan) }}
95+
runs-on: "docker-build-gaudi"
96+
strategy:
97+
matrix:
98+
image: ${{ fromJSON(needs.image-list.outputs.matrix) }}
99+
fail-fast: false
100+
steps:
101+
- name: Harden Runner
102+
uses: step-security/harden-runner@v2.8.1
103+
with:
104+
egress-policy: audit
105+
106+
- name: Pull Image
107+
run: docker pull ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
108+
109+
- name: Scan Container
110+
uses: opea-project/validation/actions/trivy-scan@main
111+
with:
112+
image-ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
113+
output: ${{ inputs.example }}-${{ matrix.image }}-scan.txt
114+
115+
- name: Cleanup
116+
if: always()
117+
run: docker rmi -f ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
118+
- uses: actions/upload-artifact@v4.3.4
119+
with:
120+
name: ${{ inputs.example }}-scan
121+
path: ${{ inputs.example }}-${{ matrix.image }}-scan.txt
122+
overwrite: true
123+
124+
####################################################################################################
125+
# Docker Compose Test
126+
####################################################################################################
127+
test-example-compose:
128+
needs: [build-images]
129+
if: ${{ fromJSON(inputs.test_compose) }}
130+
strategy:
131+
matrix:
132+
hardware: ["xeon", "gaudi"]
133+
fail-fast: false
134+
uses: ./.github/workflows/_run-docker-compose.yml
135+
with:
136+
tag: ${{ inputs.tag }}
137+
example: ${{ inputs.example }}
138+
hardware: ${{ matrix.hardware }}
139+
secrets: inherit
140+
141+
142+
####################################################################################################
143+
# K8S Test
144+
####################################################################################################
145+
# TODO
146+
147+
148+
####################################################################################################
149+
# Publish
150+
####################################################################################################
151+
publish:
152+
needs: [image-list, build-images, scan-images, test-example-compose]
153+
if: ${{ fromJSON(inputs.publish) }}
154+
strategy:
155+
matrix:
156+
image: ${{ fromJSON(needs.image-list.outputs.matrix) }}
157+
runs-on: "docker-build-gaudi"
158+
steps:
159+
- name: Image Publish
160+
uses: opea-project/validation/actions/image-publish@main
161+
with:
162+
local_image_ref: ${OPEA_IMAGE_REPO}opea/${{ matrix.image }}:${{ inputs.tag }}
163+
image_name: opea/${{ matrix.image }}
164+
publish_tags: ${{ inputs.publish_tags }}

‎.github/workflows/docker/compose/AudioQnA-compose.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ services:
77
build:
88
context: docker
99
dockerfile: ./Dockerfile
10-
image: ${REGISTRY}opea/audioqna:${TAG:-latest}
10+
image: ${REGISTRY:-opea}/audioqna:${TAG:-latest}

‎.github/workflows/docker/compose/ChatQnA-compose.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ services:
77
build:
88
context: docker
99
dockerfile: ./Dockerfile
10-
image: ${REGISTRY}opea/chatqna:${TAG:-latest}
10+
image: ${REGISTRY:-opea}/chatqna:${TAG:-latest}
1111
chatqna-ui:
1212
build:
1313
context: docker/ui
1414
dockerfile: ./docker/Dockerfile
15-
image: ${REGISTRY}opea/chatqna-ui:${TAG:-latest}
15+
image: ${REGISTRY:-opea}/chatqna-ui:${TAG:-latest}
1616
chatqna-conversation-ui:
1717
build:
1818
context: docker/ui
1919
dockerfile: ./docker/Dockerfile.react
20-
image: ${REGISTRY}opea/chatqna-conversation-ui:${TAG:-latest}
20+
image: ${REGISTRY:-opea}/chatqna-conversation-ui:${TAG:-latest}

‎.github/workflows/docker/compose/CodeGen-compose.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ services:
77
build:
88
context: docker
99
dockerfile: ./Dockerfile
10-
image: ${REGISTRY}opea/codegen:${TAG:-latest}
10+
image: ${REGISTRY:-opea}/codegen:${TAG:-latest}
1111
codegen-ui:
1212
build:
1313
context: docker/ui
1414
dockerfile: ./docker/Dockerfile
15-
image: ${REGISTRY}opea/codegen-ui:${TAG:-latest}
15+
image: ${REGISTRY:-opea}/codegen-ui:${TAG:-latest}
1616
codegen-react-ui:
1717
build:
1818
context: docker/ui
1919
dockerfile: ./docker/Dockerfile.react
20-
image: ${REGISTRY}opea/codegen-conversation-ui:${TAG:-latest}
20+
image: ${REGISTRY:-opea}/codegen-conversation-ui:${TAG:-latest}

‎.github/workflows/docker/compose/CodeTrans-compose.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ services:
77
build:
88
context: docker
99
dockerfile: ./Dockerfile
10-
image: ${REGISTRY}opea/codetrans:${TAG:-latest}
10+
image: ${REGISTRY:-opea}/codetrans:${TAG:-latest}
1111
codetrans-ui:
1212
build:
1313
context: docker/ui
1414
dockerfile: ./docker/Dockerfile
15-
image: ${REGISTRY}opea/codetrans-ui:${TAG:-latest}
15+
image: ${REGISTRY:-opea}/codetrans-ui:${TAG:-latest}

‎.github/workflows/docker/compose/DocSum-compose.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ services:
77
build:
88
context: docker
99
dockerfile: ./Dockerfile
10-
image: ${REGISTRY}opea/docsum:${TAG:-latest}
10+
image: ${REGISTRY:-opea}/docsum:${TAG:-latest}
1111
docsum-ui:
1212
build:
1313
context: docker/ui
1414
dockerfile: ./docker/Dockerfile
15-
image: ${REGISTRY}opea/docsum-ui:${TAG:-latest}
15+
image: ${REGISTRY:-opea}/docsum-ui:${TAG:-latest}
1616
docsum-react-ui:
1717
build:
1818
context: docker/ui
1919
dockerfile: ./docker/Dockerfile.react
20-
image: ${REGISTRY}opea/docsum-react-ui:${TAG:-latest}
20+
image: ${REGISTRY:-opea}/docsum-react-ui:${TAG:-latest}

‎.github/workflows/docker/compose/FaqGen-compose.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ services:
77
build:
88
context: docker
99
dockerfile: ./Dockerfile
10-
image: ${REGISTRY}opea/faqgen:${TAG:-latest}
10+
image: ${REGISTRY:-opea}/faqgen:${TAG:-latest}
1111
faqgen-ui:
1212
build:
1313
context: docker/ui
1414
dockerfile: ./docker/Dockerfile
15-
image: ${REGISTRY}opea/faqgen-ui:${TAG:-latest}
15+
image: ${REGISTRY:-opea}/faqgen-ui:${TAG:-latest}
1616
faqgen-react-ui:
1717
build:
1818
context: docker/ui
1919
dockerfile: ./docker/Dockerfile.react
20-
image: ${REGISTRY}opea/faqgen-react-ui:${TAG:-latest}
20+
image: ${REGISTRY:-opea}/faqgen-react-ui:${TAG:-latest}

‎.github/workflows/docker/compose/SearchQnA-compose.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ services:
77
build:
88
context: docker
99
dockerfile: ./Dockerfile
10-
image: ${REGISTRY}opea/searchqna:${TAG:-latest}
10+
image: ${REGISTRY:-opea}/searchqna:${TAG:-latest}
1111
searchqna-ui:
1212
build:
1313
context: docker/ui
1414
dockerfile: ./docker/Dockerfile
15-
image: ${REGISTRY}opea/searchqna-ui:${TAG:-latest}
15+
image: ${REGISTRY:-opea}/searchqna-ui:${TAG:-latest}

‎.github/workflows/docker/compose/Translation-compose.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ services:
77
build:
88
context: docker
99
dockerfile: ./Dockerfile
10-
image: ${REGISTRY}opea/translation:${TAG:-latest}
10+
image: ${REGISTRY:-opea}/translation:${TAG:-latest}
1111
translation-ui:
1212
build:
1313
context: docker/ui
1414
dockerfile: ./docker/Dockerfile
15-
image: ${REGISTRY}opea/translation-ui:${TAG:-latest}
15+
image: ${REGISTRY:-opea}/translation-ui:${TAG:-latest}

‎.github/workflows/manual-cd-workflow.yml

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Examples CD workflow on manual event
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
examples:
9+
default: "AudioQnA,ChatQnA,CodeGen,CodeTrans,DocSum,FaqGen,SearchQnA,Translation"
10+
description: 'List of examples to test'
11+
required: true
12+
type: string
13+
tag:
14+
default: "latest"
15+
description: "Tag to apply to images"
16+
required: true
17+
type: string
18+
build:
19+
default: true
20+
description: 'Build test required images for Examples'
21+
required: false
22+
type: boolean
23+
scan:
24+
default: true
25+
description: 'Scan all images with Trivy'
26+
required: false
27+
type: boolean
28+
test_compose:
29+
default: true
30+
description: 'Test examples with docker compose'
31+
required: false
32+
type: boolean
33+
test_k8s:
34+
default: false
35+
description: 'Test examples with k8s'
36+
required: false
37+
type: boolean
38+
publish:
39+
default: false
40+
description: 'Publish images to docker hub'
41+
required: false
42+
type: boolean
43+
publish_tags:
44+
default: "latest,v0.9"
45+
description: 'Tag list apply to publish images'
46+
required: false
47+
type: string
48+
49+
permissions: read-all
50+
jobs:
51+
get-test-matrix:
52+
runs-on: ubuntu-latest
53+
outputs:
54+
matrix: ${{ steps.get-matrix.outputs.matrix }}
55+
steps:
56+
- name: Create Matrix
57+
id: get-matrix
58+
run: |
59+
examples=($(echo ${{ github.event.inputs.examples }} | tr ',' ' '))
60+
examples_json=$(printf '%s\n' "${examples[@]}" | sort -u | jq -R '.' | jq -sc '.')
61+
echo "matrix=$examples_json" >> $GITHUB_OUTPUT
62+
63+
run-examples:
64+
needs: [get-test-matrix]
65+
strategy:
66+
matrix:
67+
example: ${{ fromJson(needs.get-test-matrix.outputs.matrix) }}
68+
fail-fast: false
69+
uses: ./.github/workflows/_example-workflow.yml
70+
with:
71+
example: ${{ matrix.example }}
72+
tag: ${{ inputs.tag }}
73+
build: ${{ fromJSON(inputs.build) }}
74+
scan: ${{ fromJSON(inputs.scan) }}
75+
test_compose: ${{ fromJSON(inputs.test_compose) }}
76+
test_k8s: ${{ fromJSON(inputs.test_k8s) }}
77+
publish: ${{ fromJSON(inputs.publish) }}
78+
publish_tags: ${{ fromJSON(inputs.publish_tags) }}
79+
secrets: inherit

0 commit comments

Comments
 (0)