diopi gen data #288
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: diopi gen data | |
| on: | |
| workflow_dispatch: | |
| # push: | |
| # branches: | |
| # - "wgs/*" | |
| schedule: | |
| - cron: '01 22 * * *' | |
| concurrency: | |
| group: data-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CLUSTER_V100: SCO | |
| CLUSTER_CAMB: CAMB | |
| CLUSTER_ASCEND_910B: ASCEND-910B | |
| CLUSTER_TOPSRIDER: TOPSRIDER | |
| CLUSTER_SUPA: SUPA | |
| DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}' | |
| HASH_FILE: 'diopi/diopi_case_items.cfg' | |
| HASH_FILE2: 'diopi_test/python/conformance/customized_test.py' | |
| DATA_DIR: '/mnt/cache/share/deeplinkci/gen-data' | |
| ENV_PATH: '/mnt/cache/share/platform/env' | |
| GEN_DATA: 'diopi_test/python/cache' | |
| ENV_NAME: 'pt2.0_diopi' | |
| jobs: | |
| Clone: | |
| name: checkout code | |
| runs-on: tps-diopi-ci | |
| steps: | |
| - name: clone repo | |
| run: | | |
| set -e | |
| cd ${GITHUB_WORKSPACE} && rm -rf source | |
| git clone ${{ github.event.repository.clone_url }} source | |
| cd source && git submodule update --init --recursive | |
| - name: Rsync to Server | |
| run: | | |
| ssh ${CLUSTER_V100} "mkdir -p ${DATA_DIR} && cd ${DATA_DIR} && rm -rf source && mkdir source" \ | |
| && rsync -a --delete ${GITHUB_WORKSPACE}/source/ ${CLUSTER_V100}:${DATA_DIR}/source/ | |
| Gen-Data: | |
| name: Gen-Data | |
| runs-on: tps-sco-ci | |
| needs: [Clone] | |
| steps: | |
| - name: gen-test-data | |
| run: | | |
| set -e | |
| srun bash -c "cd ${DATA_DIR}/source/diopi_test/python \ | |
| && source ${ENV_PATH}/pt2.0_diopi \ | |
| && python main.py --mode gen_data \ | |
| || ( cd ${DATA_DIR}/source/diopi_test/python && git clean -xdf ${GEN_DATA} && exit 1 ) " | |
| CheckAndRsync: | |
| name: check and rsync | |
| runs-on: tps-diopi-ci | |
| needs: Gen-Data | |
| outputs: | |
| output: ${{ steps.get_gen_data.outputs.to_gen_data }} | |
| steps: | |
| - name: check | |
| id: get_gen_data | |
| run: | | |
| ssh ${CLUSTER_V100} """ | |
| set -e | |
| cd ${DATA_DIR} | |
| if [[ -f "pre_hash" ]] && [[ -f "pre_hash2" ]]; then | |
| sha1sum source/${GEN_DATA}/${HASH_FILE} > new_hash | |
| diff_output=\$(diff -q --ignore-all-space "new_hash" "pre_hash" ) && echo "false" > to_gen_data || echo "true" > to_gen_data | |
| sha1sum source/${HASH_FILE2} > new_hash2 | |
| diff_output=\$(diff -q --ignore-all-space "new_hash2" "pre_hash2" ) || echo "true" > to_gen_data | |
| else | |
| echo "true" > to_gen_data | |
| fi | |
| cp new_hash pre_hash | |
| cp new_hash2 pre_hash2 | |
| """ | |
| export file_path="/home/autolink/rsync/sourcecode" | |
| rsync -a --delete ${CLUSTER_V100}:${DATA_DIR}/to_gen_data ${file_path}/ | |
| if grep -q "true" "${file_path}/to_gen_data"; then | |
| echo "to_gen_data=true" >> "$GITHUB_ENV" | |
| else | |
| echo "to_gen_data=false" >> "$GITHUB_ENV" | |
| fi | |
| - name: rsync | |
| if: ${{ env.to_gen_data == 'true' }} | |
| run: | | |
| set -e | |
| ssh ${CLUSTER_CAMB} """ | |
| mkdir -p ${DATA_DIR}/source/${GEN_DATA} | |
| rsync -a --delete ${CLUSTER_V100}:${DATA_DIR}/source/${GEN_DATA}/diopi/ ${DATA_DIR}/source/${GEN_DATA}/diopi/ | |
| """ | |
| # ssh ${CLUSTER_ASCEND_910B} """ | |
| # mkdir -p ${DATA_DIR}/source/${GEN_DATA} | |
| # rsync -a --delete ${CLUSTER_V100}:${DATA_DIR}/source/${GEN_DATA}/diopi/ ${DATA_DIR}/source/${GEN_DATA}/diopi/ | |
| # """ | |
| source ~/Aoss_env.sh | |
| ads-cli --dryrun --deleteSrc cp s3://${Platform_ci_aoss_name}:${Platform_ci_aoss_url}@platform.aoss.cn-sh-01c.sensecoreapi-oss.cn${DATA_DIR}/source/${GEN_DATA}/diopi/ /dev/null 2>&1 >/dev/null | |
| ads-cli cp ${DATA_DIR}/source/${GEN_DATA}/diopi/ s3://${Platform_ci_aoss_name}:${Platform_ci_aoss_url}@platform.aoss.cn-sh-01c.sensecoreapi-oss.cn${DATA_DIR}/source/${GEN_DATA}/diopi/ | |
| # ssh ${CLUSTER_SUPA} """ | |
| # rsync -a ${CLUSTER_V100}:${DATA_DIR}/source/${GEN_DATA}/diopi/ ${DATA_DIR}/source/${GEN_DATA}/diopi/ | |
| # """ | |
| Copy-Gen-Data-Ascend-910b: | |
| name: Copy-Gen-Data-Ascend-910b | |
| runs-on: tps-ascend-ci-910b | |
| needs: CheckAndRsync | |
| if: needs.CheckAndRsync.outputs.to_gen_data == 'true' | |
| steps: | |
| - name: Copy Gen-Data | |
| run: | | |
| set -e | |
| source ~/Aoss_env.sh | |
| ads-cli cp s3://${Platform_ci_aoss_name}:${Platform_ci_aoss_url}@platform.aoss-internal.cn-sh-01c.sensecoreapi-oss.cn${DATA_DIR}/source/${GEN_DATA}/diopi/ ${DATA_DIR}/source/${GEN_DATA}/diopi/ |