Refactor conda env installation #4679
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Test on pull request | |
on: | |
push: | |
branches: [ master ] | |
pull_request: | |
workflow_dispatch: | |
permissions: | |
checks: write | |
jobs: | |
python-tests: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
python-version: [ '3.8', '3.9', '3.10', '3.11' ] | |
timeout-minutes: 35 | |
steps: | |
- name: "Checkout" | |
uses: actions/checkout@v3 | |
- name: "Set up Python ${{ matrix.python-version }}" | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: Install dependencies | |
run: python -m pip install tox tox-gh-actions | |
- name: Build protobuf | |
run: cd pylzy && ./scripts/gen_proto.sh && cd .. | |
- name: Test | |
run: cd pylzy && tox -- -vv && cd .. | |
- name: 'upload coverage badge' | |
uses: exuanbo/actions-deploy-gist@v1 | |
if: github.ref == 'refs/heads/master' && matrix.python-version == '3.9' | |
with: | |
token: ${{ secrets.GIST_SECRET }} | |
gist_id: 0095e900fb0fcbe5575ddc3c717fb65b | |
file_path: pylzy/coverage.svg | |
file_type: binary | |
gist_file_name: ${{ (github.head_ref || github.ref_name) }}-coverage.svg | |
java-unit-tests: | |
runs-on: ubuntu-latest | |
timeout-minutes: 35 | |
defaults: | |
run: | |
shell: bash -l {0} | |
steps: | |
- name: 'Checkout' | |
uses: actions/checkout@v3 | |
- name: 'Setup JDK 17' | |
uses: actions/setup-java@v3 | |
with: | |
java-version: '17' | |
distribution: 'temurin' | |
- name: 'Setup fuse' | |
run: | | |
sudo apt-get update && sudo apt-get install fuse | |
echo "user_allow_other" | sudo tee -a /etc/fuse.conf | |
- name: Tests | |
run: mvn clean install | |
- name: 'generate java coverage badge' | |
uses: cicirello/jacoco-badge-generator@v2 | |
with: | |
generate-branches-badge: true | |
jacoco-csv-file: coverage/target/site/jacoco-aggregate/jacoco.csv | |
badges-directory: coverage/target/site/jacoco-aggregate | |
generate-coverage-badge: true | |
coverage-badge-filename: jacoco.svg | |
coverage-label: java-coverage | |
- name: 'upload java coverage badge' | |
uses: exuanbo/actions-deploy-gist@v1 | |
if: github.ref == 'refs/heads/master' | |
with: | |
token: ${{ secrets.GIST_SECRET }} | |
gist_id: be651155cb12a8006cecdee948ce1a0a | |
file_path: coverage/target/site/jacoco-aggregate/jacoco.svg | |
file_type: binary | |
gist_file_name: ${{ (github.head_ref || github.ref_name) }}-java-coverage.svg | |
- name: 'Clear after tests' | |
if: always() | |
run: | | |
SHA=${{ github.sha }} | |
sudo mkdir -p /tmp/old-lzy-logs/$SHA | |
sudo tar -czvf /tmp/old-lzy-logs/$SHA/logs.tar.gz /tmp/lzy-log | |
sudo rm -rf /tmp/lzy* /tmp/h2* /tmp/local_modules* /tmp/resources* /tmp/term* /tmp/worker* /tmp/tmp* | |
sudo rm -rf /tmp/temp* /tmp/test* /tmp/*json /tmp/*txt /tmp/*so /tmp/*lck /tmp/*sh /tmp/gen-rsa-keys* | |
sudo rm -rf /tmp/kafka-logs* /tmp/zookeeper-logs* /tmp/s3mockFileStore* /tmp/Log4j2Plugins* | |
- name: 'Publish Test Report' | |
uses: EnricoMi/publish-unit-test-result-action@v1 | |
if: github.ref == 'refs/heads/master' | |
with: | |
files: "**/surefire-reports/TEST-*.xml" | |
integration-tests: | |
runs-on: self-hosted | |
timeout-minutes: 60 | |
defaults: | |
run: | |
shell: bash -l {0} | |
steps: | |
- name: "Chown for checkout" | |
run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/lzy/ | |
- name: "Checkout" | |
uses: actions/checkout@v3 | |
- name: 'Clear before tests' | |
run: | | |
sudo rm -rf /tmp/lzy* /tmp/h2* /tmp/local_modules* /tmp/resources* /tmp/term* /tmp/worker* | |
sudo rm -rf /tmp/temp* /tmp/test* /tmp/*json /tmp/*txt /tmp/*so /tmp/*lck /tmp/*sh /tmp/gen-rsa-keys* | |
sudo rm -rf /tmp/kafka-logs* /tmp/zookeeper-logs* /tmp/s3mockFileStore* /tmp/Log4j2Plugins* | |
rm -rf pylzy/ai | |
- name: 'install python test env' | |
run: | | |
# conda activate py39 || conda create --name py39 -y "python=3.9.7" pip && python -m ensurepip | |
eval "$(conda shell.bash hook)" && conda activate py39 && pip install -r lzy/worker/docker/requirements.txt | |
- name: 'install pylzy' | |
run: | | |
cd pylzy/ | |
eval "$(conda shell.bash hook)" && pip install -r proto_requirements.txt -I --no-cache && ./scripts/build.sh | |
eval "$(conda shell.bash hook)" && conda activate py39 && pip install . --force-reinstall --upgrade --no-cache | |
cd .. | |
- name: 'java compile' | |
timeout-minutes: 10 | |
run: mvn -T 8 clean install -DskipTests -Dmaven.test.skip -Dmaven.javadoc.skip=true | |
- name: 'SchedulerTest' | |
timeout-minutes: 5 | |
run: | | |
cd test && mvn test -Dtest=SchedulerTest -DargLine="-Djava.util.concurrent.ForkJoinPool.common.parallelism=64" && cd .. | |
- name: 'StorageClientTest' | |
timeout-minutes: 5 | |
run: | | |
cd test && mvn test -Dtest=StorageClientTest -DargLine="-Djava.util.concurrent.ForkJoinPool.common.parallelism=64" && cd .. | |
- name: 'WorkflowTest' | |
timeout-minutes: 5 | |
run: | | |
cd test && mvn test -Dtest=WorkflowTest -DargLine="-Djava.util.concurrent.ForkJoinPool.common.parallelism=64" && cd .. | |
# - name: 'UserImagePyTest' | |
# timeout-minutes: 10 | |
# run: | | |
# BRANCH=${{ (github.head_ref || github.ref_name) }} | |
# USER_IMAGE=$(deployment/latest-docker-image-on-branches.sh user-test $BRANCH master) | |
# cd test && mvn test -Dtest=UserImagePyTest -DargLine="-Djava.util.concurrent.ForkJoinPool.common.parallelism=64" -Dscheduler.userTestImage="$USER_IMAGE" && cd .. | |
- name: 'PyApiTest' | |
timeout-minutes: 35 | |
run: | | |
eval "$(conda shell.bash hook)" && pip install -r pylzy/proto_requirements.txt -I --no-cache | |
cd test && mvn test -Dtest=PyApiTest -DargLine="-Djava.util.concurrent.ForkJoinPool.common.parallelism=64" && cd .. | |
- name: 'Clean docker files' | |
run: | | |
docker stop $(docker ps -a -q) && docker rm $(docker ps -a -q) | |
docker image rm -f $(docker image ls 6a7b4d -a -q) | |
docker image prune -f | |
docker volume prune -f | |
- name: 'Clear after tests' | |
if: always() | |
run: | | |
SHA=${{ github.sha }} | |
sudo mkdir -p /tmp/old-lzy-logs/$SHA | |
sudo tar -czvf /tmp/old-lzy-logs/$SHA/logs.tar.gz /tmp/lzy-log | |
sudo rm -rf /tmp/lzy* /tmp/h2* /tmp/local_modules* /tmp/resources* /tmp/term* /tmp/worker* /tmp/tmp* | |
sudo rm -rf /tmp/temp* /tmp/test* /tmp/*json /tmp/*txt /tmp/*so /tmp/*lck /tmp/*sh | |
- name: 'clean pylzy' | |
if: always() | |
run: | | |
cd pylzy/ | |
eval "$(conda shell.bash hook)" && ./scripts/clean.sh | |
cd .. | |
- name: 'Publish Test Report' | |
uses: mikepenz/action-junit-report@v4 | |
if: always() | |
with: | |
report_paths: "**/surefire-reports/TEST-*.xml" |