-
Notifications
You must be signed in to change notification settings - Fork 58
112 lines (112 loc) · 4.73 KB
/
integration-test-multiple-sizes.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
name: integration-test-multiple-sizes
on: [push]
concurrency: transfer-test
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
SKYPLANE_USAGE_STATS_ENABLED: 0
jobs:
integration:
runs-on: ubuntu-latest
strategy:
max-parallel: 4
matrix:
pairs:
# AWS to AWS
- aws:us-east-1 aws:us-east-1 --multipart
- aws:us-east-2 aws:us-west-2 --multipart
# GCP to GCP
- gcp:us-central1-a gcp:us-central1-a --multipart
- gcp:us-west1-a gcp:us-east1-a --multipart
# Azure to Azure
- azure:westus azure:westus --multipart
- azure:eastus azure:westus --multipart
# cross cloud tests
- aws:us-west-1 gcp:us-west2-a --multipart
- gcp:us-west2-a aws:us-west-1 --multipart
- aws:us-west-1 azure:westus --multipart
- azure:westus aws:us-west-1 --multipart
- gcp:us-west2-a azure:westus --multipart
- azure:westus gcp:us-west2-a --multipart
timeout-minutes: 40
env:
STRATEGY_UUID: itest-${{ github.run_id }}-${{ github.run_attempt }}-${{ strategy.job-index }}
steps:
- uses: actions/checkout@v1
- name: Install poetry
run: pipx install poetry
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "poetry"
- name: Set Poetry config
run: |
poetry config virtualenvs.in-project false
poetry config virtualenvs.path ~/.virtualenvs
- name: Install Dependencies
run: poetry install -E aws -E azure -E gcp
if: steps.cache.outputs.cache-hit != 'true'
- id: 'auth'
uses: 'google-github-actions/auth@v1'
with:
credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}'
- name: Log into Azure
uses: azure/login@v1
with:
creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}'
- name: Skyplane init
run: |
poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
poetry run skyplane config set native_cmd_enabled false
cat ~/.skyplane/config
poetry run skyplane init -y
poetry run skyplane config set usage_stats false
- name: Single small file test
run: poetry run python tests/integration/cp.py ${{ matrix.pairs }} --n-files 1 --file-size-mb 32
- name: 128 small files test
run: poetry run python tests/integration/cp.py ${{ matrix.pairs }} --n-files 128 --file-size-mb 1
- name: Single large file test
run: poetry run python tests/integration/cp.py ${{ matrix.pairs }} --n-files 1 --file-size-mb 2000
- name: Cleanup GCP service account
if: always()
run: gcloud iam service-accounts delete ${{ env.STRATEGY_UUID }}@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com
deprovision:
runs-on: ubuntu-latest
if: ${{ always() }}
needs: [integration]
env:
STRATEGY_UUID: itest-d-${{ github.run_id }}-${{ github.run_attempt }}
steps:
- uses: actions/checkout@v1
- name: Install poetry
run: pipx install poetry
- name: Set up Python 3.10
uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: "poetry"
- name: Install dependencies
run: poetry install -E aws -E azure -E gcp
if: steps.cache.outputs.cache-hit != 'true'
- id: 'auth'
uses: 'google-github-actions/auth@v1'
with:
credentials_json: '${{ secrets.GCP_CREDENTIALS_JSON }}'
- name: Skyplane init
run: |
poetry run skyplane config set gcp_service_account_name ${{ env.STRATEGY_UUID }}
poetry run skyplane init -y --disable-config-azure
poetry run skyplane config set usage_stats false
- name: Deprovision
run: poetry run skyplane deprovision
- name: Delete matching S3 buckets
run: |
for pattern in "test-skyplane-" "skyplane-integration-" "integrationus-east-1-"; do
aws s3api list-buckets --query "Buckets[?starts_with(Name, \`${pattern}\`) == \`true\`].Name" --output text | tr '\t' '\n' | while read bucket; do aws s3 rb "s3://$bucket" --force; done
done
- name: Cleanup GCP service account
if: always()
run: gcloud iam service-accounts delete --quiet ${{ env.STRATEGY_UUID }}@${{ secrets.GCP_PROJECT_ID }}.iam.gserviceaccount.com