-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add parallel pipeline for multiple distributions
- Loading branch information
SergeRadinovich
committed
Nov 15, 2019
1 parent
8188a48
commit 03d061d
Showing
8 changed files
with
240 additions
and
55 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| #!/bin/sh | ||
|
|
||
| set -e | ||
|
|
||
| detect_os(){ | ||
| OS=$(uname) | ||
| if [ ! "$OS" = "Linux" ]; then | ||
| echo "Operating System $OS is not supported" | ||
| exit 1 | ||
| fi | ||
| if [ -f /etc/os-release ]; then | ||
| . /etc/os-release | ||
| DIST=$NAME | ||
| VER=$VERSION_ID | ||
| elif type lsb_release >/dev/null 2>&1; then | ||
| DIST=$(lsb_release -si) | ||
| VER=$(lsb_release -sr) | ||
| elif [ -f /etc/lsb-release ]; then | ||
| # For some versions of Debian/Ubuntu without lsb_release command | ||
| . /etc/lsb-release | ||
| DIST=$DISTRIB_ID | ||
| VER=$DISTRIB_RELEASE | ||
| elif [ -f /etc/debian_version ]; then | ||
| # Older Debian/Ubuntu/etc. | ||
| DIST=Debian | ||
| VER=$(cat /etc/debian_version) | ||
| elif [ -f /etc/SuSe-release ]; then | ||
| # Older SuSE/etc. | ||
| ... | ||
| elif [ -f /etc/redhat-release ]; then | ||
| # Older Red Hat, CentOS, etc. | ||
| ... | ||
| else | ||
| # Fall back to uname, e.g. "Linux <version>", also works for BSD, etc. | ||
| DIST=$(uname -s) | ||
| VER=$(uname -r) | ||
| fi | ||
| DIST=$(echo "$DIST" | sed "y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/") | ||
| } | ||
|
|
||
| install_docker(){ | ||
| if [ -z "$(command -v docker)" ]; then | ||
| curl -fsSL https://get.docker.com/ | sh | ||
| fi | ||
| sudo usermod -aG docker $USER | ||
| if [ -z "$(command -v docker)" ]; then | ||
| echo "Failed to install Docker" | ||
| echo "Visit https://docs.docker.com/install/ for instructions on manual installation of Docker" | ||
| fi | ||
| } | ||
|
|
||
| install_iofogctl(){ | ||
| case "$DIST" in | ||
| *ubuntu*|*debian*|*raspbian*) | ||
| curl https://packagecloud.io/install/repositories/iofog/iofogctl/script.deb.sh | sudo bash | ||
| sudo apt-get install iofogctl=1.3.0-rc2 | ||
| ;; | ||
| *fedora*|*centos*) | ||
| curl https://packagecloud.io/install/repositories/iofog/iofogctl/script.rpm.sh | sudo bash | ||
| sudo yum install iofogctl-1.3.0-rc2-1.x86_64 | ||
| ;; | ||
| *) | ||
| echo "Failed to install iofogctl" | ||
| echo "Linux distribution $DIST is not supported" | ||
| exit 1 | ||
| ;; | ||
| esac | ||
| } | ||
|
|
||
| detect_os | ||
| install_docker | ||
| install_iofogctl |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| parameters: | ||
| name: '' | ||
| distro: '' | ||
|
|
||
| jobs: | ||
| - job: ${{ parameters.name }} | ||
| pool: | ||
| vmImage: 'ubuntu-16.04' | ||
|
|
||
| steps: | ||
|
|
||
| - task: InstallSSHKey@0 | ||
| inputs: | ||
| knownHostsEntry: $(ssh.knownhost) | ||
| sshPublicKey: $(ssh.pub) | ||
| sshKeySecureFile: id_rsa | ||
|
|
||
| - task: DownloadSecureFile@1 | ||
| displayName: 'Download SSH keys to' | ||
| inputs: | ||
| secureFile: 'id_rsa' | ||
|
|
||
| - script: | | ||
| cat $(Agent.TempDirectory)/id_rsa > $(key) | ||
| chmod 600 $(key) | ||
| displayName: 'Prepare SSH key' | ||
| - template: vm-up.yaml | ||
| parameters: | ||
| distro: ${{ parameters.distro }} | ||
|
|
||
| - script: | | ||
| echo "key: $(key)" | ||
| echo "agent: $(agent)" | ||
| echo "dir: $(dir)" | ||
| $(cssh) $(agent) -- sudo apt -y install rsync | ||
| rsync -Pavr -e "ssh -i $(key) -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" . $(agent):$(dir) | ||
| $(cssh) $(agent) -- ls $(dir) | ||
| displayName: 'Prepare VM' | ||
| - script: | | ||
| echo "agent: $(agent)" | ||
| echo "dir: $(dir)" | ||
| $(cssh) $(agent) -- sh $(dir)/bootstrap.sh | ||
| $(cssh) $(agent) -- docker ps | ||
| $(cssh) $(agent) -- iofogctl version | ||
| displayName: 'Bootstrap' | ||
| - script: | | ||
| echo $(gcp.svcacc) > /tmp/svcacc.json | ||
| rsync -Pavr -e "ssh -i $(key) -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" /tmp/svcacc.json $(agent):/tmp/ | ||
| $(cssh) $(agent) -- cat /tmp/svcacc.json | docker login -u _json_key --password-stdin https://gcr.io | ||
| displayName: 'Docker connect to Registry' | ||
| # - script: | | ||
| # docker pull gcr.io/focal-freedom-236620/controller:develop | ||
| # docker pull gcr.io/focal-freedom-236620/connector:develop | ||
| # docker pull gcr.io/focal-freedom-236620/agent:develop | ||
| # displayName: 'Pull latest images' | ||
|
|
||
| - script: | | ||
| $(cssh) $(agent) -- bash $(dir)/start.sh tutorial | ||
| # Use dev version of agent and controller until 1.3.0 release | ||
| # ./start.sh -ct gcr.io/focal-freedom-236620/controller:develop -a gcr.io/focal-freedom-236620/agent:develop -cn gcr.io/focal-freedom-236620/connector:develop | ||
| displayName: 'Start Connector, Controller, Agent, and tutorial microservices' | ||
| - script: | | ||
| $(cssh) $(agent) -- bash $(dir)/test.sh | ||
| displayName: 'Run Tests' | ||
| - script: | | ||
| $(cssh) $(agent) -- bash $(dir)/stop.sh | ||
| displayName: 'Stop Connector, Controller, and Agent' | ||
| - script: | | ||
| echo "===== IOFOG AGENT LOG =====" | ||
| docker exec iofog-agent cat /var/log/agent.out.log | ||
| displayName: 'Print logs' | ||
| condition: failed() | ||
| - template: vm-down.yaml |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| steps: | ||
| - script: | | ||
| gcloud compute --project=$(gcp.project.name) instances delete demo-ci-$(jobuuid) --zone=$(gcp.vm.zone) --delete-disks=all -q | ||
| displayName: 'Teardown VMs' | ||
| condition: always() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| parameters: | ||
| distro: '' | ||
|
|
||
| steps: | ||
| - task: DownloadSecureFile@1 | ||
| displayName: 'Download secure file' | ||
| inputs: | ||
| secureFile: 'azure-gcp.json' | ||
| - bash: | | ||
| echo "gcp.project.name: $(gcp.project.name)" | ||
| CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" | ||
| echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list | ||
| curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - | ||
| sudo apt-get update && sudo apt-get install google-cloud-sdk | ||
| gcloud --quiet auth activate-service-account --key-file=$(Agent.TempDirectory)/azure-gcp.json | ||
| gcloud --quiet config set project $(gcp.project.name) | ||
| displayName: 'set up gcloud' | ||
| - script: | | ||
| distro=${{ parameters.distro }} | ||
| repo=$([[ $distro == *"debian"* ]] && echo "debian-cloud" || echo "ubuntu-os-cloud") | ||
| echo "distro: $distro" | ||
| echo "repo: $repo" | ||
| echo "gcp.project.name: $(gcp.project.name)" | ||
| echo "gcp.vm.zone: $(gcp.vm.zone)" | ||
| echo "gcp.svcacc.name: $(gcp.svcacc.name)" | ||
| echo "jobuuid: $(jobuuid)" | ||
| gcloud compute --project=$(gcp.project.name) instances create demo-ci-$(jobuuid) --zone=$(gcp.vm.zone) --machine-type=n1-standard-1 --subnet=default --network-tier=PREMIUM --maintenance-policy=MIGRATE --service-account=$(gcp.svcacc.name) --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append --image=$distro --image-project=$repo --boot-disk-size=200GB --boot-disk-type=pd-standard --boot-disk-device-name=demo-ci-$(jobuuid) | ||
| vm_host=$(gcloud compute instances list | grep demo-ci-$(jobuuid) | awk '{print $5}') | ||
| echo "vm_host: $vm_host" | ||
| echo "##vso[task.setvariable variable=agent]$(gcp.vm.user)@$vm_host" | ||
| displayName: 'Deploy Test VM' | ||
| - script: | | ||
| echo "agent: $(agent)" | ||
| seconds=0 | ||
| until $(cssh) $(agent) -- echo "SSH success"; do | ||
| if [ $seconds -gt 60 ]; then | ||
| echo "Timed out waiting for $(agent)" | ||
| exit 1 | ||
| fi | ||
| seconds=$((seconds+1)) | ||
| sleep 1 | ||
| done | ||
| displayName: 'Wait for SSH access' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters