Hotfix/pipeline

azure-pipelines.yml

@@ -13,52 +13,35 @@ schedules:
     - master
     - develop
 
-pool:
-  vmImage: 'ubuntu-16.04'
-
-steps:
-
-- task: DockerInstaller@0
-  displayName: Docker Installer
-  inputs:
-    dockerVersion: 19.03.2
-    releaseType: stable
-
-- script: |
-    # curl -s https://packagecloud.io/install/repositories/iofog/iofogctl/script.deb.sh | sudo bash
-    # Using dev version of iofogctl until 1.3.0 release
-    curl -s https://8c90601638aff0b3fb520971175089bbaba2cf7f29be9528:@packagecloud.io/install/repositories/iofog/iofogctl-snapshots/script.deb.sh | sudo bash
-    sudo apt install iofogctl=1.3.0-dev
-  displayName: 'Install iofogctl'
-
-- script: |
-    echo $(gcp.svcacc) | docker login -u _json_key --password-stdin https://gcr.io
-  displayName: 'Docker connect to Registry'
-
-- script: |
-    ./start.sh tutorial
-    # Use dev version of agent and controller until 1.3.0 release
-    # ./start.sh -ct gcr.io/focal-freedom-236620/controller:develop -a gcr.io/focal-freedom-236620/agent:develop -cn gcr.io/focal-freedom-236620/connector:develop
-  displayName: 'Start Connector, Controller, Agent, and tutorial microservices'
-
-- script: |
-    ./test.sh
-  displayName: 'Run Tests'
-
-- script: |
-    ./stop.sh
-  displayName: 'Stop Connector, Controller, and Agent'
-
-- script: |
-    tar -c --transform 's,^\.,demo,' --exclude-from=.artifactignore -v --bzip2 -f $BUILD_ARTIFACTSTAGINGDIRECTORY/demo.tar.bz2 .
-
-- task: PublishBuildArtifacts@1
-  inputs:
-    pathtoPublish: '$(Build.ArtifactStagingDirectory)/demo.tar.bz2'
-    artifactName: 'demo.$(Build.BuildId).tar.bz2'
-
-- script: |
-    echo "===== IOFOG AGENT LOG ====="
-    docker exec iofog-agent cat /var/log/agent.out.log
-  displayName: 'Print logs'
-  condition: failed()
+variables:
+  jobuuid: $(Build.BuildId)$(Agent.Id)
+  agent: ''
+  key: '/tmp/id_rsa'
+  cssh: 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i $(key)'
+  dir: '/tmp/demo'
+
+jobs:
+
+  - template: templates/job.yaml
+    parameters:
+      name: bionic
+      distro: $(gcp.vm.distro.bionic)
+      repo: $(gcp.vm.repo.ubuntu)
+
+  - template: templates/job.yaml
+    parameters:
+      name: xenial
+      distro: $(gcp.vm.distro.xenial)
+      repo: $(gcp.vm.repo.ubuntu)
+
+  - template: templates/job.yaml
+    parameters:
+      name: buster
+      distro: $(gcp.vm.distro.buster)
+      repo: $(gcp.vm.repo.debian)
+
+  - template: templates/job.yaml
+    parameters:
+      name: stretch
+      distro: $(gcp.vm.distro.stretch)
+      repo: $(gcp.vm.repo.debian)

bootstrap.sh

@@ -0,0 +1,72 @@
+#!/bin/sh
+
+set -e
+
+detect_os(){
+  OS=$(uname)
+  if [ ! "$OS" = "Linux" ]; then
+    echo "Operating System $OS is not supported"
+    exit 1
+  fi
+  if [ -f /etc/os-release ]; then
+      . /etc/os-release
+      DIST=$NAME
+      VER=$VERSION_ID
+  elif type lsb_release >/dev/null 2>&1; then
+      DIST=$(lsb_release -si)
+      VER=$(lsb_release -sr)
+  elif [ -f /etc/lsb-release ]; then
+      # For some versions of Debian/Ubuntu without lsb_release command
+      . /etc/lsb-release
+      DIST=$DISTRIB_ID
+      VER=$DISTRIB_RELEASE
+  elif [ -f /etc/debian_version ]; then
+      # Older Debian/Ubuntu/etc.
+      DIST=Debian
+      VER=$(cat /etc/debian_version)
+  elif [ -f /etc/SuSe-release ]; then
+      # Older SuSE/etc.
+      ...
+  elif [ -f /etc/redhat-release ]; then
+      # Older Red Hat, CentOS, etc.
+      ...
+  else
+      # Fall back to uname, e.g. "Linux <version>", also works for BSD, etc.
+      DIST=$(uname -s)
+      VER=$(uname -r)
+  fi
+  DIST=$(echo "$DIST" | sed "y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/")
+}
+
+install_docker(){
+    if [ -z "$(command -v docker)" ]; then
+        curl -fsSL https://get.docker.com/ | sh
+    fi
+    sudo usermod -aG docker $USER
+    if [ -z "$(command -v docker)" ]; then
+        echo "Failed to install Docker"
+        echo "Visit https://docs.docker.com/install/ for instructions on manual installation of Docker"
+    fi
+}
+
+install_iofogctl(){
+    case "$DIST" in
+        *ubuntu*|*debian*|*raspbian*)
+            curl https://packagecloud.io/install/repositories/iofog/iofogctl/script.deb.sh | sudo bash
+            sudo apt-get install iofogctl=1.3.0-rc2
+            ;;
+        *fedora*|*centos*)
+            curl https://packagecloud.io/install/repositories/iofog/iofogctl/script.rpm.sh | sudo bash
+            sudo yum install iofogctl-1.3.0-rc2-1.x86_64
+            ;;
+        *)
+            echo "Failed to install iofogctl"
+            echo "Linux distribution $DIST is not supported"
+            exit 1
+            ;;
+    esac
+}
+
+detect_os
+install_docker
+install_iofogctl

start.sh

@@ -72,7 +72,7 @@ spec:
 " >| init/iofog/local-stack.yaml
 
     echoInfo "Deploying containers for ioFog stack..."
-    iofogctl deploy -f init/iofog/local-stack.yaml
+    iofogctl deploy -f init/iofog/local-stack.yaml -v
 }
 
 checkProvisioning() {
@@ -88,7 +88,7 @@ startEnvironment() {
     local ENVIRONMENT="$1"
 
     echoInfo "Deploying ${ENVIRONMENT} application..."
-    iofogctl deploy application -f "init/${ENVIRONMENT}/config.yaml"
+    iofogctl deploy application -f "init/${ENVIRONMENT}/config.yaml" -v
     echoInfo "It may take a while before ioFog stack creates all ${ENVIRONMENT} microservices."
     echo ""
 }

stop.sh

@@ -39,7 +39,7 @@ prettyHeader "Stopping ioFog Demo..."
 # Stop ioFog stack
 echoInfo "Stopping all containers..."
 
-iofogctl delete all || iofogctl disconnect
+iofogctl delete all -v || iofogctl disconnect
 
 # Remove generated files
 find test/conf -type f -not -name ".gitignore" -exec rm -f {} \;

templates/job.yaml

@@ -0,0 +1,83 @@
+parameters:
+  name: ''
+  distro: ''
+  repo: ''
+
+jobs:
+  - job: ${{ parameters.name }}
+    pool:
+      vmImage: 'ubuntu-16.04'
+
+    steps:
+
+    - task: InstallSSHKey@0
+      inputs:
+        knownHostsEntry: $(ssh.knownhost)
+        sshPublicKey: $(ssh.pub)
+        sshKeySecureFile: id_rsa
+
+    - task: DownloadSecureFile@1
+      displayName: 'Download SSH keys to'
+      inputs:
+        secureFile: 'id_rsa'
+
+    - script: |
+        cat $(Agent.TempDirectory)/id_rsa > $(key)
+        chmod 600 $(key)
+      displayName: 'Prepare SSH key'
+
+    - template: vm-up.yaml
+      parameters:
+        distro: ${{ parameters.distro }}
+        repo: ${{ parameters.repo }}
+
+    - script: |
+        echo "key: $(key)"
+        echo "agent: $(agent)"
+        echo "dir: $(dir)"
+        $(cssh) $(agent) -- sudo apt -y install rsync
+        rsync -Pavr -e "ssh -i $(key) -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" . $(agent):$(dir)
+        $(cssh) $(agent) -- ls $(dir)
+      displayName: 'Prepare VM'
+
+    - script: |
+        echo "agent: $(agent)"
+        echo "dir: $(dir)"
+        $(cssh) $(agent) -- sh $(dir)/bootstrap.sh
+        $(cssh) $(agent) -- docker ps
+        $(cssh) $(agent) -- iofogctl version
+      displayName: 'Bootstrap'
+
+    - script: |
+        echo $(gcp.svcacc) > /tmp/svcacc.json
+        rsync -Pavr -e "ssh -i $(key) -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" /tmp/svcacc.json $(agent):/tmp/
+        $(cssh) $(agent) -- cat /tmp/svcacc.json | docker login -u _json_key --password-stdin https://gcr.io
+      displayName: 'Docker connect to Registry'
+
+    # - script: |
+    #     docker pull gcr.io/focal-freedom-236620/controller:develop
+    #     docker pull gcr.io/focal-freedom-236620/connector:develop
+    #     docker pull gcr.io/focal-freedom-236620/agent:develop
+    #   displayName: 'Pull latest images'
+
+    - script: |
+        $(cssh) $(agent) -- bash $(dir)/start.sh tutorial
+        # Use dev version of agent and controller until 1.3.0 release
+        # ./start.sh -ct gcr.io/focal-freedom-236620/controller:develop -a gcr.io/focal-freedom-236620/agent:develop -cn gcr.io/focal-freedom-236620/connector:develop
+      displayName: 'Start Connector, Controller, Agent, and tutorial microservices'
+
+    - script: |
+        $(cssh) $(agent) -- bash $(dir)/test.sh
+      displayName: 'Run Tests'
+
+    - script: |
+        $(cssh) $(agent) -- bash $(dir)/stop.sh
+      displayName: 'Stop Connector, Controller, and Agent'
+
+    - script: |
+        echo "===== IOFOG AGENT LOG ====="
+        docker exec iofog-agent cat /var/log/agent.out.log
+      displayName: 'Print logs'
+      condition: failed()
+
+    - template: vm-down.yaml

templates/vm-down.yaml

@@ -0,0 +1,5 @@
+steps:
+- script: |
+    gcloud compute --project=$(gcp.project.name) instances delete demo-ci-$(jobuuid) --zone=$(gcp.vm.zone) --delete-disks=all -q
+  displayName: 'Teardown VMs'
+  condition: always()

templates/vm-up.yaml

@@ -0,0 +1,44 @@
+parameters:
+  distro: ''
+  repo: ''
+
+steps:
+- task: DownloadSecureFile@1
+  displayName: 'Download secure file'
+  inputs:
+    secureFile: 'azure-gcp.json'
+- bash: |
+    echo "gcp.project.name: $(gcp.project.name)"
+    CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)"
+    echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+    sudo apt-get update && sudo apt-get install google-cloud-sdk
+    gcloud --quiet auth activate-service-account --key-file=$(Agent.TempDirectory)/azure-gcp.json
+    gcloud --quiet config set project $(gcp.project.name)
+  displayName: 'set up gcloud'
+- script: |
+    distro=${{ parameters.distro }}
+    repo=${{ parameters.repo }}
+    echo "distro: $distro"
+    echo "repo: $repo"
+    echo "gcp.project.name: $(gcp.project.name)"
+    echo "gcp.vm.zone: $(gcp.vm.zone)"
+    echo "gcp.svcacc.name: $(gcp.svcacc.name)"
+    echo "jobuuid: $(jobuuid)"
+    gcloud compute --project=$(gcp.project.name) instances create demo-ci-$(jobuuid) --zone=$(gcp.vm.zone) --machine-type=n1-standard-1 --subnet=default --network-tier=PREMIUM --maintenance-policy=MIGRATE --service-account=$(gcp.svcacc.name) --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append --image=$distro --image-project=$repo --boot-disk-size=200GB --boot-disk-type=pd-standard --boot-disk-device-name=demo-ci-$(jobuuid)
+    vm_host=$(gcloud compute instances list | grep demo-ci-$(jobuuid) | awk '{print $5}')
+    echo "vm_host: $vm_host"
+    echo "##vso[task.setvariable variable=agent]$(gcp.vm.user)@$vm_host"
+  displayName: 'Deploy Test VM'
+- script: |
+    echo "agent: $(agent)"
+    seconds=0
+    until $(cssh) $(agent) -- echo "SSH success"; do
+      if [ $seconds -gt 60 ]; then
+        echo "Timed out waiting for $(agent)"
+        exit 1
+      fi
+      seconds=$((seconds+1))
+      sleep 1
+    done
+  displayName: 'Wait for SSH access'

test.sh

@@ -53,10 +53,15 @@ if [[ -f "${CONFIGURE_SSH_LOG_FILE}" ]]; then
 fi
 echo '' > "${CONFIGURE_SSH_LOG_FILE}"
 {
-    echo 'Removing /var/lib/apt/lists/lock' >> "${CONFIGURE_SSH_LOG_FILE}"
-    docker exec iofog-agent sudo rm /var/lib/apt/lists/lock >> "${CONFIGURE_SSH_LOG_FILE}" 2>&1
     echo 'Updating apt-get' >> "${CONFIGURE_SSH_LOG_FILE}"
-    docker exec iofog-agent apt-get update -y  >> "${CONFIGURE_SSH_LOG_FILE}" 2>&1
+    SECONDS=1
+    until docker exec iofog-agent apt-get update -y  >> "${CONFIGURE_SSH_LOG_FILE}" 2>&1; do
+        if [ $SECONDS -gt 60 ]; then
+            exit 1
+        fi
+        sleep 1
+        SECONDS=$((SECONDS+1))
+    done
     echo 'Installing Openssh-server' >> "${CONFIGURE_SSH_LOG_FILE}"
     docker exec iofog-agent apt-get install -y --fix-missing openssh-server  >> "${CONFIGURE_SSH_LOG_FILE}" 2>&1
     echo 'Running apt-get install -fy' >> "${CONFIGURE_SSH_LOG_FILE}"