Fix docker support and update github actions

* Ensure that docker is fully functional with the new changes * Determined that docker, unlike podman, only assigns the host IP after the instance has been created * Updated the GitHub Actions to test both podman and docker * Used a simple acceptance test for validation since the upstream beaker tests are wired for Vagrant * Fail if the container is in an inaccessible state instead of trying to SSH forever
voxpupuli · Feb 28, 2021 · 50912bf · 50912bf
1 parent 958080b
commit 50912bf
Show file tree

Hide file tree

Showing 7 changed files with 151 additions and 42 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,13 +1,19 @@
 name: Test
 
 on:
-  - pull_request
+  push:
+    branches:
+      - test_me_github
+  pull_request:
+    branches:
+      - main
+      - master
 
 jobs:
-  test:
+  rspec:
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         ruby:
           - "2.4"
@@ -16,17 +22,84 @@ jobs:
           - "2.7"
     env:
       BUNDLE_WITHOUT: release
-    name: Ruby ${{ matrix.ruby }}
+    name: RSpec - Ruby ${{ matrix.ruby }}
     steps:
       - uses: actions/checkout@v2
       - name: Install Ruby ${{ matrix.ruby }}
         uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
-      - name: Run spec tests
+      - name: install bundler
+        run: |
+          gem install bundler -v '~> 1.17.3'
+          bundle update
+      - name: spec tests
         run: bundle exec rake test:spec
-      # It seems some additonal setup of Docker may be needed for
-      # the acceptance tests to work.
-      # - name: Run acceptance tests
-      #   run: bundle exec rake test:acceptance
+
+  docker:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        ruby:
+          - "2.6"
+    env:
+      BUNDLE_WITHOUT: release
+    name: Docker - Ruby ${{ matrix.ruby }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: install bundler
+        run: |
+          gem install bundler -v '~> 1.17.3'
+          bundle update
+      - name: install container runtime
+        run: |
+          sudo apt-get remove -y docker docker-engine docker.io containerd runc ||:
+          sudo apt-get update -y
+          sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+          sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+          sudo apt-get update -y
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+          sudo systemctl start docker
+      - name: Run acceptance tests
+        run: bundle exec rake test:acceptance
+
+  podman:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        ruby:
+          - "2.6"
+    env:
+      BUNDLE_WITHOUT: release
+    name: Podman - Ruby ${{ matrix.ruby }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: install bundler
+        run: |
+          gem install bundler -v '~> 1.17.3'
+          bundle update
+      # We need the latest version of podman for this to work
+      - name: install container runtime
+        run: |
+          . /etc/os-release
+          curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add -
+          echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/podman.list > /dev/null
+          sudo apt-get update
+          sudo apt-get -y install podman
+          sudo systemctl start podman
+      - name: Run acceptance tests
+        run: bundle exec rake test:acceptance
diff --git a/Gemfile.local b/Gemfile.local
@@ -0,0 +1,5 @@
+group :acceptance_testing do
+  # Needed for podman testing
+  gem "docker-api", :git => 'https://github.com/trevor-vaughan/docker-api', :branch => 'podman-compat'
+  gem "beaker-rspec"
+end
diff --git a/Rakefile b/Rakefile
@@ -32,11 +32,15 @@ A quick acceptance test, named because it has no pre-suites to run
       beaker_test_base_dir = File.join(beaker_gem_dir, 'acceptance/tests/base')
       load_path_option = File.join(beaker_gem_dir, 'acceptance/lib')
 
+      ENV['BEAKER_setfile'] = 'acceptance/config/nodes/hosts.yaml'
       sh("beaker",
          "--hosts", "acceptance/config/nodes/hosts.yaml",
-         "--tests", beaker_test_base_dir,
+          # We can't run these tests until the rsync support in the main
+          # beaker/host.rb is updated to work with passwords.
+          # "--tests", beaker_test_base_dir,
+          # "--load-path", load_path_option,
+         "--tests", 'acceptance/tests/',
          "--log-level", "debug",
-         "--load-path", load_path_option,
          "--debug")
     end
 

diff --git a/acceptance/config/nodes/hosts.yaml b/acceptance/config/nodes/hosts.yaml
@@ -12,12 +12,6 @@ HOSTS:
       - classifier
       - default
     docker_cmd: '["/sbin/init"]'
-    docker_cap_add:
-      - AUDIT_WRITE
-    dockeropts:
-      Labels:
-        one: '1'
-        two: '2'
   centos7:
     platform: el-7-x86_64
     hypervisor: docker
@@ -26,10 +20,21 @@ HOSTS:
       - agent
     docker_cmd: '/usr/sbin/sshd -D -E /var/log/sshd.log'
     use_image_entrypoint: true
-    dockeropts:
-      HostConfig:
-        Privileged: true
 CONFIG:
   nfs_server: none
   consoleport: 443
   log_level: verbose
+  # Ubuntu runners need to run with full privileges
+  # RHEL derivitives just need the docker cap AUDIT_WRITE
+  dockeropts:
+    HostConfig:
+      Privileged: true
+  # docker_cap_add:
+  #   - AUDIT_WRITE
+  type: aio
+  ssh:
+    verify_host_key: false
+    user_known_hosts_file: '/dev/null'
+    password: root
+    auth_methods:
+      - password
diff --git a/acceptance/tests/00_default_spec.rb b/acceptance/tests/00_default_spec.rb
@@ -0,0 +1,10 @@
+require 'beaker'
+require 'beaker-rspec'
+
+RSpec.describe 'it can connect' do
+  hosts.each do |host|
+    context "on #{host}" do
+      on(host, 'ls /tmp')
+    end
+  end
+end
diff --git a/lib/beaker/hypervisor/docker.rb b/lib/beaker/hypervisor/docker.rb
@@ -135,32 +135,35 @@ def get_ssh_connection_info(container)
         port: nil
       }
 
+      container_json = container.json
+      network_settings = container_json['NetworkSettings']
+      host_config = container_json['HostConfig']
+
+      ip = nil
+      port = nil
       # Talking against a remote docker host which is a normal docker host
       if @docker_type == 'docker' && ENV['DOCKER_HOST'] && !ENV.fetch('DOCKER_HOST','').include?(':///')
         ip = URI.parse(ENV['DOCKER_HOST']).host
       else
         # Swarm or local docker host
         if in_container?
-          ip = container.json["NetworkSettings"]["Gateway"]
+          gw = network_settings['Gateway']
+          ip = gw unless (gw.nil? || gw.empty?)
         else
-          ip = container.json["NetworkSettings"]["Ports"]["22/tcp"][0]["HostIp"]
+          port22 = network_settings.dig('Ports','22/tcp')
+          ip = port22[0]["HostIp"] if port22
         end
       end
 
-      network_settings = container.json['NetworkSettings']
-      host_config = container.json['HostConfig']
-
-      port = '22'
-      if host_config['NetworkMode'] == 'bridge' && network_settings['IPAddress'] && !network_settings['IPAddress'].empty?
-        ssh_connection_info[:ip] = network_settings['IPAddress']
+      if host_config['NetworkMode'] != 'slirp4netns' && network_settings['IPAddress'] && !network_settings['IPAddress'].empty?
+        ip = network_settings['IPAddress']
       else
-        port = network_settings['Ports']['22/tcp'][0]['HostPort']
-
-        # Update host metadata
-        ssh_connection_info[:ip] = (ip == '0.0.0.0') ? '127.0.0.1' : ip
+        port22 = network_settings.dig('Ports','22/tcp')
+        port = port22[0]['HostPort'] if port22
       end
 
-      ssh_connection_info[:port] = port
+      ssh_connection_info[:ip] = (ip == '0.0.0.0') ? '127.0.0.1' : ip
+      ssh_connection_info[:port] = port || '22'
       ssh_connection_info
     end
 
@@ -242,7 +245,8 @@ def provision
           while(!ok && (retries < 5))
             container = ::Docker::Container.create(container_opts)
 
-            if (get_ssh_connection_info(container)[:port].to_i < 1024) && (Process.uid != 0)
+            ssh_info = get_ssh_connection_info(container)
+            if ssh_info[:ip] == '127.0.0.1' && (ssh_info[:port].to_i < 1024) && (Process.uid != 0)
               @logger.debug("#{host} was given a port less than 1024 but you are not running as root, retrying")
 
               container.delete
@@ -267,26 +271,33 @@ def provision
         @logger.debug("Starting container #{container.id}")
         container.start
 
+        begin
+          container.stats
+        rescue StandardError => e
+          container.delete
+          raise "Container '#{container.id}' in a bad state: #{e}"
+        end
+
         # Preserve the ability to talk directly to the underlying API
         #
         # You can use any method defined by the docker-api gem on this object
         # https://github.com/swipely/docker-api
         host[:docker_container] = container
 
-        ssh_connection_info = get_ssh_connection_info(container)
-
-        ip = ssh_connection_info[:ip]
-        port = ssh_connection_info[:port]
-
-        @logger.info("Using container connection at #{ip}:#{port}")
-
         if install_and_run_ssh(host)
           @logger.notify("Installing ssh components and starting ssh daemon in #{host} container")
           install_ssh_components(container, host)
           # run fixssh to configure and start the ssh service
           fix_ssh(container, host)
         end
 
+        ssh_connection_info = get_ssh_connection_info(container)
+
+        ip = ssh_connection_info[:ip]
+        port = ssh_connection_info[:port]
+
+        @logger.info("Using container connection at #{ip}:#{port}")
+
         forward_ssh_agent = @options[:forward_ssh_agent] || false
 
         host['ip'] = ip
@@ -298,7 +309,7 @@ def provision
           :auth_methods => ['password', 'publickey', 'hostbased', 'keyboard-interactive']
         }
 
-        @logger.debug("node available as  ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@#{ip} -p #{port}")
+        @logger.debug("node available as ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@#{ip} -p #{port}")
         host['docker_container_id'] = container.id
         host['docker_image_id'] = image.id
         host['vm_ip'] = container.json["NetworkSettings"]["IPAddress"].to_s

diff --git a/spec/beaker/hypervisor/docker_spec.rb b/spec/beaker/hypervisor/docker_spec.rb
@@ -88,6 +88,7 @@ module Beaker
       container = double('Docker::Container')
       allow( container ).to receive(:id).and_return('abcdef')
       allow( container ).to receive(:start)
+      allow( container ).to receive(:stats)
       allow( container ).to receive(:info).and_return(
         *(0..2).map { |index| { 'Names' => ["/spec-container-#{index}"] } }
       )