Enable s2i windows testing

- add 'build-sti-base-windows' vagrant command to build windows AMI with required prereqs
- hooks to enable simplification of Vagrantfile (auto-discover AMI id, auto-configure standard Vagrantfile parameters for AWS)
- hook to wait for Windows instance to report ready after sysprep
- provisioners to allow a Windows instance to be autoconfigured to reasonably securely use Docker on a helper RHEL instance
- miscellaneous fixes

lib/vagrant-openshift/action.rb

@@ -66,6 +66,12 @@ def self.install_golang(options)
         end
       end
 
+      def self.build_sti_base_windows(options)
+        Vagrant::Action::Builder.new.tap do |b|
+          b.use BuildStiBaseWindows, options
+        end
+      end
+
       def self.install_origin(options)
         Vagrant::Action::Builder.new.tap do |b|
           b.use SetHostName
@@ -386,6 +392,7 @@ def self.install_rhc(options)
       autoload :InstallOriginRhel7, action_root.join("install_origin_rhel7")
       autoload :BuildOrigin, action_root.join("build_origin")
       autoload :BuildSti, action_root.join("build_sti")
+      autoload :BuildStiBaseWindows, action_root.join("build_sti_base_windows")
       autoload :PrepareSshConfig, action_root.join("prepare_ssh_config")
       autoload :SyncLocalRepository, action_root.join("sync_local_repository")
       autoload :SyncUpstreamRepository, action_root.join("sync_upstream_repository")

lib/vagrant-openshift/action/build_sti_base_windows.rb

@@ -0,0 +1,127 @@
+#--
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#++
+
+require "fog"
+require_relative "../aws"
+
+module Vagrant
+  module Openshift
+    module Action
+      class BuildStiBaseWindows
+        include CommandHelper
+
+        def initialize(app, env, options)
+          @app = app
+          @env = env
+          @options = options
+        end
+
+        def call(env)
+          aws_creds = Vagrant::Openshift::AWS::aws_creds()
+          compute = Fog::Compute.new(Vagrant::Openshift::AWS::fog_config(aws_creds))
+
+          @env[:ui].info("Finding base AMI...")
+          images = compute.images.all({"Owner" => "amazon", "name" => "Windows_Server-2012-R2_RTM-English-64Bit-Base-*", "state" => "available"})
+          # https://github.com/fog/fog-aws/issues/320 would make this less hacky
+          images.sort_by! {|image| image.name[-10,10]}
+          @env[:ui].info("Using base AMI #{images.last.name}, #{images.last.id}")
+
+          @env[:ui].info("Creating instance...")
+          instance = compute.servers.create(
+            :tags => {"Name" => "#{@options[:instance_prefix]}-windows2012r2_#{Time.now.to_i}"},
+            :image_id => images.last.id,
+            :flavor_id => @options[:flavor_id],
+            :key_name => aws_creds["AWSKeyPairName"],
+            :subnet_id => @options[:subnet_id],
+            :user_data => <<'USERDATA'
+<powershell>
+Set-ExecutionPolicy -ExecutionPolicy Bypass
+
+# Download and install Cygwin
+$client = New-Object System.Net.WebClient
+$client.DownloadFile("https://cygwin.com/setup-x86_64.exe", "C:\Windows\Temp\setup-x86_64.exe")
+&C:\Windows\Temp\setup-x86_64.exe -q -D -L -d -o -s http://mirrors.kernel.org/sourceware/cygwin -l C:\Windows\Temp\cygwin -R C:\cygwin -P curl -P gcc-core -P git -P make -P openssh -P rsync | Out-Null
+Remove-Item -Recurse C:\Windows\Temp\cygwin
+Remove-Item C:\Windows\Temp\setup-x86_64.exe
+$env:Path += ";C:\cygwin\bin"
+
+# Configure and start OpenSSH
+$env:LOGONSERVER = "\\" + $env:COMPUTERNAME  # http://petemoore.github.io/general/taskcluster/2016/03/30/windows-sshd-cygwin-ec2-aws.html
+&bash -c "ssh-host-config --yes -w Pa$$w0rd"
+&bash -c "sed -i -e 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/sshd_config"
+&bash -c "sed -i -e 's/#ChallengeResponseAuthentication yes/ChallengeResponseAuthentication no/' /etc/sshd_config"
+&cygrunsrv -S sshd
+&netsh advfirewall firewall add rule name=SSH profile=any localport=22 enable=yes action=allow dir=in protocol=tcp
+
+# Download and install Golang
+$client.DownloadFile("https://storage.googleapis.com/golang/go1.7.1.windows-amd64.msi", "C:\Windows\Temp\go1.7.1.windows-amd64.msi")
+&msiexec /qb /i C:\Windows\Temp\go1.7.1.windows-amd64.msi | Out-Null
+Remove-Item C:\Windows\Temp\go1.7.1.windows-amd64.msi
+
+# Create Administrator's home directory
+Out-Null | &bash --login
+&bash -c "mkdir /home/Administrator/.ssh; curl -o /home/Administrator/.ssh/authorized_keys http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key"
+&bash -c "mkdir /data"
+$stream = New-Object System.IO.StreamWriter @("c:\cygwin\home\Administrator\.bash_profile", $true)
+$stream.Write("export GOPATH='C:\cygwin\data'`n")
+$stream.Close()
+
+# Download and install Docker client
+$client.DownloadFile("https://get.docker.com/builds/Windows/x86_64/docker-1.10.3.exe", "C:\cygwin\usr\local\bin\docker.exe")
+
+# Install a fake sudo
+$stream = New-Object System.IO.StreamWriter "C:\cygwin\bin\sudo"
+$stream.Write("#!/bin/bash`n`nwhile [[ `$# -ge 0 && `$1 = -* ]]; do`n  shift`ndone`n`n`"`$@`"`n")
+$stream.Close()
+$stream = New-Object System.IO.StreamWriter "C:\cygwin\etc\sudoers"
+$stream.Close()
+
+# Start sysprep (will power off instance when done)
+&"C:\Program Files\Amazon\Ec2ConfigService\ec2config.exe" -sysprep
+</powershell>
+USERDATA
+          )
+
+          @env[:ui].info("Instance ID is #{instance.id}")
+
+          @env[:ui].info("Waiting for instance state == running...")
+          instance.wait_for { ready? }
+
+          @env[:ui].info("Waiting for instance state == stopped...")
+          instance.wait_for(3600) { state == "stopped" }
+
+          @env[:ui].info("Creating AMI...")
+          image_req = compute.create_image(instance.id, "#{@options[:ami_prefix]}-windows2012r2_#{Time.now.to_i}", "", false)
+
+          @env[:ui].info("AMI ID is #{image_req.body['imageId']}")
+
+          @env[:ui].info("Waiting for AMI state == available...")
+          Fog.wait_for {
+            image = compute.images.get(image_req.body["imageId"])
+            !image.nil? && image.ready?
+          }
+
+          @env[:ui].info("Setting instance name to terminate...")
+          compute.create_tags(instance.id, "Name" => "terminate")
+
+          @env[:ui].info("Done")
+
+          @app.call(env)
+        end
+      end
+    end
+  end
+end

lib/vagrant-openshift/action/clone_upstream_repositories.rb

@@ -27,9 +27,9 @@ def initialize(app, env, options={})
         end
 
         def call(env)
-          ssh_user = env[:machine].ssh_info[:username]
+          ssh_user, ssh_group = get_ssh_user_and_group(env[:machine])
 
-          remote_write(env[:machine], "/#{ssh_user}/.ssh/config", "#{ssh_user}:#{ssh_user}", "0600") {
+          remote_write(env[:machine], "/#{ssh_user}/.ssh/config", "#{ssh_user}:#{ssh_group}", "0600") {
 %{Host github.com
 StrictHostKeyChecking no
 UserKnownHostsFile=/dev/null
@@ -52,7 +52,7 @@ def call(env)
           end
 
           sudo(env[:machine], "mkdir -p #{Constants.build_dir}")
-          sudo(env[:machine], "mkdir -p #{Constants.build_dir + "builder"} && chown -R #{ssh_user}:#{ssh_user} #{Constants.build_dir}")
+          sudo(env[:machine], "mkdir -p #{Constants.build_dir + "builder"} && chown -R #{ssh_user}:#{ssh_group} #{Constants.build_dir}")
           do_execute env[:machine], git_clone_commands
 
           @app.call(env)

lib/vagrant-openshift/action/download_artifacts_sti.rb

@@ -34,12 +34,17 @@ def call(env)
 
           artifacts_dir = Pathname.new(File.expand_path(machine.env.root_path + "artifacts"))
           download_map = {
-            "/var/log/yum.log"                     => artifacts_dir + "yum.log",
-            "/var/log/secure"                      => artifacts_dir + "secure",
-            "/var/log/audit/audit.log"             => artifacts_dir + "audit.log",
             "/tmp/sti/"                            => artifacts_dir + "sti/"
           }
 
+          if !is_windows?(machine)
+            download_map.merge!({
+              "/var/log/yum.log"                     => artifacts_dir + "yum.log",
+              "/var/log/secure"                      => artifacts_dir + "secure",
+              "/var/log/audit/audit.log"             => artifacts_dir + "audit.log"
+            })
+          end
+
           download_map.each do |source,target|
             machine.ui.info "Downloading artifacts from '#{source}' to '#{target}'"
             if target.to_s.end_with? '/'

lib/vagrant-openshift/action/generate_template.rb

@@ -15,6 +15,7 @@
 #++
 require 'yaml'
 require 'fog'
+require_relative '../aws'
 
 module Vagrant
   module Openshift
@@ -48,10 +49,14 @@ def call(env)
           @openstack_creds_file = Pathname.new(File.expand_path(@openstack_creds_file))
           box_info[:openstack_creds_file] = @openstack_creds_file
 
-          @aws_creds_file = ENV['AWS_CREDS'].nil? || ENV['AWS_CREDS'] == '' ? "~/.awscred" : ENV['AWS_CREDS']
-          @aws_creds_file = Pathname.new(File.expand_path(@aws_creds_file))
-          box_info[:aws_creds_file] = @aws_creds_file
-          find_ami_from_tag(box_info)
+          if !box_info[:aws].nil? && !box_info[:aws][:ami_tag_prefix].nil?
+            begin
+              aws_creds = Vagrant::Openshift::AWS::aws_creds()
+              compute = Fog::Compute.new(Vagrant::Openshift::AWS::fog_config(aws_creds, box_info[:aws][:ami_region]))
+              box_info[:aws][:ami] = Vagrant::Openshift::AWS::find_ami_from_tag(compute, box_info[:aws][:ami_tag_prefix], @options[:required_name_tag])
+            rescue AWSCredentialsNotConfiguredError
+            end
+          end
 
           gopath = nil
           if ENV['GOPATH'] && !ENV['GOPATH'].empty?
@@ -103,33 +108,6 @@ def call(env)
 
           @app.call(env)
         end
-
-        private
-
-        def find_ami_from_tag(box_info)
-          return if box_info[:aws].nil? || box_info[:aws][:ami_tag_prefix].nil?
-          @env[:ui].info("Reading AWS credentials from #{@aws_creds_file.to_s}")
-          if @aws_creds_file.exist?
-            aws_creds = @aws_creds_file.exist? ? Hash[*(File.open(@aws_creds_file.to_s).readlines.map{ |l| l.strip!
-                                                          l.split('=') }.flatten)] : {}
-
-            fog_config = {
-                :provider              => :aws,
-                :region                => box_info[:aws][:ami_region],
-                :aws_access_key_id     => aws_creds['AWSAccessKeyId'],
-                :aws_secret_access_key => aws_creds['AWSSecretKey'],
-            }
-
-            aws_compute = Fog::Compute.new(fog_config)
-            @env[:ui].info("Searching for latest base AMI")
-            image_filter = {'Owner' => 'self', 'name' => "#{box_info[:aws][:ami_tag_prefix]}*", 'state' => 'available' }
-            image_filter['tag:Name'] = @options[:required_name_tag] unless @options[:required_name_tag].nil?
-            images = aws_compute.images.all(image_filter)
-            latest_image = images.sort_by{ |i| i.name.split("_")[-1].to_i }.last
-            box_info[:aws][:ami] = latest_image.id
-            @env[:ui].info("Found: #{latest_image.id} (#{latest_image.name})")
-          end
-        end
       end
     end
   end

lib/vagrant-openshift/action/run_sti_tests.rb

@@ -34,7 +34,7 @@ def call(env)
           cmds = ['OUTPUT_COVERAGE=/tmp/sti/artifacts/coverage hack/test-go.sh']
 
           if @options[:all]
-            cmds << 'STI_TIMEOUT="--timeout 240s" hack/test-integration.sh'
+            cmds << 'hack/test-integration.sh'
             cmds << 'hack/test-stirunimage.sh'
           end
 
@@ -49,6 +49,8 @@ def call(env)
 "
           end
 
+          # TODO: the PATH export below should be removed once
+          # https://github.com/openshift/source-to-image/pull/625 merges.
           _,_,env[:test_exit_code] = sudo(env[:machine], %{
 set -e
 pushd #{Constants.build_dir}/source-to-image >/dev/null

lib/vagrant-openshift/aws.rb

@@ -0,0 +1,60 @@
+#--
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#++
+
+require "pathname"
+require "fog"
+
+module Vagrant
+  module Openshift
+    class AWSCredentialsNotConfiguredError < RuntimeError
+      def initialize(msg="AWS credentials not configured")
+        super
+      end
+    end
+
+    class AWS
+      def self.aws_creds()
+        aws_creds_file = ENV["AWS_CREDS"].nil? || ENV["AWS_CREDS"] == "" ? "~/.awscred" : ENV["AWS_CREDS"]
+        aws_creds_file = Pathname.new(File.expand_path(aws_creds_file))
+
+        raise AWSCredentialsNotConfiguredError if !aws_creds_file.exist?
+
+        return Hash[*(aws_creds_file.open.readlines.map{|l| l.strip.split("=")}.flatten)]
+      end
+
+      def self.fog_config(aws_creds, region="us-east-1")
+        {
+          :provider              => :aws,
+          :aws_access_key_id     => aws_creds["AWSAccessKeyId"],
+          :aws_secret_access_key => aws_creds["AWSSecretKey"],
+          :region                => region
+        }
+      end
+
+      def self.find_ami_from_tag(compute, ami_tag_prefix, required_name_tag=nil)
+        image_filter = {"Owner" => "self", "name" => "#{ami_tag_prefix}*", "state" => "available"}
+        image_filter["tag:Name"] = required_name_tag unless required_name_tag.nil?
+        images = compute.images.all(image_filter)
+        latest_image = images.sort_by{|i| i.name.split("_")[-1].to_i}.last
+        if !latest_image.nil?
+          return latest_image.id
+        end
+
+        nil
+      end
+    end
+  end
+end

lib/vagrant-openshift/command/build_sti_base_windows.rb

@@ -0,0 +1,66 @@
+#--
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#++
+require_relative "../action"
+
+module Vagrant
+  module Openshift
+    module Commands
+      class BuildStiBaseWindows < Vagrant.plugin(2, :command)
+        include CommandHelper
+
+        def self.synopsis
+          "install the prereqs for source-to-image on Windows"
+        end
+
+        def execute
+          options = {
+            :ami_prefix => ENV["USER"],
+            :flavor_id => "m4.large",
+            :instance_prefix => ENV["USER"]
+          }
+
+          opts = OptionParser.new do |o|
+            o.banner = "Usage: vagrant build-sti-base-windows subnet_id"
+            o.separator ""
+
+            o.on("--ami-prefix prefix", "Prefix for AMI name.") do |prefix|
+              options[:ami_prefix] = prefix
+            end
+
+            o.on("--flavor flavor", "Flavor of instance.") do |flavor|
+              options[:flavor_id] = flavor
+            end
+
+            o.on("--instance-prefix prefix", "Prefix for instance name.") do |prefix|
+              options[:instance_prefix] = prefix
+            end
+          end
+
+          # Parse the options
+          argv = parse_options(opts)
+          return if !argv
+
+          raise Errors::CLIInvalidOptions, help: opts.help.chomp if argv.length != 1
+          options[:subnet_id] = argv[0]
+
+          actions = Vagrant::Openshift::Action.build_sti_base_windows(options)
+          @env.action_runner.run actions
+          0
+        end
+      end
+    end
+  end
+end