Prepping for merge back to master

including backing out (partially) a few ideas/WIPs

.example.env

@@ -1,21 +1,24 @@
-export ATLAS_TOKEN='00-AAAAAAAAAAAAAAAAAAAAAAAAAAAA-BBBBBBBBBBBBBBBBBBBBBBBBB-CCCCCCCCC'
 export AWS_ACCESS_KEY_ID='AKDDDDDDDDDDDDDDDDDD'
 export AWS_REGION='us-east-1'
 export AWS_SECRET_ACCESS_KEY='ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ'
-export TRAVIS_COOKBOOKS_BRANCH='master'
-export TRAVIS_COOKBOOKS_SHA='fffffff'
 export DOCKER_WORKERS_VPC_EXTERNAL_ZONE_ID='ZAAAAAAAAAAAAA'
 export DOCKER_WORKERS_VPC_INTERNAL_ZONE_ID='ZBBBBBBBBBBBBB'
 export EDITOR='vim'
 export JOB_BOARD_IMAGES_URL='https://auth:token@job-board-server.example.com/images'
+export JUPITER_BRAIN_BINARY='/dev/null'
 export OPSMATIC_INTEGRATION_TOKEN='aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'
 export PACKER_LOG='nonempty'
-export SSH_KEY_PATH='/path/to/id_rsa'
+export SSH_KEY_PATH='.example_passwordless_key_rsa'
 export SSH_USERNAME='whoami'
+export TRAVIS_CI_BASE_AMI='ami-fafafafa'
+export TRAVIS_COOKBOOKS_BRANCH='master'
 export TRAVIS_COOKBOOKS_DIR='/path/to/travis-cookbooks'
+export TRAVIS_COOKBOOKS_SHA='fafafaf'
 export TRAVIS_SUBNET_ID='subnet-aaaaaaaa'
 export TRAVIS_VPC_ID='vpc-bbbbbbbb'
 export TRAVIS_WORKER_BINARY_DIR='/tmp'
+export TRAVIS_WORKER_AMI='ami-fafafafa'
+export UBUNTU_TRUSTY_AMI='ami-fafafafa'
 export VSPHERE_CLUSTER='cluster-name'
 export VSPHERE_DATACENTER='datacenter-name'
 export VSPHERE_DATASTORE='datastore-name'

.example_passwordless_key_rsa

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAuFeHK+j6XY3Lyl2JnAi81q24klN6GezzVwDFQpgCZRPQgRkJ
+I6r2dS99MHgpH4youWl48mwBwz/TbBguWemN0Y2Ta0S9+HzG0oKfFNIzOm9nc8x7
+10qYSDXFYfU2/qnkOxa2LE6rR68rtGnrL9mn0vMjnnkMWPLXgOP7axwi3qHZ6C+U
+mdqneRb+CAQpyB93p4HWPNbOBzqZ+o0KWa82vCD9aHm4onDluGEzjKm8nyjMaQ/6
+/ICFyC9xu/asnS+RI030+x7PvAiS7dvB5JTOIrcfEs/qypHXI1cHLqjbvCVt4owK
+AG1N8KODNz6GwgF/ys4mTP7dQSufRk2W4cTdwwIDAQABAoIBAGkfWPEcafIB044y
+ay/096NA6/ld60pvipE52kPQpcT4WLXWDhUuAF/xVeXXEDRypSEe2Z79IsAyXKHO
+5FfXzU0L/F52lBLpLQSNAqT//cC8GPW7TYAHhPHg4A+to8/0GDfuCrMXpas6ZKT8
+pebiq0ar6ssLYpQy39wbppt9LxFfp+nUj3qboMaSicld/AdAqTFOHWSnJqfaNOar
+rL/0Wau3zQ+uCajfOwbkTZom78JEfIE65tQZtu4r4EsZgh24zH0x8X3TxJYtfWQk
++ffpI7+DZadRpzpNM+PEosuOooHAKDbSUh/PVTMNRXSdqd6BXapxZyan4yMuF9QE
+eEMHg6ECgYEA8OgCCGwgVDH/ssKJl97kafsDUIY4Czo9te1P3zXa53gwx4elcg1c
+U0ub7bHxLvJ16ThOUSmZZ9p+c5eVSLh/kgXOS+MkRog+8EFfrb/7NtZGX8AJQQgR
+LOIO+sEUsS2R/tOchL1BxfcJLHEgdAnWVXZaYpgFCaE8xthrZ5d0/ZECgYEAw+RC
+8AVm5e6TK99NCUSxNCSTvQjSWFxUPfM/Ia0NT1cbnNsWnUSb4MeXN8SXl6yL4Qjh
+ckO65NsvMsUQ4BtwZOP795g9l+HGEBGQzHr/Hz5O4ygss6o8yBYcKFjF1YwnQtCx
+r+wlb0kUIgfclciXzZPCGHB6GwG4V0t5gNBfTBMCgYEA4dIJvc1ljKcFspLLGdK1
+OGc01zHJrTSeDkBzI6A/H8U5rIAJ5VyC1A3HD8xCciiTwNeoCkq9Y22cJUzNgkp4
+QvAwTlCQe5FRtjMBtkSE/wVjGHRR+GTQfMs3EXUWqXM3LLE+QeuKkUCdl8w/iKaE
+BL6ZwVwRNNKwvxC1tXhzERECgYAlNdnlQBFwweXg4pT+mrbSgdQyh1d1g2xZLDik
+y/uIQ7KyFAfyrdKyWxeFd2mYXSVhBM/WZbk9ucj4auiuuWebMpdtS9T/Ydq+uKd5
+wC9wHuTbT4lWQ5MyKskEfQBvdSUMExp9oWzm+8nAQSAMr8LuQRAQlmULdkMvCd41
+st83UwKBgGnDCQZ0NVmzyxc1XD+kC35PIC3wnj8Gwj3cuVbXe4W2GfA/awbj1GCb
+0lGFf4q71B5cUG5kHpZ6SOgsiNQLsLaVLUuCSCTctEW7w3FOvIeElO67Tu5VTQgW
+0tbQsktkTOMzqeMCULmbsQeXl57tI988RoKriuwOdE6UpprRI4vh
+-----END RSA PRIVATE KEY-----

.example_passwordless_key_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4V4cr6PpdjcvKXYmcCLzWrbiSU3oZ7PNXAMVCmAJlE9CBGQkjqvZ1L30weCkfjKi5aXjybAHDP9NsGC5Z6Y3RjZNrRL34fMbSgp8U0jM6b2dzzHvXSphINcVh9Tb+qeQ7FrYsTqtHryu0aesv2afS8yOeeQxY8teA4/trHCLeodnoL5SZ2qd5Fv4IBCnIH3engdY81s4HOpn6jQpZrza8IP1oebiicOW4YTOMqbyfKMxpD/r8gIXIL3G79qydL5EjTfT7Hs+8CJLt28HklM4itx8Sz+rKkdcjVwcuqNu8JW3ijAoAbU3wo4M3PobCAX/KziZM/t1BK59GTZbhxN3D foo@bar.local

.gitignore

@@ -7,7 +7,8 @@ tmp/*
 ~*
 /Gemfile.lock
 .vagrant
-.env
+.env*
+*env
 gce-packer-acct.json
 packer_cache
 /output-*

.travis.yml

@@ -1,14 +1,30 @@
 language: generic
-sudo: true
+
 dist: trusty
+
+sudo: true
+
+env:
+  global:
+  - PATH="/opt/chefdk/bin:/opt/chefdk/embedded/bin:/opt/chef/bin:/usr/local/packer:$PATH"
+  - PACKER_VERSION="0.10.1"
+
+addons:
+  apt:
+    sources:
+    - chef-stable-precise
+    packages:
+    - chefdk
+
 install:
-- wget https://releases.hashicorp.com/packer/0.10.1/packer_0.10.1_linux_amd64.zip
-- sudo unzip packer_0.10.1_linux_amd64.zip -d /usr/local/packer
-- export PATH=$PATH:/usr/local/packer
-before_install:
-- openssl aes-256-cbc -K $encrypted_aba34ae14add_key -iv $encrypted_aba34ae14add_iv
-  -in .env.enc -out .env -d
-before_script:
-- source .env
+- curl -sSL -o packer.zip  "https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip"
+- sudo unzip packer.zip -d /usr/local/packer
+
 script:
-- packer build -only=googlecompute <(bin/yml2json < worker.yml)
+- make hackcheck
+- make
+- git diff --exit-code
+- git diff --cached --exit-code
+- source .example.env
+- unset PACKER_LOG
+- ./runtests

README.md

@@ -22,4 +22,6 @@ packer build -only=docker <(bin/yml2json < ci-minimal.yml)
 ## env config bits
 
 Most of the templates in here require some env vars.  Take a look at
-[`.example.env`](./.example.env) for an example. This repository uses a directory-based env called autoenv. https://github.com/kennethreitz/autoenv
+[`.example.env`](./.example.env) for an example.  Use of
+[autoenv](https://github.com/kennethreitz/autoenv) is encouraged but not
+required.

ci-mega.json

@@ -21,7 +21,7 @@
       "image_description": "Travis CI Mega",
       "account_file": "{{ user `gce_account_file` }}",
       "project_id": "{{ user `gce_project_id` }}",
-      "source_image": "ubuntu-1404-trusty-v20150909a",
+      "source_image": "ubuntu-1404-trusty-v20160627",
       "zone": "us-central1-a",
       "image_name": "{{ user `gce_image_name` }}",
       "machine_type": "n1-standard-4",

ci-mega.yml

@@ -26,7 +26,7 @@ builders:
   account_file: "{{ user `gce_account_file` }}"
   project_id: "{{ user `gce_project_id` }}"
   <% if @dist == 'trusty' %>
-  source_image: ubuntu-1404-trusty-v20150909a
+  source_image: ubuntu-1404-trusty-v20160627
   <% else %>
   source_image: busted-busted-busted-nope
   <% end %>

ci-minimal.json

@@ -26,7 +26,7 @@
       "image_description": "Travis CI Minimal",
       "account_file": "{{ user `gce_account_file` }}",
       "project_id": "{{ user `gce_project_id` }}",
-      "source_image": "ubuntu-1404-trusty-v20150909a",
+      "source_image": "ubuntu-1404-trusty-v20160627",
       "zone": "us-central1-a",
       "image_name": "{{ user `gce_image_name` }}",
       "machine_type": "n1-standard-4",

ci-minimal.yml

@@ -28,7 +28,7 @@ builders:
   account_file: "{{ user `gce_account_file` }}"
   project_id: "{{ user `gce_project_id` }}"
   <% if @dist == 'trusty' %>
-  source_image: ubuntu-1404-trusty-v20150909a
+  source_image: ubuntu-1404-trusty-v20160627
   <% else %>
   source_image: ubuntu-1204-precise-v20150910
   <% end %>

cookbooks/travis_internal_base/attributes/default.rb

@@ -1,5 +1,3 @@
-default['travis_internal_base']['opsmatic_disabled'] = false
-
 override['openssh']['server']['password_authentication'] = 'no'
 override['openssh']['server']['pubkey_authentication'] = 'yes'
 override['openssh']['server']['permit_root_login'] = 'no'

cookbooks/travis_internal_base/metadata.rb

@@ -10,7 +10,6 @@
 
 depends 'apt'
 depends 'openssh'
-#depends 'opsmatic'
 depends 'papertrail'
 depends 'sudo'
 depends 'users'

cookbooks/travis_internal_base/recipes/default.rb

@@ -24,13 +24,8 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-def opsmatic_disabled?
-  node['travis_internal_base']['opsmatic_disabled']
-end
-
 include_recipe 'apt'
 include_recipe 'openssh'
-include_recipe 'opsmatic::agent' unless opsmatic_disabled?
 include_recipe 'papertrail'
 include_recipe 'users'
 include_recipe 'sudo'

cookbooks/travis_internal_bastion/metadata.rb

@@ -12,6 +12,5 @@
 depends 'duo_unix'
 depends 'iptables'
 depends 'openssh'
-depends 'opsmatic'
 depends 'travis_internal_base'
 depends 'users'

cookbooks/travis_internal_bastion/recipes/default.rb

@@ -29,6 +29,5 @@
 include_recipe 'iptables'
 include_recipe 'openssh'
 include_recipe 'openssh::iptables'
-include_recipe 'opsmatic::agent'
 include_recipe 'users'
 include_recipe 'duo_unix'

internal-nat.yml

@@ -1,6 +1,10 @@
 ---
 push:
   name: travis-ci/internal-nat
+variables:
+  aws_access_key: "{{ env `AWS_ACCESS_KEY` }}"
+  aws_secret_key: "{{ env `AWS_SECRET_KEY` }}"
+  ubuntu_trusty_ami: "{{ env `UBUNTU_TRUSTY_AMI` }}"
 builders:
 - type: amazon-ebs
   access_key: "{{ user `aws_access_key` }}"

jupiter-brain.yml

@@ -8,6 +8,7 @@ variables:
   subnet_id: "{{ env `JUPITER_BRAIN_SUBNET_ID` }}"
   vpc_id: "{{ env `JUPITER_BRAIN_VPC_ID` }}"
   base_ami: "{{ env `TRAVIS_CI_BASE_AMI` }}"
+  jupiter_brain_binary: "{{ env `JUPITER_BRAIN_BINARY` }}"
 builders:
 - type: amazon-ebs
   access_key: "{{ user `aws_access_key` }}"
@@ -27,7 +28,7 @@ push:
   name: travis-ci/jupiter-brain
 provisioners:
 - type: file
-  source: "/packer/jupiter-brain/jb-server"
+  source: "{{ user `jupiter_brain_binary` }}"
   destination: "/tmp/jb-server"
 - type: file
   source: packer-assets/jupiter-brain.conf

min.yml

@@ -1,20 +1,20 @@
 ---
 description: minimal test
 variables:
-  ssh_private_key: "{{ env `SSH_PRIVATE_KEY`}}"
+  ssh_private_key: "{{ env `SSH_KEY_PATH`}}"
+  ssh_username: "{{ env `SSH_USERNAME` }}"
 builders:
 - type: "null"
   name: "null1"
   ssh_host: 127.0.0.1
-  ssh_username: carmen
+  ssh_username: "{{ user `ssh_username` }}"
   ssh_private_key_file: "{{ user `ssh_private_key`}}"
 - type: "null"
   name: "null2"
   ssh_host: 127.0.0.1
+  ssh_username: "{{ user `ssh_username` }}"
+  ssh_private_key_file: "{{ user `ssh_private_key`}}"
 
 provisioners:
 - type: shell
   inline: echo "hi"
-- type: shell
-  scripts:
-    #-foo.sh

play.yml

@@ -18,7 +18,7 @@ builders:
   image_description: Travis play
   account_file: "{{ user `gce_account_file` }}"
   project_id: "{{ user `gce_project_id` }}"
-  # source_image: ubuntu-1404-trusty-v20150909a
+  # source_image: ubuntu-1404-trusty-v20160627
   source_image: ubuntu-1204-precise-v20150910
   zone: us-central1-a
   image_name: "{{ user `gce_image_name` }}"

runtests

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 set -o errexit
+set -o pipefail
 
 unset GEM_PATH
 
@@ -12,7 +13,13 @@ done
 for f in $(git ls-files '*.yml') ; do
   echo -en "$f "
   ruby -ryaml -e "YAML.load_file('$f')"
-  echo "✓"
+  if grep -q ^builders: "$f" ; then
+    echo -en "(packer validate ...) "
+    packer validate <(bin/yml2json < "$f")
+    echo "\`--> ✓"
+  else
+    echo "✓"
+  fi
 done
 
 for f in $(git grep -l '^#!/usr/bin/env bash') ; do

worker-ec2.yml

@@ -5,7 +5,7 @@ push:
 variables:
   aws_access_key: "{{ env `AWS_ACCESS_KEY` }}"
   aws_secret_key: "{{ env `AWS_SECRET_KEY` }}"
-  base_ami: "ami-55fe133e"
+  base_ami: "{{ env `TRAVIS_WORKER_AMI` }}"
   docker_login_email: "{{ env `DOCKER_LOGIN_EMAIL` }}"
   docker_login_password: "{{ env `DOCKER_LOGIN_PASSWORD` }}"
   docker_login_server: "{{ env `DOCKER_LOGIN_SERVER` }}"
@@ -47,7 +47,7 @@ builders:
   image_description: Travis Worker
   account_file: "{{ user `gce_account_file` }}"
   project_id: "{{ user `gce_project_id` }}"
-  source_image: ubuntu-1404-trusty-v20150909a
+  source_image: ubuntu-1404-trusty-v20160627
   zone: us-central1-a
   image_name: "{{ user `gce_image_name` }}"
   machine_type: n1-standard-4

worker.json

@@ -55,7 +55,7 @@
       "image_description": "Travis Worker",
       "account_file": "{{ user `gce_account_file` }}",
       "project_id": "{{ user `gce_project_id` }}",
-      "source_image": "ubuntu-1404-trusty-v20150909a",
+      "source_image": "ubuntu-1404-trusty-v20160627",
       "zone": "us-central1-a",
       "image_name": "{{ user `gce_image_name` }}",
       "machine_type": "n1-standard-4",

worker.yml

@@ -5,7 +5,7 @@ push:
 variables:
   aws_access_key: "{{ env `AWS_ACCESS_KEY` }}"
   aws_secret_key: "{{ env `AWS_SECRET_KEY` }}"
-  base_ami: "ami-55fe133e"
+  base_ami: "{{ env `TRAVIS_WORKER_AMI` }}"
   docker_login_email: "{{ env `DOCKER_LOGIN_EMAIL` }}"
   docker_login_password: "{{ env `DOCKER_LOGIN_PASSWORD` }}"
   docker_login_server: "{{ env `DOCKER_LOGIN_SERVER` }}"
@@ -47,7 +47,7 @@ builders:
   image_description: Travis Worker
   account_file: "{{ user `gce_account_file` }}"
   project_id: "{{ user `gce_project_id` }}"
-  source_image: ubuntu-1404-trusty-v20150909a
+  source_image: ubuntu-1404-trusty-v20160627
   zone: us-central1-a
   image_name: "{{ user `gce_image_name` }}"
   machine_type: n1-standard-4