[WIP] Add registry type and url to worker payload for ECR dispatching

* fixes dev vm to run with airlocked workers

Signed-off-by: Travis Elliott Davis <edavis@chef.io>

.studiorc

@@ -11,6 +11,45 @@ export GITHUB_WEBHOOK_SECRET
 export WORKER_AUTH_TOKEN
 export SSL_CERT_FILE
 
+sup-run() {
+  mkdir -p /hab/sup/default
+  echo "--> Launching the Habitat Supervisor in the background..."
+  echo "    Running: hab sup run $*"
+  hab sup run $* > /hab/sup/default/sup.log 2>&1 &
+  echo "    * Use 'hab svc start' & 'hab svc stop' to start and stop services"
+  echo "    * Use 'sup-log' to tail the Supervisor's output (Ctrl+c to stop)"
+  echo "    * Use 'sup-term' to terminate the Supervisor"
+  if [[ -z "${HAB_STUDIO_SUP:-}" ]]; then
+    echo "    * To pass custom arguments to run the Supervisor, export"
+    echo "      'HAB_STUDIO_SUP' with the arguments before running"
+    echo "      'hab studio enter'."
+  fi
+  echo ""
+}
+
+sup-term() {
+  local pid_file="/hab/sup/default/LOCK "
+  if [ -f $pid_file ]; then
+    echo "--> Killing Habitat Supervisor running in the background..."
+    kill $(cat $pid_file) \
+      && (echo "    Supervisor killed." && rm -f $pid_file) \
+      || echo "--> Error killing Supervisor."
+  else
+    echo "--> No Launcher pid file found, Supervisor may not be running."
+  fi
+}
+
+sup-log() {
+  mkdir -p /hab/sup/default
+  touch /hab/sup/default/sup.log
+  echo "--> Tailing the Habitat Supervisor's output (use 'Ctrl+c' to stop)"
+  tail -f /hab/sup/default/sup.log
+}
+
+alias sr='sup-run'
+alias st='sup-term'
+alias sl='sup-log'
+
 init-datastore() {
   mkdir -p /hab/svc/builder-datastore
   cp -f /src/support/builder/datastore.toml /hab/svc/builder-datastore/user.toml
@@ -26,7 +65,7 @@ configure() {
 }
 
 install-packages() {
-  hab pkg install core/cacerts core/net-tools core/procps-ng core/postgresql -b -c stable
+  hab pkg install core/cacerts core/net-tools core/procps-ng core/postgresql core/shadow -b -c stable
   hab pkg install \
   core/builder-admin \
   core/builder-admin-proxy \
@@ -132,6 +171,7 @@ start-builder() {
   start-originsrv
   start-sessionsrv
   start-worker
+  sleep 2
   upload_github_keys
   generate_bldr_keys
 }
@@ -173,7 +213,8 @@ start-sessionsrv() {
 }
 
 start-worker() {
-  hab svc load core/builder-worker --bind jobsrv:builder-jobsrv.default --bind depot:builder-api-proxy.default
+  # hab svc load core/builder-worker --bind jobsrv:builder-jobsrv.default --bind depot:builder-api-proxy.default
+  hab svc load /src/results/edavis-builder-worker-6075-20171107201424-x86_64-linux.hart --bind jobsrv:builder-jobsrv.default --bind depot:builder-api-proxy.default
 }
 
 stop-builder() {
@@ -231,8 +272,10 @@ stop-worker() {
 
 generate_bldr_keys() {
   KEY_NAME=$(hab user key generate bldr | grep -Po "bldr-\d+")
-  hab file upload builder-api.default $(date +%s) "/hab/cache/keys/${KEY_NAME}.pub"
-  hab file upload builder-api.default $(date +%s) "/hab/cache/keys/${KEY_NAME}.box.key"
+  for svc in builder-api builder-jobsrv builder-worker ; do
+    hab file upload "${svc}.default" $(date +%s) "/hab/cache/keys/${KEY_NAME}.pub"
+    hab file upload "${svc}.default" $(date +%s) "/hab/cache/keys/${KEY_NAME}.box.key"
+  done
 }
 
 load_packages() {
@@ -250,6 +293,14 @@ origin() {
 }
 
 keys() {
+  if [ -f ~/.hab/cache/keys/core-20160810182414.pub ]; then
+    cat ~/.hab/cache/keys/core-20160810182414.pub | hab origin key import
+  fi
+
+  if [ -f ~/.hab/cache/keys/core-20160810182414.sig.key ]; then
+    cat ~/.hab/cache/keys/core-20160810182414.sig.key | hab origin key import
+  fi
+
   cat /hab/cache/keys/core-20160810182414.pub | \
   curl http://localhost:9636/v1/depot/origins/core/keys/20160810182414 \
     --data-binary @- \
@@ -315,6 +366,7 @@ DOCS
 }
 
 install-packages
+hab pkg exec core/shadow useradd hab || echo "User 'hab' already exists"
 
 # NOTE: these are dev secrets (not for the actual Builder services);
 # it's OK
@@ -338,8 +390,10 @@ configure
 start-builder
 fi
 
-hab pkg exec core/busybox-static addgroup krangschnak
-hab pkg exec core/busybox-static adduser krangschnak -G krangschnak -D
+touch /etc/subuid
+touch /etc/subgid
+hab pkg exec core/shadow groupadd --force krangschnak
+hab pkg exec core/shadow useradd --groups=tty --create-home -g krangschnak krangschnak || echo "User 'krangschnak' already exists"
 
 # Print out the docs
 dev_docs

Vagrantfile

@@ -2,8 +2,9 @@
 # vi: set ft=ruby :
 
 $script = <<SCRIPT
-cd /vagrant
+cd /src
 sh support/linux/install_dev_0_ubuntu_latest.sh
+echo 'eval "$(direnv hook bash)"' >> /root/.bashrc
 echo 'eval "$(direnv hook bash)"' >> /home/vagrant/.bashrc
 apt-get install -y direnv
 sh components/hab/install.sh
@@ -13,6 +14,8 @@ Vagrant.configure("2") do |config|
   config.vm.box = "bento/ubuntu-17.04"
   config.vm.provision "shell", inline: $script, privileged: true
 
+  config.vm.synced_folder ".", "/src", nfs: true, :linux__nfs_options => ["no_root_squash"], :map_uid => 0, :map_gid => 0
+  config.vm.synced_folder "~/.hab", "/root/.hab", nfs: true, :linux__nfs_options => ["no_root_squash"], :map_uid => 0, :map_gid => 0
   config.vm.synced_folder "~/.hab", "/home/vagrant/.hab", nfs: true, :linux__nfs_options => ["no_root_squash"], :map_uid => 0, :map_gid => 0
 
   config.vm.network "forwarded_port", guest: 80, host: 9636

components/builder-worker/src/runner/docker.rs

@@ -45,6 +45,8 @@ const DOCKER_HOST_ENVVAR: &'static str = "DOCKER_HOST";
 pub struct DockerExporterSpec {
     pub username: String,
     pub password: String,
+    pub registry_type: Option<String>,
+    pub registry_url: Option<String>,
     pub docker_hub_repo_name: String,
     pub latest_tag: bool,
     pub version_tag: bool,
@@ -115,6 +117,15 @@ impl<'a> DockerExporter<'a> {
         cmd.arg("--password");
         cmd.arg(&self.spec.password);
         cmd.arg("--rm-image");
+        if let Some(ref registry_url) = self.spec.registry_url {
+            cmd.arg("--registry-url");
+            cmd.arg(registry_url);
+        }
+        if let Some(ref registry_type) = self.spec.registry_type {
+            cmd.arg("--registry-type");
+            cmd.arg(registry_type);
+        }
+
         cmd.arg(self.workspace.last_built()?.path); // Locally built artifact
         debug!(
             "building docker export command, cmd={}",

components/builder-worker/src/runner/studio.rs

@@ -173,6 +173,7 @@ impl<'a> Studio<'a> {
         } else {
             let mut cmd = Command::new(&*STUDIO_PROGRAM);
             cmd.env_clear();
+            debug!("HAB_CACHE_KEY_PATH: {:?}", key_path());
             cmd.env("NO_ARTIFACT_PATH", "true"); // Disables artifact cache mounting
             cmd.env("HAB_CACHE_KEY_PATH", key_path()); // Sets key cache to build user's home
 

components/builder-worker/src/runner/util.rs

@@ -56,12 +56,12 @@ pub fn validate_integrations(workspace: &Workspace) -> Result<()> {
         }
 
         let prj_integration = prj_integrations.first().unwrap();
-        if prj_integration.get_integration() != "docker" {
-            return Err(Error::InvalidIntegrations(format!(
-                "integration '{}' not supported",
-                prj_integration.get_integration()
-            )));
-        }
+        // if prj_integration.get_integration() != "docker" {
+        //     return Err(Error::InvalidIntegrations(format!(
+        //         "integration '{}' not supported",
+        //         prj_integration.get_integration()
+        //     )));
+        // }
         if prj_integration.get_integration_name() != "default" {
             return Err(Error::InvalidIntegrations(format!(
                 "integration name '{}' not supported",
@@ -139,18 +139,19 @@ pub fn validate_integrations(workspace: &Workspace) -> Result<()> {
             )));
         }
         let org_integration = org_integrations.first().unwrap();
+        // TED TODO: Amazon will go here soon.cat
         if org_integration.get_integration() != "docker" {
             return Err(Error::InvalidIntegrations(format!(
                 "origin integration '{}' not supported",
                 org_integration.get_integration()
             )));
         }
-        if org_integration.get_name() != "docker" {
-            return Err(Error::InvalidIntegrations(format!(
-                "origin integration name '{}' not supported",
-                org_integration.get_name()
-            )));
-        }
+        // if org_integration.get_name() != "docker" {
+        //     return Err(Error::InvalidIntegrations(format!(
+        //         "origin integration name '{}' not supported",
+        //         org_integration.get_name()
+        //     )));
+        // }
         // TODO fn: use a struct and serde to do heavy lifting
         let creds: JsonValue = match serde_json::from_str(org_integration.get_body()) {
             Ok(json) => json,
@@ -218,17 +219,9 @@ pub fn docker_exporter_spec(workspace: &Workspace) -> DockerExporterSpec {
             .expect("Project integrations must not be empty")
             .get_body(),
     ).expect("Project integrations body must be JSON");
-    let custom_tag = match opts.get("custom_tag") {
-        Some(val) => {
-            let val = val.as_str().expect("custom_tag value is a string");
-            if val.is_empty() {
-                None
-            } else {
-                Some(val.to_string())
-            }
-        }
-        None => None,
-    };
+    let custom_tag = get_optional_args(&opts, String::from("custom_tag"));
+    let registry_url = get_optional_args(&creds, String::from("registry_url"));
+    let registry_type = get_optional_args(&opts, String::from("integration"));
 
     DockerExporterSpec {
         username: creds
@@ -243,6 +236,8 @@ pub fn docker_exporter_spec(workspace: &Workspace) -> DockerExporterSpec {
             .as_str()
             .expect("password value is a string")
             .to_string(),
+        registry_type: registry_type,
+        registry_url: registry_url,
         docker_hub_repo_name: opts.get("docker_hub_repo_name")
             .expect("docker_hub_repo_name key is present")
             .as_str()
@@ -263,3 +258,18 @@ pub fn docker_exporter_spec(workspace: &Workspace) -> DockerExporterSpec {
         custom_tag: custom_tag,
     }
 }
+
+fn get_optional_args(opts: &JsonValue, arg: String) -> Option<String> {
+    let is_string = format!("{} value is a string", arg);
+    match opts.get(arg) {
+        Some(key) => {
+            let key = key.as_str().expect(&is_string);
+            if key.is_empty() {
+                None
+            } else {
+                Some(key.to_string())
+            }
+        }
+        None => None,
+    }
+}

support/builder/config.sh

@@ -24,6 +24,18 @@ www_url          = "http://$APP_HOSTNAME/#/sign-in"
 
 EOT
 
+mkdir -p /hab/svc/builder-api-proxy
+cat <<EOT > /hab/svc/builder-api-proxy/user.toml
+app_url = "http://localhost:9636"
+
+[github]
+url = "$GITHUB_API_URL"
+web_url = "$GITHUB_WEB_URL"
+client_id = "$GITHUB_CLIENT_ID"
+client_secret = "$GITHUB_CLIENT_SECRET"
+app_id = 5629
+EOT
+
 mkdir -p /hab/svc/builder-jobsrv
 cat <<EOT > /hab/svc/builder-jobsrv/user.toml
 [datastore]
@@ -322,7 +334,15 @@ EOT
 
 mkdir -p /hab/svc/builder-worker
 cat <<EOT > /hab/svc/builder-worker/user.toml
-auth_token = "$GITHUB_CLIENT_SECRET"
-bldr_url = "http://$APP_HOSTNAME:9636"
+auth_token = "${HAB_AUTH_TOKEN}"
 auto_publish = true
+log_level = "debug"
+airlock_enabled = false
+
+[github]
+url = "$GITHUB_API_URL"
+web_url = "$GITHUB_WEB_URL"
+client_id = "$GITHUB_CLIENT_ID"
+client_secret = "$GITHUB_CLIENT_SECRET"
+app_id = 5629
 EOT