Working draft [fast&dirty]

CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## [0.0.1]
+
+### Added
+
+- Initial draft #2
+- Changelog #6

Makefile

@@ -1,3 +1,6 @@
+# todo: refactor
+LAST_COMMIT_HASH = $(shell git rev-parse --verify HEAD)
+
 include dev/.env
 export
 export PATH := $(shell pwd)/tmp:$(PATH)
@@ -28,20 +31,32 @@ example: custom_ca
 ifdef CI # CI is set in Github Actions
 	cd test_example; SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} vagrant up --provision
 else
-	cp -f docker/conf/certificates/*.crt test_example/docker/conf/certificates
+	if [ -f "docker/conf/certificates/*.crt" ]; then cp -f docker/conf/certificates/*.crt test_example/docker/conf/certificates; fi
 	cd test_example; SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} CUSTOM_CA=${CUSTOM_CA} ANSIBLE_ARGS='--extra-vars "local_test=true"' vagrant up --provision
 endif
 
 # clean commands
-destroy-box:
+destroy:
 	vagrant destroy -f
+	rm terraform.tfstate || true
+	rm terraform.tfstate.backup || true
+	rm example/terraform.tfstate || true
+status:
+	vagrant global-status
+format:
+	./format.sh
 
 remove-tmp:
 	rm -rf ./tmp
 
-clean: destroy-box remove-tmp
+clean: destroy remove-tmp
 
 # helper commands
 update-box:
 	@SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} vagrant box update || (echo '\n\nIf you get an SSL error you might be behind a transparent proxy. \nMore info https://github.com/fredrikhgrelland/vagrant-hashistack/blob/master/README.md#if-you-are-behind-a-transparent-proxy\n\n' && exit 2)
 
+proxy-to-minio:
+	consul connect proxy -service minio-local -upstream minio:9000 -log-level debug
+docker-local-build:
+	docker build docker -t fredrikhgrelland/hive:${LAST_COMMIT_HASH}
+	docker tag fredrikhgrelland/hive:${LAST_COMMIT_HASH} fredrikhgrelland/hive:test

README.md

@@ -1,90 +1,12 @@
-# Starter template for `fredrikhgrelland/hashistack`
+# Terraform-nomad-hive
 
-This repository can be used as a base for developing services on the hashistack.
-On github, you may use the ["Use this template"](https://github.com/fredrikhgrelland/vagrant-hashistack-template/generate) button to generate a new repository from this template.
-
-If you found this in `fredrikhgrelland/vagrant-hashistack`, you may be interested in this separate repository [vagrant-hashistack-template](https://github.com/fredrikhgrelland/vagrant-hashistack-template/generate) button to start a new repository from this repo
-.
-
-Documentation on [parent repository](https://github.com/fredrikhgrelland/vagrant-hashistack#usage).
-
-## Customizing and using the vagrant box
-
-### Building and testing docker image
-See docker [README.md](docker/README.md).
-
-### Starting a box
-The vagrant box ships with a default startup scheme. `make` from this directory will start the box, and it will run all books in [dev/ansible](dev/ansible) in lexical order (NB: `playbook.yml` is run first, but is only used to run all other playbooks) after the bootstrap-process for the hashistack is done. In the [example](test_example/dev/ansible/playbook.yml) we use it to start terraform which then starts a nomad-job.
-
-### Pre and post hashistack procedure
-You may change the hashistack configuration or add aditional pre and post steps to the startup procedure to match your needs.
-Detailed documentation in [dev/vagrant/conf/README.md](dev/vagrant/conf/README.md)
-
-### Pre packaged configuration switches
-
-The box comes standard with a set of environment switches to simplify testing of different scenarios and enable staged development efforts.
-
-NB: All lowercase variables will automatically get a corresponding TF_VAR_ prepended variant for use directly in terraform.
-To change from the default value, you may add the environment variable to [.env](dev/.env)
-
-#### Enterprise vs Open Source Software (OSS)
-As long as Enterprise is not set to `true` the box will utilise OSS version of the binaries.
-
-#### Nomad
-
-| default   | environment variable  |  value  |
-|:---------:|:----------------------|:-------:|
-|           | nomad_enterprise      |  true   |
-|     x     | nomad_enterprise      |  false  |
-|           | nomad_acl             |  true   |
-|     x     | nomad_acl             |  false  |
-
-When ACLs in Nomad are enabled the bootstrap token will be available in vault under `secret/nomad/management-token` with the two key-value pairs `accessor-id` and `secret-id`. `secret-id` is the token itself. These can be accessed in several ways:
-- From inside the vagrant box with `vault kv get secret/nomad-bootstrap-token`
-- From local machine with `vagrant ssh -c vault kv get secret/nomad-bootstrap-token"`
-- By going to vault's UI on `localhost:8200`, and signing in with the root token.
-
-#### Consul
-
-| default   | environment variable             |  value  |
-|:---------:|:---------------------------------|:-------:|
-|           | consul_enterprise                |  true   |
-|     x     | consul_enterprise                |  false  |
-|     x     | consul_acl                       |  true   |
-|           | consul_acl                       |  false  |
-|     x     | consul_acl_default_policy        |  allow  |
-|           | consul_acl_default_policy        |  deny   |
-
-#### Vault
-
-| default   | environment variable             |  value  |
-|:---------:|:---------------------------------|:-------:|
-|           | vault_enterprise                 |  true   |
-|     x     | vault_enterprise                 |  false  |
-
-##### Consul secrets engine
-
-If `consul_acl_default_policy` has value `deny`, it will also enable [consul secrets engine](https://www.vaultproject.io/docs/secrets/consul) in vault.  
-Ansible will provision additional custom roles (admin-team, dev-team), [policies](../ansible/templates/consul-policies) and tokens for test purpose with different access level.
-
-How to generate token:
 ```text
-# generate token for dev team member
-vagrant ssh -c 'vault read consul/creds/dev-team'
-
-# generate token for admin team member
-vagrant ssh -c 'vault read consul/creds/admin-team'
+make test
 ```
 
-*Tokens can be used to access UI (different access level depends on role)
+## Modes
+Hive can be run in two modes:
+- [hivemetastore](./docker/bin/hivemetastore)
+- [hiveserver](./docker/bin/hiveserver)
 
-## Vagrant box life-cycle
-1. `/home/vagrant/.env_default` - _preloaded_ - default variables
-1. `vagrant/.env` - _user provided_ - variables override
-1. `vagrant/.env_override` - _system provided_ - variables are overridden for test purposes
-1. `vagrant/dev/vagrant/conf/pre_ansible.sh` - _user provided_ - script running before ansible bootstrap procedure
-1. `vagrant/dev/vagrant/conf/pre_bootstrap/*.yml` - _user provided_ - pre bootstrap tasks, running before hashistack software runs and ready
-1. `/etc/ansible/bootstrap.yml` - _preloaded_ - verify ansible variables and software configuration, run hashistack software & verify that it started correctly
-1. `vagrant/conf/post_bootstrap/*.yml` - _user provided_ - poststart scripts, running after hasistack software runs and ready
-1. `vagrant/dev/conf/pre_ansible.sh` - _user provided_ - script running after ansible bootstrap procedure
-1. `vagrant/ansible/*.yml` - _user provided_ - ansible tasks included in playbook
+`NB!` current implementation supports only [`hivemetastore`](./conf/nomad/hive.hcl#L99)

Vagrantfile

@@ -3,10 +3,11 @@ Vagrant.configure("2") do |config|
     config.vm.box_version = ">= 0.3, < 0.4"
     config.vm.provider "virtualbox" do |vb|
         vb.linked_clone = true
-        vb.memory = 2048
+        vb.memory = 4096
+        vb.cpus = 2
+    end
+    config.vm.provision "ansible_local" do |ansible|
+        ansible.provisioning_path = "/vagrant/dev/ansible"
+        ansible.playbook = "playbook.yml" # Note this playbook is, in this context, /ansible/playbook.yml
     end
-        config.vm.provision "ansible_local" do |ansible|
-            ansible.provisioning_path = "/vagrant/dev/ansible"
-            ansible.playbook = "playbook.yml" # Note this playbook is, in this context, /ansible/playbook.yml
-        end
 end

conf/nomad/hive.hcl

@@ -0,0 +1,145 @@
+job "${service_name}" {
+
+  type = "service"
+  datacenters = "${datacenters}"
+  namespace = "${namespace}"
+
+  update {
+    max_parallel = 1
+    health_check = "checks"
+    min_healthy_time = "10s"
+    healthy_deadline = "10m"
+    progress_deadline = "12m"
+    auto_revert = true
+    auto_promote = true
+    canary = 1
+    stagger = "30s"
+  }
+
+  group "metastoreserver" {
+    count = 1
+
+    service {
+      name = "${service_name}"
+      port = "${port}"
+
+      check {
+        name = "beeline"
+        type = "script"
+        task = "metastoreserver"
+        command = "/bin/bash"
+        args = [
+          "-c",
+          "beeline -u jdbc:hive2:// -e \"SHOW DATABASES;\" &> /tmp/check_script_beeline_metastoreserver && echo \"return code $?\""]
+        interval = "30s"
+        timeout = "120s"
+      }
+
+      connect {
+        sidecar_service {
+          proxy {
+            upstreams {
+              destination_name  = "${postgres_service_name}"
+              local_bind_port   = "${postgres_local_bind_port}"
+            }
+            upstreams {
+              destination_name  = "${minio_service_name}"
+              local_bind_port   = "${minio_local_bind_port}"
+            }
+          }
+        }
+      }
+    }
+
+    network {
+      mode = "bridge"
+    }
+
+    task "waitfor-hive-database" {
+      restart {
+        attempts = 5
+        delay    = "15s"
+      }
+      lifecycle {
+        hook = "prestart"
+      }
+      driver = "docker"
+      resources {
+        memory = 32
+      }
+      config {
+        image = "consul:latest"
+        entrypoint = ["/bin/sh"]
+        args = ["-c", "jq </local/service.json -e '.[].Status|select(. == \"passing\")'"]
+        volumes = ["tmp/service.json:/local/service.json" ]
+      }
+      template {
+        destination = "tmp/service.json"
+        data = <<EOH
+          {{- service "${postgres_service_name}" | toJSON -}}
+        EOH
+      }
+    }
+
+    task "metastoreserver" {
+      driver = "docker"
+# todo: optional rendering START (docker image may be already available in registry)
+      artifact {
+        // vagrant-hashistack:minio
+        source = "s3::http://127.0.0.1:9000/dev/tmp/hive_local.tar"
+        options {
+          aws_access_key_id     = "minioadmin"
+          aws_access_key_secret = "minioadmin"
+        }
+      }
+
+      config {
+        load = "hive_local.tar"
+        image = "fredrikhgrelland/hive:local"
+        command = "hivemetastore"
+      }
+# todo: optional rendering END
+      resources {
+        cpu = 500
+        memory = 1024
+      }
+
+      logs {
+        max_files = 10
+        max_file_size = 2
+      }
+
+      template {
+        destination = "local/config.env"
+        env = true
+        data = <<EOH
+HIVE_SITE_CONF_javax_jdo_option_ConnectionURL="jdbc:postgresql://{{ env "NOMAD_UPSTREAM_ADDR_${postgres_service_name}" }}/${postgres_database_name}"
+HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName="org.postgresql.Driver"
+HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
+HIVE_SITE_CONF_hive_metastore_uris="thrift://127.0.0.1:9083"
+HIVE_SITE_CONF_hive_metastore_schema_verification=true
+HIVE_SITE_CONF_hive_execution_engine="mr"
+HIVE_SITE_CONF_hive_support_concurrency=false
+HIVE_SITE_CONF_hive_driver_parallel_compilation=true
+HIVE_SITE_CONF_hive_metastore_warehouse_dir="s3a://${hive_bucket}/warehouse"
+HIVE_SITE_CONF_hive_metastore_event_db_notification_api_auth=false
+CORE_CONF_fs_defaultFS = "s3a://${default_bucket}"
+CORE_CONF_fs_s3a_connection_ssl_enabled = false
+CORE_CONF_fs_s3a_endpoint = "http://{{ env "NOMAD_UPSTREAM_ADDR_${minio_service_name}" }}"
+CORE_CONF_fs_s3a_path_style_access = true
+EOH
+      }
+
+      template {
+        destination = "secrets/.env"
+        env = true
+        data = <<EOH
+CORE_CONF_fs_s3a_access_key = "${minio_access_key}"
+CORE_CONF_fs_s3a_secret_key = "${minio_secret_key}"
+HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName="${postgres_username}"
+HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword="${postgres_password}"
+EOH
+      }
+    }
+  }
+}

dev/ansible/01_build_docker_image.yml

@@ -0,0 +1,32 @@
+- name: Remove docker image
+  docker_image:
+    name: docker_image
+    tag: local
+    force_absent: true
+    state: absent
+
+- name: Build docker image
+  docker_image:
+    name: fredrikhgrelland/hive
+    tag: local
+    build:
+      path: /vagrant/docker
+      pull: false
+      args:
+        TEST_DOWNLOAD_BUILD_ARGUMENT: test-arg
+    source: build
+
+- name: Create tmp if it does not exist
+  file:
+    path: /vagrant/dev/tmp
+    state: directory
+    mode: '0755'
+    owner: vagrant
+    group: vagrant
+
+- name: Archive docker image
+  docker_image:
+    name: fredrikhgrelland/hive
+    tag: local
+    archive_path: /vagrant/dev/tmp/hive_local.tar
+    source: local

dev/ansible/02_run_terraform.yml

@@ -0,0 +1,10 @@
+- name: Terraform
+  terraform:
+    project_path: ../../example
+    force_init: true
+    state: present
+  register: terraform
+
+- name: Terraform stdout
+  debug:
+    msg: "{{terraform.stdout}}"