RackHD support (#698)

* Setup basic structure

* Setup basic structure

* Create an interface & mocks for RackHD monorail client

Interface and mocks may move into gorackhd project once
proven out in this project.

* Find compute node to provision

* Create README.md

* Create LICENSE

* Add Login() to monorail client

* Switch to SKUs and add auth

* Refactor and add workflow options

* Exclude build contents, but keep build dir

* Add instance destroy support

* Switch to Ginkgo tests

* Rename some specs

* Fixes and clean up

* Add label and describe support

* Lock node for Infrakit when provisioning

* Add spec validation to plugin

* Add Vagrant demo to project

* Update to latest infrakit spec

* new logo

create a new logo to have before DockerCon showcase. Also added a small blurb at the beginning to throw some more marketing magic on it.

Signed-off-by: Kendrick Coleman <kendrickcoleman@gmail.com>

* Remove inadvertently added vagrant files

* Update to 2.x RackHD API

* Update demo

* Improve provisioning, destruction and descriptions

  * provisioning and destructions are now prevented from nodes with
  active workflows
  * descriptions now include node IP addresses as learned from Ohai node
  catalog
  * Some cleanup/refactoring

  TODO: update tests to match

* Reorganize files before import into infrakit repo

Signed-off-by: David Chung <david.chung@docker.com>

* Add vendored sources

Signed-off-by: David Chung <david.chung@docker.com>

* Code fixes to get rackhd to compile

Signed-off-by: David Chung <david.chung@docker.com>

* Driver to include rackhd as built-in; disable test temporarily

Signed-off-by: David Chung <david.chung@docker.com>

* fix lint

Signed-off-by: David Chung <david.chung@docker.com>

.gitignore

@@ -8,7 +8,10 @@ coverage.txt
 .tags
 
 # Build binaries
-build/
+build/*
+
+# Demo files
+demo/*
 
 # Vim temporary files.
 .*.swp

circle.yml

@@ -11,7 +11,7 @@ machine:
     GOPATH: "$HOME/.go_workspace"
     WORKDIR: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME"
     E2E_CLEANUP: "false"
-    SKIP_TESTS: "docker,etcd,terraform,flaky"
+    SKIP_TESTS: "docker,etcd,terraform,rackhd,flaky"
     S3_MINIO: "s3bucket"
     S3_BUCKET: "s3bucket/infrakit/build/linux-amd64"
     S3_BUILD_RELEASE: "infrakit-$CIRCLE_BRANCH.tar.gz"

cmd/infrakit/main.go

@@ -49,6 +49,7 @@ import (
 	_ "github.com/docker/infrakit/pkg/run/v0/ingress"
 	_ "github.com/docker/infrakit/pkg/run/v0/kubernetes"
 	_ "github.com/docker/infrakit/pkg/run/v0/manager"
+	_ "github.com/docker/infrakit/pkg/run/v0/rackhd"
 	_ "github.com/docker/infrakit/pkg/run/v0/selector"
 	_ "github.com/docker/infrakit/pkg/run/v0/simulator"
 	_ "github.com/docker/infrakit/pkg/run/v0/swarm"

pkg/provider/codedellemc/rackhd/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 {code} by Dell EMC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pkg/provider/codedellemc/rackhd/Makefile

@@ -0,0 +1,130 @@
+# REPO
+REPO?=github.com/codedellemc/infrakit.rackhd
+
+# Set an output prefix, which is the local directory if not specified
+PREFIX?=$(shell pwd -L)
+
+# Used to populate version variable in main package.
+VERSION?=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always)
+REVISION?=$(shell git rev-list -1 HEAD)
+
+# Allow turning off function inlining and variable registerization
+ifeq (${DISABLE_OPTIMIZATION},true)
+	GO_GCFLAGS=-gcflags "-N -l"
+	VERSION:="$(VERSION)-noopt"
+endif
+
+.PHONY: clean all fmt vet lint build test containers get-tools
+.DEFAULT: all
+all: clean fmt vet lint build test
+
+ci: fmt vet lint coverage
+
+AUTHORS: .mailmap .git/HEAD
+	 git log --format='%aN <%aE>' | sort -fu > $@
+
+# Package list
+PKGS_AND_MOCKS := $(shell go list ./... | grep -v ^${REPO}/vendor/)
+PKGS := $(shell echo $(PKGS_AND_MOCKS) | tr ' ' '\n' | grep -v /mock$)
+
+vet:
+	@echo "+ $@"
+	@go vet $(PKGS)
+
+fmt:
+	@echo "+ $@"
+	@test -z "$$(gofmt -s -l . 2>&1 | grep -v ^vendor/ | tee /dev/stderr)" || \
+		(echo >&2 "+ please format Go code with 'gofmt -s', or use 'make fmt-save'" && false)
+
+fmt-save:
+	@echo "+ $@"
+	@gofmt -s -l . 2>&1 | grep -v ^vendor/ | xargs gofmt -s -l -w
+
+lint:
+	@echo "+ $@"
+	$(if $(shell which golint || echo ''), , \
+		$(error Please install golint: `go get -u github.com/golang/lint/golint`))
+	@test -z "$$(golint ./... 2>&1 | grep -v ^vendor/ | grep -v mock/ | tee /dev/stderr)"
+
+build:
+	@echo "+ $@"
+	@go build ${GO_LDFLAGS} $(PKGS)
+
+clean:
+	@echo "+ $@"
+	rm -rf build
+	mkdir -p build
+
+define build_binary
+	go build -o build/$(1) \
+		-ldflags "-X github.com/codedellemc/infrakit.rackhd/plugin.Version=$(VERSION) -X github.com/codedellemc/infrakit.rackhd/plugin.Revision=$(REVISION)" $(2)
+endef
+binaries: clean build-binaries
+build-binaries:
+	@echo "+ $@"
+ifneq (,$(findstring .m,$(VERSION)))
+	@echo "\nWARNING - repository contains uncommitted changes, tagging binaries as dirty\n"
+endif
+
+	$(call build_binary,infrakit-instance-rackhd,github.com/codedellemc/infrakit.rackhd/plugin/instance/cmd)
+
+
+install:
+	@echo "+ $@"
+	@go install ${GO_LDFLAGS} $(PKGS)
+
+generate:
+	@echo "+ $@"
+	@go generate -x $(PKGS_AND_MOCKS)
+
+test:
+	@echo "+ $@"
+	@go test -test.short -race -v $(PKGS)
+
+coverage:
+	@echo "+ $@"
+	@for pkg in $(PKGS); do \
+	  go test -test.short -coverprofile="../../../$$pkg/coverage.txt" $${pkg} || exit 1; \
+	done
+
+test-full:
+	@echo "+ $@"
+	@go test -race $(PKGS)
+
+get-tools:
+	@echo "+ $@"
+	@go get -u \
+		github.com/golang/lint/golint \
+		github.com/wfarner/blockcheck \
+		github.com/rancher/trash
+
+# Current working environment.  Set these explicitly if you want to cross-compile
+# in the build container (see the build-in-container target):
+GOOS?=$(shell go env GOOS)
+GOARCH?=$(shell go env GOARCH)
+DOCKER_BUILD_FLAGS?=--no-cache --pull
+build-in-container:
+	@echo "+ $@"
+	@docker build ${DOCKER_BUILD_FLAGS} -t infrakit-build -f ${CURDIR}/dockerfiles/Dockerfile.build .
+	@docker run --rm \
+		-e GOOS=${GOOS} -e GOARCCH=${GOARCH} -e DOCKER_CLIENT_VERSION=${DOCKER_CLIENT_VERSION} \
+		-v ${CURDIR}/build:/go/src/${REPO}/build \
+		infrakit-build
+
+# For packaging as Docker container images.  Set the environment variables DOCKER_PUSH, DOCKER_TAG_LATEST
+# if also push to remote repo.  You must have access to the remote repo.
+DOCKER_IMAGE?=infrakit/rackhd
+DOCKER_TAG?=dev
+build-docker:
+	@echo "+ $@"
+	GOOS=linux GOARCH=amd64 make build-in-container
+	@docker build ${DOCKER_BUILD_FLAGS} \
+	-t ${DOCKER_IMAGE}:${DOCKER_TAG} \
+	-f ${CURDIR}/dockerfiles/Dockerfile.bundle .
+ifeq (${DOCKER_PUSH},true)
+	@docker push ${DOCKER_IMAGE}:${DOCKER_TAG}
+ifeq (${DOCKER_TAG_LATEST},true)
+	@docker tag ${DOCKER_IMAGE}:${DOCKER_TAG} ${DOCKER_IMAGE}:latest
+	@docker push ${DOCKER_IMAGE}:latest
+endif
+endif

pkg/provider/codedellemc/rackhd/README.md

@@ -0,0 +1,29 @@
+# InfraKit.RackHD
+
+**Docker for Private Clouds**
+
+The first and only open source toolkit for creating and managing declarative, self-healing, and platform-agnostic infrastructure in your data center. 
+
+
+[InfraKit](https://github.com/docker/infrakit) plugins for creating and managing resources in [RackHD](https://github.com/RackHD/RackHD).
+
+![logo](img/rackhd-infrakit-logo.png "Logo")
+
+## Instance plugin
+
+An InfraKit instance plugin is provided, which runs RackHD workflows to provision compute nodes.
+
+### Building and running
+
+To build the RackHD Instance plugin, run `make binaries`.  The plugin binary will be located at
+`./build/infrakit-instance-rackhd`.
+
+### Example
+
+TODO
+
+## Licensing
+infrakit.rackhd is freely distributed under the [MIT License](http://codedellemc.github.io/sampledocs/LICENSE "LICENSE"). See LICENSE for details.
+
+##Support
+Please file bugs and issues on the Github issues page for this project. This is to help keep track and document everything related to this repo. For general discussions and further support you can join the [{code} by Dell EMC Community slack team](http://community.codedellemc.com/) and join the **#rackhd** channel. The code and documentation are released with no warranties or SLAs and are intended to be supported through a community driven process.

pkg/provider/codedellemc/rackhd/demo/Vagrantfile

@@ -0,0 +1,60 @@
+# vim:set ft=ruby
+Vagrant.configure("2") do |config|
+    config.vm.box = "rackhd/rackhd"
+    config.vm.box_version = "2.1.0"
+    config.vm.provider "virtualbox" do |v|
+        v.memory = 4096
+        v.cpus = 4
+        v.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+    end
+
+    config.vm.network "private_network", ip: "172.31.128.1", virtualbox__intnet: "closednet", auto_config: false, nic_type: "82540EM"
+    config.vm.network "forwarded_port", guest: 8080, host: 9090
+    config.vm.network "forwarded_port", guest: 5672, host: 9091
+    config.vm.network "forwarded_port", guest: 9080, host: 9092
+    config.vm.network "forwarded_port", guest: 8443, host: 9093
+    config.vm.network "forwarded_port", guest: 7080, host: 6080
+
+    # If true, then any SSH connections made will enable agent forwarding.
+    # Default value: false
+
+    config.ssh.forward_agent = true
+
+    # Enable admin user
+    config.vm.provision "shell", inline: <<-EOS
+      /usr/bin/curl -sk -H "Content-Type: application/json" -X POST https://localhost:8443/api/current/users -d '{"username": "admin", "password": "admin123", "role": "Administrator"}'
+      if [ ! -f on-tools ]
+      then
+        git clone https://github.com/RackHD/on-tools.git
+      fi
+      if [ ! -f /var/mirrors/Centos/7.0 ]
+      then
+        sudo python ./on-tools/scripts/setup_iso.py /vagrant/CentOS-7-x86_64-DVD-1611.iso /var/mirrors --link /var/renasar
+      fi
+      sudo ln -s /var/mirrors/Centos /var/renasar/on-http/static/http/centos
+      sudo mkdir /var/mirrors/Centos/7.0/os
+      sudo ln -s /var/mirrors/Centos/7.0 /var/mirors/Centos/7.0/os/x86_64
+      sudo ln -s /var/mirrors/Centos /var/renasar/on-http/static/http/centos
+      sudo perl -pi -e 's|monorail-undionly.kpxe|monorail.ipxe|g' /var/renasar/on-dhcp-proxy/lib/message-handler.js
+      sudo service on-dhcp-proxy restart
+    EOS
+
+    # Install Docker Infrakit
+    # config.vm.provision "shell",
+    #   env: {
+    #     "GOROOT" => "/home/vagrant/go",
+    #     "GOPATH" => "/home/vagrant/gocode"
+    #   },
+    #   inline: <<-EOS
+    #     mkdir /home/vagrant/go
+    #     mkdir /home/vagrant/gocode
+    #     /usr/bin/apt-get install -y gccgo-go
+    #     go get gofmt
+    #     go get github.com/docker/infrakit
+    #     go get github.com/codedellemc/infrakit.rackhd
+    #     cd infrakit
+    #     make binaries
+    #     build/infrakit-group-default &
+    #     build/infrakit-flavor-vanilla &
+    #   EOS
+end

pkg/provider/codedellemc/rackhd/demo/demo.bash

@@ -0,0 +1,12 @@
+vagrant up
+
+if [ ! -f ./CentOS-7-x86_64-DVD-1611.iso ]
+then
+    wget http://mirrors.usc.edu/pub/linux/distributions/centos/7/isos/x86_64/CentOS-7-x86_64-DVD-1611.iso
+fi
+
+export TOKEN=`curl -sk -H "Content-Type: application/json" -X POST -d '{"username": "admin", "password": "admin123"}' https://localhost:9093/login | jq '.["token"]' | sed 's|"||g'`
+
+echo "Create VirtualBox SKU in RackHD"
+
+curl -sk -H "Authorization: JWT $TOKEN" -H "Content-Type: application/json" -X POST -d '{ "name": "VirtualBox", "rules": [ { "path": "dmi.Base Board Information.Product Name", "equals": "VirtualBox" } ] }' https://localhost:9093/api/current/skus

pkg/provider/codedellemc/rackhd/demo/rackhd-instance.json

@@ -0,0 +1,18 @@
+{
+  "Properties": {
+    "Workflow": {
+      "name": "Graph.InstallCentOS",
+      "options": {
+        "install-os": {
+          "version": "7.0",
+          "repo": "{{file.server}}/Centos/7.0",
+          "rootPassword": "root"
+        }
+      }
+    },
+    "SKUName": "vQuanta D51 SKU"
+  },
+  "Tags": {
+    "Name": "infrakit-example"
+  }
+}

pkg/provider/codedellemc/rackhd/demo/rackhd-vanilla.json

@@ -0,0 +1,39 @@
+{
+    "ID": "rackhd-example",
+    "Properties": {
+        "Allocation": {
+            "Size": 1
+        },
+        "Instance": {
+            "Plugin": "rackhd",
+            "Properties": {
+                "Workflow": {
+                    "name": "Graph.InstallCentOS",
+                    "options": {
+                        "defaults": {
+                            "version": "7.0",
+                            "rootPassword": "root"
+                        }
+                    }
+                },
+                "SKUName": "VirtualBox"
+            },
+            "Tags": {
+                "Name": "infrakit-example"
+            }
+        },
+        "Flavor": {
+            "Plugin": "flavor-vanilla",
+            "Init": [
+                "sudo yum install -y nginx",
+                "sudo service nginx start"
+            ],
+            "Properties": {
+                "Tags": {
+                    "tier": "web",
+                    "project": "infrakit"
+                }
+            }
+        }
+    }
+}

pkg/provider/codedellemc/rackhd/plugin/instance/builder.go

@@ -0,0 +1,34 @@
+package instance
+
+import (
+	"github.com/docker/infrakit/pkg/spi/instance"
+	"github.com/spf13/pflag"
+	"github.com/spiegela/gorackhd/monorail"
+)
+
+// Options contain parameters required to connect to RackHD
+type Options struct {
+	Endpoint string
+	Username string
+	Password string
+}
+
+// Builder is a ProvisionerBuilder that creates a RackHD instance provisioner
+type Builder struct {
+	options Options
+}
+
+// Flags returns the flags required.
+func (b *Builder) Flags() *pflag.FlagSet {
+	flags := pflag.NewFlagSet("rackhd", pflag.PanicOnError)
+	flags.StringVar(&b.options.Endpoint, "endpoint", "http://localhost:9090", "RackHD API Endpoint")
+	flags.StringVar(&b.options.Username, "username", "admin", "RackHD Username")
+	flags.StringVar(&b.options.Password, "password", "admin123", "RackHD Password")
+	return flags
+}
+
+// BuildInstancePlugin creates an instance Provisioner configured with the Flags.
+func (b *Builder) BuildInstancePlugin() (instance.Plugin, error) {
+	mc := monorail.New(b.options.Endpoint)
+	return NewInstancePlugin(mc, b.options.Username, b.options.Password), nil
+}