diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 8b63e1b5bea..0b0f024cfbe 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -20,6 +20,7 @@ import ( _ "github.com/hashicorp/nomad/e2e/metrics" _ "github.com/hashicorp/nomad/e2e/nomad09upgrade" _ "github.com/hashicorp/nomad/e2e/nomadexec" + _ "github.com/hashicorp/nomad/e2e/podman" _ "github.com/hashicorp/nomad/e2e/spread" _ "github.com/hashicorp/nomad/e2e/systemsched" _ "github.com/hashicorp/nomad/e2e/taskevents" diff --git a/e2e/podman/input/redis.nomad b/e2e/podman/input/redis.nomad new file mode 100644 index 00000000000..0e141f34761 --- /dev/null +++ b/e2e/podman/input/redis.nomad @@ -0,0 +1,28 @@ +job "redis" { + datacenters = ["dc1"] + type = "service" + + group "redis" { + task "redis" { + driver = "podman" + + config { + image = "docker://redis" + + port_map { + redis = 6379 + } + } + + resources { + cpu = 500 + memory = 256 + + network { + mbits = 20 + port "redis" {} + } + } + } + } +} diff --git a/e2e/podman/podman.go b/e2e/podman/podman.go new file mode 100644 index 00000000000..95cf1b49f18 --- /dev/null +++ b/e2e/podman/podman.go @@ -0,0 +1,78 @@ +package podman + +import ( + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/framework" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/stretchr/testify/require" +) + +type PodmanTest struct { + framework.TC + jobIDs []string +} + +func init() { + framework.AddSuites(&framework.TestSuite{ + Component: "Podman", + CanRunLocal: true, + Cases: []framework.TestCase{ + new(PodmanTest), + }, + }) +} + +func (tc *PodmanTest) BeforeAll(f *framework.F) { + e2eutil.WaitForLeader(f.T(), tc.Nomad()) + e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 2) +} + +func (tc *PodmanTest) TestRedisDeployment(f *framework.F) { + t := f.T() + nomadClient := tc.Nomad() + uuid := uuid.Generate() + jobID := "deployment" + uuid[0:8] + tc.jobIDs = append(tc.jobIDs, jobID) + e2eutil.RegisterAndWaitForAllocs(t, nomadClient, "podman/input/redis.nomad", jobID, "") + ds := e2eutil.DeploymentsForJob(t, nomadClient, jobID) + require.Equal(t, 1, len(ds)) + + jobs := nomadClient.Jobs() + allocs, _, err := jobs.Allocations(jobID, true, nil) + require.NoError(t, err) + + var allocIDs []string + for _, alloc := range allocs { + allocIDs = append(allocIDs, alloc.ID) + } + + // Wait for allocations to get past initial pending state + e2eutil.WaitForAllocsNotPending(t, nomadClient, allocIDs) + + jobs = nomadClient.Jobs() + allocs, _, err = jobs.Allocations(jobID, true, nil) + require.NoError(t, err) + + require.Len(t, allocs, 1) + require.Equal(t, allocs[0].ClientStatus, "running") +} + +func (tc *PodmanTest) AfterEach(f *framework.F) { + nomadClient := tc.Nomad() + + // Mark all nodes eligible + nodesAPI := tc.Nomad().Nodes() + nodes, _, _ := nodesAPI.List(nil) + for _, node := range nodes { + nodesAPI.ToggleEligibility(node.ID, true, nil) + } + + jobs := nomadClient.Jobs() + // Stop all jobs in test + for _, id := range tc.jobIDs { + jobs.Deregister(id, true, nil) + } + tc.jobIDs = []string{} + // Garbage collect + nomadClient.System().GarbageCollect() +} diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf index 2530c784e29..df2d4658b6c 100644 --- a/e2e/terraform/main.tf +++ b/e2e/terraform/main.tf @@ -136,9 +136,9 @@ ssh into nodes with: ssh -i keys/${local.random_name}.pem ubuntu@${aws_instance.server[0].public_ip} # clients -%{ for ip in aws_instance.client_linux.*.public_ip ~} +%{for ip in aws_instance.client_linux.*.public_ip~} ssh -i keys/${local.random_name}.pem ubuntu@${ip} -%{ endfor ~} +%{endfor~} ``` EOM diff --git a/e2e/terraform/packer/linux/setup.sh b/e2e/terraform/packer/linux/setup.sh index 1a5d870963d..8bf320e17d9 100755 --- a/e2e/terraform/packer/linux/setup.sh +++ b/e2e/terraform/packer/linux/setup.sh @@ -4,17 +4,17 @@ set -e # Disable interactive apt prompts export DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + sudo mkdir -p /ops/shared sudo chown -R ubuntu:ubuntu /ops/shared - cd /ops CONSULVERSION=1.7.3 CONSULDOWNLOAD=https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip CONSULCONFIGDIR=/etc/consul.d CONSULDIR=/opt/consul - VAULTVERSION=1.1.1 VAULTDOWNLOAD=https://releases.hashicorp.com/vault/${VAULTVERSION}/vault_${VAULTVERSION}_linux_amd64.zip VAULTCONFIGDIR=/etc/vault.d @@ -25,15 +25,15 @@ NOMADVERSION=0.9.1 NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMADVERSION}_linux_amd64.zip NOMADCONFIGDIR=/etc/nomad.d NOMADDIR=/opt/nomad +NOMADPLUGINDIR=/opt/nomad/plugins # Dependencies sudo apt-get install -y software-properties-common sudo apt-get update -sudo apt-get install -y unzip tree redis-tools jq curl tmux awscli nfs-common +sudo apt-get install -y dnsmasq unzip tree redis-tools jq curl tmux awscli nfs-common # Numpy (for Spark) -sudo apt-get install -y python-setuptools -sudo easy_install pip +sudo apt-get install -y python-setuptools python-pip sudo pip install numpy # Install sockaddr @@ -80,6 +80,8 @@ sudo mkdir -p $NOMADCONFIGDIR sudo chmod 755 $NOMADCONFIGDIR sudo mkdir -p $NOMADDIR sudo chmod 755 $NOMADDIR +sudo mkdir -p $NOMADPLUGINDIR +sudo chmod 755 $NOMADPLUGINDIR echo "Install Docker" distro=$(lsb_release -si | tr '[:upper:]' '[:lower:]') @@ -108,6 +110,48 @@ sudo mkdir -p "$HADOOPCONFIGDIR" wget -O - http://apache.mirror.iphh.net/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | sudo tar xz -C /usr/local/ +echo "Install Podman" +. /etc/os-release +sudo sh -c "echo 'deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" +curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key | sudo apt-key add - +sudo apt-get update -qq +sudo apt-get -qq -y install podman + +# get catatonit (to check podman --init switch) +cd /tmp +wget https://github.com/openSUSE/catatonit/releases/download/v0.1.4/catatonit.x86_64 +mkdir -p /usr/libexec/podman +sudo mv catatonit* /usr/libexec/podman/catatonit +sudo chmod +x /usr/libexec/podman/catatonit + +echo "Install podman task driver" +# install nomad-podman-driver and move to plugin dir +wget -P /tmp https://github.com/pascomnet/nomad-driver-podman/releases/download/v0.0.3/nomad-driver-podman_linux_amd64.tar.gz +sudo tar -xf /tmp/nomad-driver-podman_linux_amd64.tar.gz -C /tmp +sudo mv /tmp/nomad-driver-podman/nomad-driver-podman $NOMADPLUGINDIR +sudo chmod +x $NOMADPLUGINDIR/nomad-driver-podman + +# disable systemd-resolved and configure dnsmasq +# to forward local requests to consul +sudo systemctl disable systemd-resolved.service +sudo rm /etc/resolv.conf +echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf +echo ' +port=53 +resolv-file=/var/run/dnsmasq/resolv.conf +bind-interfaces +listen-address=127.0.0.1 +server=/consul/127.0.0.1#8600 +' | sudo tee /etc/dnsmasq.d/default + +# add our hostname to etc/hosts +echo "127.0.0.1 $(hostname)" | sudo tee -a /etc/hosts +sudo systemctl restart dnsmasq + +# enable cgroup_memory and swap +sudo sed -i 's/GRUB_CMDLINE_LINUX="[^"]*/& cgroup_enable=memory swapaccount=1/' /etc/default/grub +sudo update-grub + # note this 'EOF' syntax avoids expansion in the heredoc sudo tee "$HADOOPCONFIGDIR/core-site.xml" << 'EOF' diff --git a/e2e/terraform/packer/packer.json b/e2e/terraform/packer/packer.json index 5232e51115b..7574db17f97 100644 --- a/e2e/terraform/packer/packer.json +++ b/e2e/terraform/packer/packer.json @@ -3,7 +3,7 @@ { "type": "amazon-ebs", "region": "us-east-1", - "source_ami": "ami-80861296", + "source_ami": "ami-7ad76705", "instance_type": "t2.medium", "ssh_username": "ubuntu", "iam_instance_profile": "packer-builder", diff --git a/e2e/terraform/shared/config/provision-client.sh b/e2e/terraform/shared/config/provision-client.sh index 6afdde20845..011eaa304bd 100755 --- a/e2e/terraform/shared/config/provision-client.sh +++ b/e2e/terraform/shared/config/provision-client.sh @@ -52,9 +52,42 @@ wget -q -O - \ https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz \ | sudo tar -C /opt/cni/bin -xz +# enable varlink socket (not included in ubuntu package) +cat > /etc/systemd/system/io.podman.service << EOF +[Unit] +Description=Podman Remote API Service +Requires=io.podman.socket +After=io.podman.socket +Documentation=man:podman-varlink(1) + +[Service] +Type=simple +ExecStart=/usr/bin/podman varlink unix:%t/podman/io.podman --timeout=60000 +TimeoutStopSec=30 +KillMode=process + +[Install] +WantedBy=multi-user.target +Also=io.podman.socket +EOF + +cat > /etc/systemd/system/io.podman.socket << EOF +[Unit] +Description=Podman Remote API Socket +Documentation=man:podman-varlink(1) https://podman.io/blogs/2019/01/16/podman-varlink.html + +[Socket] +ListenStream=%t/podman/io.podman +SocketMode=0600 + +[Install] +WantedBy=sockets.target +EOF + # enable as a systemd service sudo cp "$NOMAD_SRC/nomad.service" /etc/systemd/system/nomad.service sudo systemctl enable nomad.service sudo systemctl daemon-reload +sudo systemctl start io.podman sudo systemctl restart nomad.service diff --git a/e2e/terraform/shared/consul/consul_aws.service b/e2e/terraform/shared/consul/consul_aws.service index 02931a65aae..4ce8ac8a356 100644 --- a/e2e/terraform/shared/consul/consul_aws.service +++ b/e2e/terraform/shared/consul/consul_aws.service @@ -6,7 +6,7 @@ After=network-online.target [Service] Restart=on-failure Environment=CONSUL_ALLOW_PRIVILEGED_PORTS=true -ExecStart=/usr/local/bin/consul agent -config-dir="/etc/consul.d" -dns-port="53" -recursor="172.31.0.2" +ExecStart=/usr/local/bin/consul agent -config-dir="/etc/consul.d" -recursor="172.31.0.2" ExecReload=/bin/kill -HUP $MAINPID KillSignal=SIGTERM User=root diff --git a/e2e/terraform/shared/consul/consul_azure.service b/e2e/terraform/shared/consul/consul_azure.service index 3f9b24103cf..252fba77de7 100644 --- a/e2e/terraform/shared/consul/consul_azure.service +++ b/e2e/terraform/shared/consul/consul_azure.service @@ -6,7 +6,7 @@ After=network-online.target [Service] Restart=on-failure Environment=CONSUL_ALLOW_PRIVILEGED_PORTS=true -ExecStart=/usr/local/bin/consul agent -config-dir="/etc/consul.d" -dns-port="53" -recursor="168.63.129.16" +ExecStart=/usr/local/bin/consul agent -config-dir="/etc/consul.d" -recursor="168.63.129.16" ExecReload=/bin/kill -HUP $MAINPID KillSignal=SIGTERM User=root diff --git a/e2e/terraform/shared/nomad/client.hcl b/e2e/terraform/shared/nomad/client.hcl index 9b20cffb407..07c7ee9850e 100644 --- a/e2e/terraform/shared/nomad/client.hcl +++ b/e2e/terraform/shared/nomad/client.hcl @@ -1,3 +1,5 @@ +plugin_dir = "/opt/nomad/plugins" + client { enabled = true @@ -17,6 +19,14 @@ client { } } +plugin "nomad-driver-podman" { + config { + volumes { + enabled = true + } + } +} + vault { enabled = true address = "http://active.vault.service.consul:8200" diff --git a/e2e/terraform/shared/nomad/indexed/client-0.hcl b/e2e/terraform/shared/nomad/indexed/client-0.hcl index 127272508e4..e13ffd37f33 100644 --- a/e2e/terraform/shared/nomad/indexed/client-0.hcl +++ b/e2e/terraform/shared/nomad/indexed/client-0.hcl @@ -16,6 +16,15 @@ client { } } +plugin_dir = "/opt/nomad/plugins" +plugin "nomad-driver-podman" { + config { + volumes { + enabled = true + } + } +} + vault { enabled = true address = "http://active.vault.service.consul:8200" diff --git a/e2e/terraform/shared/nomad/indexed/client-1.hcl b/e2e/terraform/shared/nomad/indexed/client-1.hcl index ddc9e0bbdea..eabc5b878e8 100644 --- a/e2e/terraform/shared/nomad/indexed/client-1.hcl +++ b/e2e/terraform/shared/nomad/indexed/client-1.hcl @@ -11,6 +11,15 @@ client { } } +plugin_dir = "/opt/nomad/plugins" +plugin "nomad-driver-podman" { + config { + volumes { + enabled = true + } + } +} + vault { enabled = true address = "http://active.vault.service.consul:8200" diff --git a/e2e/terraform/shared/nomad/indexed/client-2.hcl b/e2e/terraform/shared/nomad/indexed/client-2.hcl index 3501dc79ae8..38abac09688 100644 --- a/e2e/terraform/shared/nomad/indexed/client-2.hcl +++ b/e2e/terraform/shared/nomad/indexed/client-2.hcl @@ -13,6 +13,15 @@ client { } } +plugin_dir = "/opt/nomad/plugins" +plugin "nomad-driver-podman" { + config { + volumes { + enabled = true + } + } +} + vault { enabled = true address = "http://active.vault.service.consul:8200" diff --git a/e2e/terraform/shared/nomad/nomad.service b/e2e/terraform/shared/nomad/nomad.service index 17eda27b31d..a4ea35a09d2 100644 --- a/e2e/terraform/shared/nomad/nomad.service +++ b/e2e/terraform/shared/nomad/nomad.service @@ -13,12 +13,9 @@ LimitNPROC=infinity TasksMax=infinity Restart=on-failure RestartSec=2 - -# systemd>=230 prefer StartLimitIntervalSec,StartLimitBurst in Unit, -# however Ubuntu 16.04 only has systemd==229. Use these old style settings -# as they will be supported by newer systemds. +StartLimitIntervalSec=10 StartLimitBurst=3 -StartLimitInterval=10 + [Install] WantedBy=multi-user.target