Skip to content
Permalink
Browse files

Merge pull request #870 from Mirantis/ivan4th/debian-image-ext-e2e

Add a CircleCI e2e job based on Debian image
  • Loading branch information...
ivan4th committed Mar 5, 2019
2 parents 3bcd031 + 042367c commit 5087718cb72164c0fee17fcfa1faee625028bbbb
@@ -204,15 +204,23 @@ e2e: &e2e
command: |
build/portforward.sh 8080&
mkdir -p ~/junit
skip_focus=("-ginkgo.skip=\[Heavy\]|\[MultiCNI\]|\[Disruptive\]|\[Flaky\]")
e2e_opts=("-ginkgo.skip=\[Heavy\]|\[MultiCNI\]|\[Disruptive\]|\[Flaky\]")
if [[ ${CIRCLE_JOB} = e2e_multi_cni ]]; then
# per-node config test requires an additional worker node
skip_focus="-ginkgo.skip=\[Heavy\]|\[Disruptive\]|\[Flaky\]|Per-node configuration"
e2e_opts="-ginkgo.skip=\[Heavy\]|\[Disruptive\]|\[Flaky\]|Per-node configuration"
fi
if [[ ${E2E_FOCUS:-} ]]; then
skip_focus+=("-ginkgo.focus=${E2E_FOCUS}")
e2e_opts+=("-ginkgo.focus=${E2E_FOCUS}")
fi
_output/virtlet-e2e-tests -test.v "${skip_focus[@]}" -junitOutput ~/junit/junit.xml -include-unsafe-tests=true
if [[ ${CIRCLE_JOB} = e2e_debian ]]; then
e2e_opts+=(-image cdimage.debian.org/cdimage/openstack/archive/9.8.0/debian-9.8.0-openstack-amd64.qcow2
-sshuser debian
-memoryLimit 256
-use-dhcp-network-config)
fi
_output/virtlet-e2e-tests -test.v "${e2e_opts[@]}" \
-junitOutput ~/junit/junit.xml \
-include-unsafe-tests=true
- store_test_results:
path: ~/junit
@@ -394,6 +402,9 @@ jobs:
e2e_1_12:
<<: *e2e

e2e_debian:
<<: *e2e

push_branch:
<<: *push_images

@@ -518,20 +529,29 @@ workflows:
only: /^master$|^.*-net$/
tags:
only: /^v[0-9].*/
- e2e_multi_cni:
# XXX: temporarily disabled, to be fixed
# - e2e_multi_cni:
# requires:
# - build
# filters:
# branches:
# only: /^master$|^.*-net$|^.*-ext-e2e$/
# tags:
# only: /^v[0-9].*/
- e2e_1_12:
requires:
- build
filters:
branches:
only: /^master$|^.*-net$/
ignore: /^.*-docs$/
tags:
only: /^v[0-9].*/
- e2e_1_12:
- e2e_debian:
requires:
- build
filters:
branches:
ignore: /^.*-docs$/
only: /^master$|^.*-ext-e2e$/
tags:
only: /^v[0-9].*/
- push_branch:
@@ -547,8 +567,10 @@ workflows:
- e2e_calico
- e2e_flannel
- e2e_weave
- e2e_multi_cni
# XXX: temporarily disabled, to be fixed
# - e2e_multi_cni
- e2e_1_12
- e2e_debian
- integration
filters:
branches:
@@ -566,4 +588,3 @@ workflows:
- build_docs:
requires:
- prepare_build

@@ -10,8 +10,12 @@ account.
There are some conventions about branch naming that are respected by
CircleCI. Namely, branches with names ending with `-net` will be
checked with extra e2e jobs that examine Weave, Flannel and multi-CNI
based network setups. For branches with names ending with `-docs` only
documentation will be built.
based network setups. For branches with names ending with `-ext-e2e`,
all the jobs that are run for `-net` branches are run, as well as an
additional job that verifies Virtlet against a Debian OpenStack image.
For branches with names ending with `-docs` only documentation will be
built. For `master` branch, the same policy is used as for `-ext-e2e`
branches.

Also, you can append `[ci skip]` to your commit message to have CI
skip the particular commit/PR altogether.
@@ -344,6 +344,21 @@ use the following construct in the pod annotations:
VirtletCloudInitUserDataScript: "@virtlet-mount-script@"
```

# Disabling cloud-init based network configuration

As the network configuration is applied only upon the first startup,
it's not used when
[persistent root filesystem](../volumes/#persistent-root-filesystem)
is used, so new network configuration can be passed using DHCP to a
new pod with the same rootfs. Moreover, in some cases it may be
desirable to force Virtlet to use DHCP instead of cloud-init based
network config. To do so, you need to add the following annotation
to the pod:

```yaml
VirtletForceDHCPNetworkConfig: "true"
```

# Additional links

These links may help to understand some basics about cloud-init:
@@ -0,0 +1,28 @@
apiVersion: v1
kind: Pod
metadata:
name: debian-vm
annotations:
kubernetes.io/target-runtime: virtlet.cloud
VirtletSSHKeys: |
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
# set root volume size
VirtletRootVolumeSize: 4Gi
# Debian image doesn't understand Virtlet's Cloud-Init network config currently
VirtletForceDHCPNetworkConfig: "true"
spec:
nodeSelector:
extraRuntime: virtlet

# This is the number of seconds Virtlet gives the VM to shut down cleanly.
# The default value of 30 seconds is ok for containers but probably too
# low for VM, so overriding it here is strongly advised.
terminationGracePeriodSeconds: 120
containers:
- name: debian-vm
# We use an 'archive' image so as to have a stable URL
image: virtlet.cloud/cdimage.debian.org/cdimage/openstack/archive/9.8.0/debian-9.8.0-openstack-amd64.qcow2
imagePullPolicy: IfNotPresent
# tty and stdin required for `kubectl attach -t` to work
tty: true
stdin: true
@@ -0,0 +1,5 @@
meta-data:
instance-id: foo.default
local-hostname: foo
network-config: null
user-data: null
@@ -115,6 +115,7 @@
CPUModel: ""
CPUSetting: null
DiskDriver: scsi
ForceDHCPNetworkConfig: false
InjectedFiles: null
MetaData: null
RootVolumeSize: 0
@@ -173,6 +174,7 @@
CPUModel: ""
CPUSetting: null
DiskDriver: scsi
ForceDHCPNetworkConfig: false
InjectedFiles: null
MetaData: null
RootVolumeSize: 0
@@ -226,6 +228,7 @@
CPUModel: ""
CPUSetting: null
DiskDriver: scsi
ForceDHCPNetworkConfig: false
InjectedFiles: null
MetaData: null
RootVolumeSize: 0
@@ -131,6 +131,7 @@
CPUModel: ""
CPUSetting: null
DiskDriver: scsi
ForceDHCPNetworkConfig: false
InjectedFiles: null
MetaData: null
RootVolumeSize: 0
@@ -177,9 +177,11 @@ func (g *CloudInitGenerator) generateUserData(volumeMap diskPathMap) ([]byte, er
}

func (g *CloudInitGenerator) generateNetworkConfiguration() ([]byte, error) {
if g.config.RootVolumeDevice() != nil {
// We don't use network config with persistent rootfs
// for now because with some cloud-init
if g.config.ParsedAnnotations.ForceDHCPNetworkConfig || g.config.RootVolumeDevice() != nil {
// Don't use cloud-init network config if asked not
// to do so.
// Also, we don't use network config with persistent
// rootfs for now because with some cloud-init
// implementations it's applied only once
return nil, nil
}
@@ -388,6 +388,18 @@ func TestCloudInitGenerator(t *testing.T) {
// make sure network config is null for the persistent rootfs case
verifyNetworkConfig: true,
},
{
name: "pod with forced dhcp network config",
config: &types.VMConfig{
PodName: "foo",
PodNamespace: "default",
ParsedAnnotations: &types.VirtletAnnotations{ForceDHCPNetworkConfig: true},
},
verifyMetaData: true,
verifyUserData: true,
// make sure network config is null
verifyNetworkConfig: true,
},
{
name: "injecting mount script into user data script",
config: &types.VMConfig{
@@ -45,6 +45,7 @@ const (
sshKeysKeyName = "VirtletSSHKeys"
chown9pfsMountsKeyName = "VirtletChown9pfsMounts"
systemUUIDKeyName = "VirtletSystemUUID"
forceDHCPNetworkConfigKeyName = "VirtletForceDHCPNetworkConfig"
// CloudInitUserDataSourceKeyName is the name of user data source key in the pod annotations.
CloudInitUserDataSourceKeyName = "VirtletCloudInitUserDataSource"
// SSHKeySourceKeyName is the name of ssh key source key in the pod annotations.
@@ -119,6 +120,10 @@ type VirtletAnnotations struct {
// SystemUUID specifies fixed SMBIOS UUID to be used for the domain.
// If not set, the SMBIOS UUID will be automatically generated from the Pod ID.
SystemUUID *uuid.UUID
// ForceDHCPNetworkConfig prevents Virtlet from using Cloud-Init based network
// configuration and makes it only provide DHCP. Note that this will
// not work for multi-CNI configuration.
ForceDHCPNetworkConfig bool
}

// ExternalDataLoader is used to load extra pod data from
@@ -318,5 +323,13 @@ func (va *VirtletAnnotations) parsePodAnnotations(ns string, podAnnotations map[
}
}

if podAnnotations[chown9pfsMountsKeyName] == "true" {
va.VirtletChown9pfsMounts = true
}

if podAnnotations[forceDHCPNetworkConfigKeyName] == "true" {
va.ForceDHCPNetworkConfig = true
}

return nil
}
@@ -163,6 +163,18 @@ func TestVirtletAnnotations(t *testing.T) {
CDImageType: "nocloud",
},
},
{
name: "force DHCP network config",
annotations: map[string]string{
"VirtletForceDHCPNetworkConfig": "true",
},
va: &VirtletAnnotations{
VCPUCount: 1,
DiskDriver: "scsi",
CDImageType: "nocloud",
ForceDHCPNetworkConfig: true,
},
},
// bad metadata items follow
{
name: "bad vcpu count",
@@ -227,7 +227,9 @@ func itShouldHaveNetworkConnectivity(podIface func() *framework.PodInterface, ss

It("Should be able to access another k8s endpoint"+suffix, func(done Done) {
defer close(done)
cmd := fmt.Sprintf("curl -s --connect-timeout 5 http://nginx.%s.svc.cluster.local", controller.Namespace())
// wget is present in CirrOS, Ubuntu and Debian images that we use, unlike curl,
// so we use it here
cmd := fmt.Sprintf("wget -O - -T 5 http://nginx.%s.svc.cluster.local", controller.Namespace())
Eventually(func() (string, error) {
return framework.RunSimple(ssh(), cmd)
}, 60).Should(ContainSubstring("Thank you for using nginx."))
@@ -38,7 +38,11 @@ import (
const (
cephContainerName = "ceph_cluster"
// avoid having the loop device on top of overlay2/aufs when using k-d-c
loopDeviceTestDir = "/dind/virtlet-e2e-tests"
loopDeviceTestDir = "/dind/virtlet-e2e-test-dir"
// The path to mkfs.ext2 may differ between the OSes. Also, on
// CirrOS, mkfs.ext2 is not under the default PATH when we do
// 'ssh sudo ...'
mkfsCommand = `export PATH="/usr/sbin:/sbin:$PATH"; sudo mkfs.ext2 %q`
)

var (
@@ -356,7 +360,7 @@ func makeVMWithMountAndSymlinkScript(nodeName string, PVCs []framework.PVCSpec,

func expectToBeUsableForFilesystem(ssh framework.Executor, devPath string) {
Eventually(func() error {
_, err := framework.RunSimple(ssh, fmt.Sprintf("sudo /usr/sbin/mkfs.ext2 %s", devPath))
_, err := framework.RunSimple(ssh, fmt.Sprintf(mkfsCommand, devPath))
return err
}, 60*5, 3).Should(Succeed())
do(framework.RunSimple(ssh, fmt.Sprintf("sudo mount %s /mnt", devPath)))
@@ -18,6 +18,7 @@ package framework

import (
"encoding/xml"
"flag"
"fmt"
"regexp"
"strconv"
@@ -29,6 +30,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var useDHCPNetworkConfig = flag.Bool("use-dhcp-network-config", false, "use DHCP network config instead of Cloud-Init-based one")

// VMInterface provides API to work with virtlet VM pods
type VMInterface struct {
controller *Controller
@@ -190,6 +193,10 @@ func (vmi *VMInterface) buildVMPod(options VMOptions) *v1.Pod {
"VirtletDiskDriver": options.DiskDriver,
"VirtletCloudInitUserDataOverwrite": strconv.FormatBool(options.OverwriteUserData),
}
if *useDHCPNetworkConfig {
annotations["VirtletForceDHCPNetworkConfig"] = "true"
}

if options.SSHKey != "" {
annotations["VirtletSSHKeys"] = options.SSHKey
}
@@ -67,7 +67,7 @@ var _ = Describe("virtletctl", func() {
_, err := controller.ConfigMaps().Create(cm)
Expect(err).NotTo(HaveOccurred())

vm = controller.VM("virtletctl-cirros-vm")
vm = controller.VM("virtletctl-test-vm")
Expect(vm.CreateAndWait(VMOptions{
SSHKeySource: "configmap/sshkey",
}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
@@ -99,8 +99,8 @@ var _ = Describe("virtletctl", func() {
defer closeFunc()
localExecutor := framework.LocalExecutor(ctx)

output := callVirtletctl(localExecutor, "ssh", "--namespace", controller.Namespace(), "cirros@virtletctl-cirros-vm", "--", "-i", tempfileName, "hostname")
Expect(output).To(Equal("virtletctl-cirros-vm"))
output := callVirtletctl(localExecutor, "ssh", "--namespace", controller.Namespace(), *sshUser+"@virtletctl-test-vm", "--", "-i", tempfileName, "hostname")
Expect(output).To(Equal("virtletctl-test-vm"))
}, 60)

It("Should dump Virtlet diagnostics on diag dump subcommand", func(done Done) {
@@ -243,5 +243,5 @@ func addFlexvolMount(pod *framework.PodInterface, name string, mountPath string,
func shouldBeMounted(ssh framework.Executor, path string) {
Eventually(func() (string, error) {
return framework.RunSimple(ssh, "ls -l "+path)
}, 60).Should(ContainSubstring("lost+found"))
}, 180, 5).Should(ContainSubstring("lost+found"))
}

0 comments on commit 5087718

Please sign in to comment.
You can’t perform that action at this time.