From 1291faa850fef38ee33d612d9434aaeafae73fa1 Mon Sep 17 00:00:00 2001 From: Lewis Marshall Date: Mon, 20 Aug 2018 17:26:40 +0100 Subject: [PATCH] test: Run test VMs in Flynn Signed-off-by: Lewis Marshall --- builder/manifest.json | 2 +- test/arg/arg.go | 3 - test/cluster/cluster.go | 89 +++++-------- test/cluster/instance.go | 225 +++++++++++++++++---------------- test/cluster/net.go | 264 --------------------------------------- test/img/packages.sh | 3 + test/rootfs/setup.sh | 12 +- test/run.sh | 22 +--- test/runner/runner.go | 112 +++++++++-------- test/scripts/setup.sh | 2 +- test/scripts/start.sh | 1 - test/vm.sh | 71 +++++++++++ 12 files changed, 295 insertions(+), 511 deletions(-) delete mode 100644 test/cluster/net.go create mode 100755 test/vm.sh diff --git a/builder/manifest.json b/builder/manifest.json index 2b2c28e358..e8f5665881 100644 --- a/builder/manifest.json +++ b/builder/manifest.json @@ -633,6 +633,7 @@ }, "copy": { "test/run.sh": "/bin/run-flynn-test.sh", + "test/vm.sh": "/bin/run-vm.sh", "test/scripts/start.sh": "/test/bin/start-runner.sh", "test/rootfs/build.sh": "/test/rootfs/build.sh", "test/rootfs/setup.sh": "/test/rootfs/setup.sh", @@ -642,7 +643,6 @@ "test/runner/assets/index.html": "/test/assets/", "test/runner/assets/style.css": "/test/assets/", "test/runner/assets/ansi_up/ansi_up.js": "/test/assets/ansi_up/" - } } ], diff --git a/test/arg/arg.go b/test/arg/arg.go index 6ff5114a8c..7f9609218a 100644 --- a/test/arg/arg.go +++ b/test/arg/arg.go @@ -30,10 +30,7 @@ type Args struct { func Parse() *Args { args := &Args{BootConfig: cluster.BootConfig{}} - flag.StringVar(&args.BootConfig.User, "user", "ubuntu", "user to run QEMU as") flag.StringVar(&args.BootConfig.Kernel, "kernel", "rootfs/vmlinuz", "path to the Linux binary") - flag.StringVar(&args.BootConfig.Network, "network", "10.52.0.1/24", "the network to use for vms") - flag.StringVar(&args.BootConfig.NatIface, "nat", "eth0", "the interface to provide NAT to vms") flag.StringVar(&args.BootConfig.BackupsDir, "backups-dir", "", "directory containing historic cluster backups") flag.StringVar(&args.RootFS, "rootfs", "rootfs/rootfs.img", "filesystem image to use with QEMU") flag.StringVar(&args.CLI, "cli", "flynn", "path to flynn-cli binary") diff --git a/test/cluster/cluster.go b/test/cluster/cluster.go index 877a6eeddd..61feb4b7da 100644 --- a/test/cluster/cluster.go +++ b/test/cluster/cluster.go @@ -14,6 +14,7 @@ import ( "time" "github.com/flynn/flynn/cli/config" + controller "github.com/flynn/flynn/controller/client" "github.com/flynn/flynn/discoverd/client" "github.com/flynn/flynn/pkg/random" "github.com/flynn/flynn/test/buildlog" @@ -28,10 +29,7 @@ const ( ) type BootConfig struct { - User string Kernel string - Network string - NatIface string BackupsDir string } @@ -49,7 +47,6 @@ type Cluster struct { bc BootConfig vm *VMManager out io.Writer - bridge *Bridge rootFS string } @@ -107,47 +104,43 @@ func (c *Cluster) BuildFlynn(rootFS, commit string, merge bool, runTests bool) ( return "", err } - uid, gid, err := lookupUser(c.bc.User) - if err != nil { - return "", err + disk := &VMDisk{ + FS: rootFS, + COW: true, + Temp: false, } - build, err := c.vm.NewInstance(&VMConfig{ - Kernel: c.bc.Kernel, - User: uid, - Group: gid, - Memory: "16384", - Cores: 8, - Drives: map[string]*VMDrive{ - "hda": {FS: rootFS, COW: true, Temp: false}, - }, + Kernel: c.bc.Kernel, + Memory: 16384, + Cores: 8, + Disk: disk, BackupsDir: c.bc.BackupsDir, }) if err != nil { - return build.Drive("hda").FS, err + return disk.FS, err } c.log("Booting build instance...") if err := build.Start(); err != nil { - return build.Drive("hda").FS, fmt.Errorf("error starting build instance: %s", err) + return disk.FS, fmt.Errorf("error starting build instance: %s", err) } c.log("Waiting for instance to boot...") if err := buildFlynn(build, commit, merge, c.out); err != nil { build.Kill() - return build.Drive("hda").FS, fmt.Errorf("error running build script: %s", err) + return disk.FS, fmt.Errorf("error running build script: %s", err) } if runTests { if err := runUnitTests(build, c.out); err != nil { build.Kill() - return build.Drive("hda").FS, fmt.Errorf("unit tests failed: %s", err) + return disk.FS, fmt.Errorf("unit tests failed: %s", err) } } if err := build.Shutdown(); err != nil { - return build.Drive("hda").FS, fmt.Errorf("error while stopping build instance: %s", err) + return disk.FS, fmt.Errorf("error while stopping build instance: %s", err) } - c.rootFS = build.Drive("hda").FS + c.rootFS = disk.FS return c.rootFS, nil } @@ -198,13 +191,6 @@ func (c *Cluster) Boot(typ ClusterType, count int, buildLog *buildlog.Log, killO }, nil } -func (c *Cluster) BridgeIP() string { - if c.bridge == nil { - return "" - } - return c.bridge.IP() -} - func (c *Cluster) AddHost() (*Instance, error) { if c.rootFS == "" { return nil, errors.New("cluster not yet booted") @@ -246,28 +232,19 @@ func (c *Cluster) Size() int { } func (c *Cluster) startVMs(typ ClusterType, rootFS string, count int, initial bool) ([]*Instance, error) { - uid, gid, err := lookupUser(c.bc.User) - if err != nil { - return nil, err - } - instances := make([]*Instance, count) for i := 0; i < count; i++ { - memory := "8192" + memory := 8192 if initial && i == 0 { // give the first instance more memory as that is where // the test binary runs, and the tests use a lot of memory - memory = "16384" + memory = 16384 } inst, err := c.vm.NewInstance(&VMConfig{ - Kernel: c.bc.Kernel, - User: uid, - Group: gid, - Memory: memory, - Cores: 2, - Drives: map[string]*VMDrive{ - "hda": {FS: rootFS, COW: true, Temp: true}, - }, + Kernel: c.bc.Kernel, + Memory: memory, + Cores: 2, + Disk: &VMDisk{FS: rootFS, COW: true, Temp: true}, BackupsDir: c.bc.BackupsDir, }) if err != nil { @@ -312,16 +289,15 @@ func (c *Cluster) setup() error { if _, err := os.Stat(c.bc.Kernel); os.IsNotExist(err) { return fmt.Errorf("cluster: not a kernel file: %s", c.bc.Kernel) } - if c.bridge == nil { - var err error - name := "flynnbr." + random.String(5) - c.logf("creating network bridge %s\n", name) - c.bridge, err = createBridge(name, c.bc.Network, c.bc.NatIface) - if err != nil { - return fmt.Errorf("could not create network bridge: %s", err) - } + instances, err := discoverd.GetInstances("controller", 10*time.Second) + if err != nil { + return err } - c.vm = NewVMManager(c.bridge) + client, err := controller.NewClient("", instances[0].Meta["AUTH_KEY"]) + if err != nil { + return err + } + c.vm = NewVMManager(client) return nil } @@ -363,13 +339,6 @@ func (c *Cluster) Shutdown() { c.logf("error killing instance %d: %s\n", i, err) } } - if c.bridge != nil { - c.logf("deleting network bridge %s\n", c.bridge.name) - if err := deleteBridge(c.bridge); err != nil { - c.logf("error deleting network bridge %s: %s\n", c.bridge.name, err) - } - c.bridge = nil - } } var flynnBuildScript = template.Must(template.New("flynn-build").Parse(` diff --git a/test/cluster/instance.go b/test/cluster/instance.go index 19fc04d9c2..7737f044e4 100644 --- a/test/cluster/instance.go +++ b/test/cluster/instance.go @@ -11,29 +11,32 @@ import ( "strconv" "strings" "sync" - "syscall" "time" + units "github.com/docker/go-units" + controller "github.com/flynn/flynn/controller/client" + ct "github.com/flynn/flynn/controller/types" + "github.com/flynn/flynn/host/resource" + host "github.com/flynn/flynn/host/types" "github.com/flynn/flynn/pkg/attempt" + "github.com/flynn/flynn/pkg/cluster" "github.com/flynn/flynn/pkg/random" "golang.org/x/crypto/ssh" ) -func NewVMManager(bridge *Bridge) *VMManager { - return &VMManager{taps: &TapManager{bridge}} +func NewVMManager(client controller.Client) *VMManager { + return &VMManager{client} } type VMManager struct { - taps *TapManager + client controller.Client } type VMConfig struct { Kernel string - User int - Group int - Memory string + Memory int Cores int - Drives map[string]*VMDrive + Disk *VMDisk Args []string Out io.Writer `json:"-"` BackupsDir string @@ -41,14 +44,18 @@ type VMConfig struct { netFS string } -type VMDrive struct { +type VMDisk struct { FS string COW bool Temp bool } func (v *VMManager) NewInstance(c *VMConfig) (*Instance, error) { - inst := &Instance{ID: random.String(8), VMConfig: c} + inst := &Instance{ + ID: random.String(8), + VMConfig: c, + client: v.client, + } if c.Kernel == "" { c.Kernel = "vmlinuz" } @@ -59,12 +66,6 @@ func (v *VMManager) NewInstance(c *VMConfig) (*Instance, error) { return nil, err } } - var err error - inst.tap, err = v.taps.NewTap(c.User, c.Group) - if err != nil { - return nil, err - } - inst.IP = inst.tap.IP.String() return inst, nil } @@ -73,8 +74,8 @@ type Instance struct { IP string `json:"ip"` *VMConfig - tap *Tap - cmd *exec.Cmd + client controller.Client + job *ct.Job tempFiles []string @@ -84,29 +85,6 @@ type Instance struct { initial bool } -func (i *Instance) writeInterfaceConfig() error { - dir, err := ioutil.TempDir("", "netfs-") - if err != nil { - return err - } - i.tempFiles = append(i.tempFiles, dir) - i.netFS = dir - - if err := os.Chmod(dir, 0755); err != nil { - os.RemoveAll(dir) - return err - } - - f, err := os.Create(filepath.Join(dir, "eth0")) - if err != nil { - os.RemoveAll(dir) - return err - } - defer f.Close() - - return i.tap.WriteInterfaceConfig(f) -} - func (i *Instance) cleanup() { for _, f := range i.tempFiles { fmt.Printf("removing temp file %s\n", f) @@ -114,9 +92,6 @@ func (i *Instance) cleanup() { fmt.Printf("could not remove temp file %s: %s\n", f, err) } } - if err := i.tap.Close(); err != nil { - fmt.Printf("could not close tap device %s: %s\n", i.tap.Name, err) - } i.tempFiles = nil i.sshMtx.Lock() @@ -127,47 +102,80 @@ func (i *Instance) cleanup() { } func (i *Instance) Start() error { - i.writeInterfaceConfig() - - macRand := random.Bytes(3) - macaddr := fmt.Sprintf("52:54:00:%02x:%02x:%02x", macRand[0], macRand[1], macRand[2]) - - i.Args = append(i.Args, - "-enable-kvm", - "-kernel", i.Kernel, - "-append", "root=/dev/sda net.ifnames=0", - "-netdev", "tap,id=vmnic,ifname="+i.tap.Name+",script=no,downscript=no", - "-device", "virtio-net,netdev=vmnic,mac="+macaddr, - "-virtfs", "fsdriver=local,path="+i.netFS+",security_model=passthrough,readonly,mount_tag=netfs", - "-virtfs", "fsdriver=local,path="+i.BackupsDir+",security_model=passthrough,readonly,mount_tag=backupsfs", - "-nographic", - ) - if i.Memory != "" { - i.Args = append(i.Args, "-m", i.Memory) + // create a copy-on-write disk if necessary + if i.Disk.COW { + fs, err := i.createCOW(i.Disk.FS, i.Disk.Temp) + if err != nil { + i.cleanup() + return err + } + i.Disk.FS = fs } - if i.Cores > 0 { - i.Args = append(i.Args, "-smp", strconv.Itoa(i.Cores)) + + // stream job events so we can grab the IP once the job has started + events := make(chan *ct.Job) + stream, err := i.client.StreamJobEvents(os.Getenv("FLYNN_APP_ID"), events) + if err != nil { + i.cleanup() + return err } - for n, d := range i.Drives { - if d.COW { - fs, err := i.createCOW(d.FS, d.Temp) - if err != nil { + defer stream.Close() + + // run the VM as a one-off job + newJob := &ct.NewJob{ + ReleaseID: os.Getenv("FLYNN_RELEASE_ID"), + Args: []string{"/bin/run-vm.sh"}, + Env: map[string]string{ + "MEMORY": strconv.Itoa(i.Memory), + "CPUS": strconv.Itoa(i.Cores), + "DISK": i.Disk.FS, + "KERNEL": i.Kernel, + }, + MountsFrom: "runner", + Resources: resource.Defaults(), + Profiles: []host.JobProfile{host.JobProfileKVM}, + } + newJob.Resources.SetLimit(resource.TypeMemory, int64(i.Memory*units.MiB)) + i.job, err = i.client.RunJobDetached(os.Getenv("FLYNN_APP_ID"), newJob) + if err != nil { + i.cleanup() + return err + } + + timeout := time.After(30 * time.Second) + for { + select { + case job, ok := <-events: + if !ok { + i.cleanup() + return stream.Err() + } + if job.ID != i.job.ID { + continue + } + switch job.State { + case ct.JobStateDown: i.cleanup() - return err + return errors.New("job stopped") + case ct.JobStateUp: + host, err := cluster.NewClient().Host(job.HostID) + if err != nil { + i.cleanup() + return err + } + hostJob, err := host.GetJob(job.ID) + if err != nil { + i.cleanup() + return err + } + i.IP = hostJob.InternalIP + return nil } - d.FS = fs + case <-timeout: + i.cleanup() + return errors.New("timed out waiting for job to start") } - i.Args = append(i.Args, fmt.Sprintf("-%s", n), d.FS) - } - - i.cmd = exec.Command("sudo", append([]string{"-u", fmt.Sprintf("#%d", i.User), "-g", fmt.Sprintf("#%d", i.Group), "-H", "/usr/bin/qemu-system-x86_64"}, i.Args...)...) - i.cmd.Stdout = i.Out - i.cmd.Stderr = i.Out - var err error - if err = i.cmd.Start(); err != nil { - i.cleanup() } - return err } func (i *Instance) createCOW(image string, temp bool) (string, error) { @@ -179,38 +187,45 @@ func (i *Instance) createCOW(image string, temp bool) (string, error) { if temp { i.tempFiles = append(i.tempFiles, dir) } - if err := os.Chown(dir, i.User, i.Group); err != nil { - return "", err - } path := filepath.Join(dir, "rootfs.img") cmd := exec.Command("qemu-img", "create", "-f", "qcow2", "-b", image, path) if err = cmd.Run(); err != nil { return "", fmt.Errorf("failed to create COW filesystem: %s", err.Error()) } - if err := os.Chown(path, i.User, i.Group); err != nil { - return "", err - } return path, nil } -func (i *Instance) Wait(timeout time.Duration) error { - done := make(chan error) - go func() { - done <- i.cmd.Wait() - }() - select { - case err := <-done: +func (i *Instance) Wait(timeout time.Duration, f func() error) error { + events := make(chan *ct.Job) + stream, err := i.client.StreamJobEvents(os.Getenv("FLYNN_APP_ID"), events) + if err != nil { + return err + } + defer stream.Close() + if err := f(); err != nil { return err - case <-time.After(timeout): - return errors.New("timeout") + } + timeoutCh := time.After(timeout) + for { + select { + case job, ok := <-events: + if !ok { + return fmt.Errorf("error streaming job events: %s", stream.Err()) + } + if job.ID == i.job.ID && job.State == ct.JobStateDown { + return nil + } + case <-timeoutCh: + return errors.New("timed out waiting for job to stop") + } } } func (i *Instance) Shutdown() error { - if err := i.Run("sudo poweroff", nil); err != nil { - return i.Kill() - } - if err := i.Wait(30 * time.Second); err != nil { + err := i.Wait(30*time.Second, func() error { + return i.Run("sudo poweroff", nil) + }) + if err != nil { return i.Kill() } i.cleanup() @@ -219,13 +234,9 @@ func (i *Instance) Shutdown() error { func (i *Instance) Kill() error { defer i.cleanup() - if err := i.cmd.Process.Signal(syscall.SIGTERM); err != nil { - return err - } - if err := i.Wait(5 * time.Second); err != nil { - return i.cmd.Process.Kill() - } - return nil + return i.Wait(10*time.Second, func() error { + return i.client.DeleteJob(os.Getenv("FLYNN_APP_ID"), i.job.ID) + }) } func (i *Instance) dialSSH(stderr io.Writer) error { @@ -318,7 +329,3 @@ func (i *Instance) run(command string, s *Streams, env map[string]string, timeou return errors.New("command timed out") } } - -func (i *Instance) Drive(name string) *VMDrive { - return i.Drives[name] -} diff --git a/test/cluster/net.go b/test/cluster/net.go deleted file mode 100644 index c060360dc3..0000000000 --- a/test/cluster/net.go +++ /dev/null @@ -1,264 +0,0 @@ -package cluster - -import ( - "bytes" - "fmt" - "html/template" - "io" - "io/ioutil" - "net" - "os" - "syscall" - "unsafe" - - "github.com/docker/libnetwork/ipallocator" - "github.com/flynn/flynn/pkg/iptables" - "github.com/flynn/flynn/pkg/random" - "github.com/vishvananda/netlink" -) - -type Bridge struct { - name string - iface *netlink.Bridge - ipAddr net.IP - ipNet *net.IPNet - alloc *ipallocator.IPAllocator -} - -func (b *Bridge) IP() string { - return b.ipAddr.String() -} - -func createBridge(name, network, natIface string) (*Bridge, error) { - la := netlink.NewLinkAttrs() - la.Name = name - br := netlink.Link(&netlink.Bridge{LinkAttrs: la}) - - ipAddr, ipNet, err := net.ParseCIDR(network) - if err != nil { - return nil, err - } - if err := netlink.LinkAdd(br); err != nil { - return nil, err - } - - iface, err := netlink.LinkByName(name) - if err != nil { - return nil, err - } - - // We need to explicitly assign the MAC address to avoid it changing to a lower value - // See: https://github.com/flynn/flynn/issues/223 - mac := random.Bytes(5) - if err := netlink.LinkSetHardwareAddr(iface, append([]byte{0xfe}, mac...)); err != nil { - return nil, err - } - if netlink.AddrAdd(iface, &netlink.Addr{IPNet: &net.IPNet{IP: ipAddr, Mask: ipNet.Mask}}); err != nil { - return nil, err - } - if err := netlink.LinkSetUp(iface); err != nil { - return nil, err - } - ipFwd := "/proc/sys/net/ipv4/ip_forward" - data, err := ioutil.ReadFile(ipFwd) - if err != nil { - return nil, err - } - if !bytes.HasPrefix(data, []byte("1")) { - if err := ioutil.WriteFile(ipFwd, []byte("1\n"), 0644); err != nil { - return nil, err - } - } - if err := setupIPTables(name, natIface); err != nil { - return nil, err - } - - bridge := &Bridge{ - name: name, - iface: iface.(*netlink.Bridge), - ipAddr: ipAddr, - ipNet: ipNet, - alloc: ipallocator.New(), - } - bridge.alloc.RequestIP(ipNet, ipAddr) - return bridge, nil -} - -func deleteBridge(bridge *Bridge) error { - if err := netlink.LinkSetDown(bridge.iface); err != nil { - return err - } - if err := netlink.LinkDel(bridge.iface); err != nil { - return err - } - cleanupIPTables(bridge.name) - return nil -} - -func cleanupIPTables(bridgeName string) { - // Delete the forwarding rule. The postrouting rule does not need deletion - // as there is usually only one per box and it doesn't change. - iptables.Raw("-D", "FORWARD", "-i", bridgeName, "-j", "ACCEPT") -} - -func setupIPTables(bridgeName, natIface string) error { - nat := []string{"POSTROUTING", "-t", "nat", "-o", natIface, "-j", "MASQUERADE"} - if !iptables.Exists(nat...) { - if output, err := iptables.Raw(append([]string{"-I"}, nat...)...); err != nil { - return fmt.Errorf("unable to enable network bridge NAT: %s", err) - } else if len(output) != 0 { - return fmt.Errorf("unknown error creating bridge NAT rule: %s", output) - } - } - - forward := []string{"FORWARD", "-i", bridgeName, "-j", "ACCEPT"} - if !iptables.Exists(forward...) { - if output, err := iptables.Raw(append([]string{"-I"}, forward...)...); err != nil { - return fmt.Errorf("unable to enable forwarding: %s", err) - } else if len(output) != 0 { - return fmt.Errorf("unknown error enabling forwarding: %s", output) - } - } - - return nil -} - -const ( - IFNAMSIZ = 16 - IFF_TAP = 0x0002 - IFF_NO_PI = 0x1000 - TUNSETIFF = 0x400454ca - TUNSETPERSIST = 0x400454cb - TUNSETOWNER = 0x400454cc - TUNSETGROUP = 0x400454ce -) - -type ifreq struct { - name [IFNAMSIZ]byte - flags uint16 -} - -func ioctl(f *os.File, req int, data uintptr) syscall.Errno { - _, _, err := syscall.Syscall(syscall.SYS_IOCTL, f.Fd(), uintptr(req), data) - return err -} - -func ioctlTap(name string) (*os.File, error) { - req := ifreq{flags: IFF_NO_PI | IFF_TAP} - copy(req.name[:IFNAMSIZ-1], name) - - f, err := os.OpenFile("/dev/net/tun", os.O_RDWR, 0) - if err != nil { - f.Close() - return nil, err - } - if err := ioctl(f, TUNSETIFF, uintptr(unsafe.Pointer(&req))); err != 0 { - f.Close() - return nil, err - } - - return f, nil -} - -func createTap(name string, uid, gid int) error { - f, err := ioctlTap(name) - if err != nil { - return err - } - defer f.Close() - - if uid > 0 { - if err := ioctl(f, TUNSETOWNER, uintptr(uid)); err != 0 { - return err - } - } - if gid > 0 { - if err := ioctl(f, TUNSETGROUP, uintptr(gid)); err != 0 { - return err - } - } - if err := ioctl(f, TUNSETPERSIST, 1); err != 0 { - return err - } - - return nil -} - -func deleteTap(name string) error { - f, err := ioctlTap(name) - if err != nil { - return err - } - defer f.Close() - if errno := ioctl(f, TUNSETPERSIST, 0); errno != 0 { - return errno - } - return nil -} - -type Tap struct { - Name string - IP net.IP - bridge *Bridge -} - -func (t *Tap) Close() error { - if err := deleteTap(t.Name); err != nil { - return err - } - if t.IP != nil { - t.bridge.alloc.ReleaseIP(t.bridge.ipNet, t.IP) - } - return nil -} - -var ifaceConfig = template.Must(template.New("eth0").Parse(` -auto eth0 -iface eth0 inet static - address {{.Address}} - gateway {{.Gateway}} - netmask 255.255.255.0 - dns-nameservers 8.8.8.8 8.8.4.4 -`[1:])) - -func (t *Tap) WriteInterfaceConfig(f io.Writer) error { - return ifaceConfig.Execute(f, map[string]string{ - "Address": t.IP.String(), - "Gateway": t.bridge.IP(), - }) -} - -type TapManager struct { - bridge *Bridge -} - -func (t *TapManager) NewTap(uid, gid int) (*Tap, error) { - tap := &Tap{Name: "flynntap." + random.String(5), bridge: t.bridge} - - if err := createTap(tap.Name, uid, gid); err != nil { - return nil, err - } - - var err error - tap.IP, err = t.bridge.alloc.RequestIP(t.bridge.ipNet, nil) - if err != nil { - tap.Close() - return nil, err - } - - iface, err := netlink.LinkByName(tap.Name) - if err != nil { - tap.Close() - return nil, err - } - if err := netlink.LinkSetUp(iface); err != nil { - tap.Close() - return nil, err - } - if err := netlink.LinkSetMaster(iface, t.bridge.iface); err != nil { - tap.Close() - return nil, err - } - - return tap, nil -} diff --git a/test/img/packages.sh b/test/img/packages.sh index dcb38b4ce6..4e13ecbcc9 100755 --- a/test/img/packages.sh +++ b/test/img/packages.sh @@ -7,6 +7,9 @@ apt-get clean curl -fsSLo "/usr/local/bin/docker" "https://get.docker.com/builds/Linux/x86_64/docker-1.9.1" chmod +x "/usr/local/bin/docker" +curl -fsLo "/usr/local/bin/jq" "https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64" +chmod +x "/usr/local/bin/jq" + export HOME="/root" git config --global "user.email" "test@flynn.io" git config --global "user.name" "Flynn Test" diff --git a/test/rootfs/setup.sh b/test/rootfs/setup.sh index 5c12a12a98..c3976f4220 100755 --- a/test/rootfs/setup.sh +++ b/test/rootfs/setup.sh @@ -16,10 +16,16 @@ echo ubuntu:ubuntu | chpasswd # set up fstab echo "LABEL=rootfs / ext4 defaults 0 1" > /etc/fstab -echo "netfs /etc/network/interfaces.d 9p trans=virtio,noauto,x-systemd.automount 0 0" >> /etc/fstab -# make sure there are no existing network interface configs -rm -rf /etc/network/interfaces.d/* +# setup networking +cat > /etc/systemd/network/10-flynn.network < /etc/hosts diff --git a/test/run.sh b/test/run.sh index 3ca9e703c9..17942bc1d6 100755 --- a/test/run.sh +++ b/test/run.sh @@ -21,23 +21,11 @@ main() { cd "${ROOT}/test" - # remove some flags from $@ - # TODO: remove once the CI runner no longer passes these flags - local args=( - "--flynnrc" "${HOME}/.flynnrc" - "--cli" "$(readlink -f ../build/bin/flynn)" - "--flynn-host" "$(readlink -f ../build/bin/flynn-host)" - ) - while [[ -n "$1" ]]; do - if [[ "$1" = "--flynnrc" ]] || [[ "$1" = "--flynn-host" ]] || [[ "$1" = "--cli" ]]; then - shift 2 - continue - fi - args+=("$1") - shift - done - - exec /bin/flynn-test ${args[@]} + exec /bin/flynn-test \ + --flynnrc "${HOME}/.flynnrc" \ + --cli "$(readlink -f ../build/bin/flynn)" \ + --flynn-host "$(readlink -f ../build/bin/flynn-host)" \ + $@ } main "$@" diff --git a/test/runner/runner.go b/test/runner/runner.go index 1af6f3d550..8dc2b6750f 100644 --- a/test/runner/runner.go +++ b/test/runner/runner.go @@ -12,6 +12,7 @@ import ( "log" "mime" "mime/multipart" + "net" "net/http" "net/url" "os" @@ -116,15 +117,13 @@ type Runner struct { rootFS string githubToken string s3 *s3.S3 - networks map[string]struct{} - netMtx sync.Mutex db *postgres.DB buildCh chan struct{} clusters map[string]*cluster.Cluster authKey string runEnv map[string]string - subnet uint64 ircMsgs chan string + ip string } var args *arg.Args @@ -139,7 +138,6 @@ func main() { runner := &Runner{ bc: args.BootConfig, events: make(chan Event), - networks: make(map[string]struct{}), buildCh: make(chan struct{}, args.ConcurrentBuilds), clusters: make(map[string]*cluster.Cluster), ircMsgs: make(chan string, 100), @@ -171,15 +169,35 @@ func (r *Runner) start() error { return errors.New("GITHUB_TOKEN not set") } + // set r.ip from /.containerconfig + if err := func() error { + f, err := os.Open("/.containerconfig") + if err != nil { + return err + } + defer f.Close() + var config struct { + IP string + } + if err := json.NewDecoder(f).Decode(&config); err != nil { + return err + } + ip, _, err := net.ParseCIDR(config.IP) + if err != nil { + return err + } + r.ip = ip.String() + return nil + }(); err != nil { + return err + } + r.s3 = s3.New(session.New(&aws.Config{Region: aws.String("us-east-1")})) - bc := r.bc - bc.Network = r.allocateNet() var err error - if r.rootFS, err = cluster.BuildFlynn(bc, args.RootFS, "origin/master", false, os.Stdout); err != nil { + if r.rootFS, err = cluster.BuildFlynn(r.bc, args.RootFS, "origin/master", false, os.Stdout); err != nil { return fmt.Errorf("could not build flynn: %s", err) } - r.releaseNet(bc.Network) shutdown.BeforeExit(func() { removeRootFS(r.rootFS) }) db := postgres.Wait(nil, nil) @@ -307,29 +325,34 @@ cd ~/go/src/github.com/flynn/flynn script/configure-docker "{{ .Cluster.ClusterDomain }}" -build/bin/flynn cluster add \ - --tls-pin "{{ .Config.TLSPin }}" \ - --git-url "{{ .Config.GitURL }}" \ - --docker-push-url "{{ .Config.DockerPushURL }}" \ - default \ - {{ .Config.ControllerURL }} \ - {{ .Config.Key }} - -git config --global user.email "ci@flynn.io" -git config --global user.name "CI" - -cd test - -cmd="bin/flynn-test \ - --flynnrc $HOME/.flynnrc \ - --cluster-api http://{{ .Cluster.BridgeIP }}/cluster/{{ .Cluster.ID }} \ - --cli $(pwd)/../build/bin/flynn \ - --flynn-host $(pwd)/../build/bin/flynn-host \ +export DISCOVERD="http://{{ (index .Cluster.Instances 0).IP }}:1111" + +# put the command in a file so the arguments aren't echoed in the logs +cat > /tmp/run-tests.sh <&2 + exit 1 +} + +main "$@"