build: implement a framework for starting test servers during tests

Test servers are implemented by docker containers and run real servers
for rclone to test against.

backend/ftp/ftp_test.go

@@ -5,13 +5,44 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/ftp"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFTP:",
+		RemoteName: "TestFTPProftpd:",
 		NilObject:  (*ftp.Object)(nil),
 	})
 }
+
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTPRclone:",
+		NilObject:  (*ftp.Object)(nil),
+	})
+}
+
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTPPureftpd:",
+		NilObject:  (*ftp.Object)(nil),
+	})
+}
+
+// func TestIntegration4(t *testing.T) {
+// 	if *fstest.RemoteName != "" {
+// 		t.Skip("skipping as -remote is set")
+// 	}
+// 	fstests.Run(t, &fstests.Opt{
+// 		RemoteName: "TestFTPVsftpd:",
+// 		NilObject:  (*ftp.Object)(nil),
+// 	})
+// }

backend/sftp/sftp_test.go

@@ -8,13 +8,24 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/sftp"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestSftp:",
+		RemoteName: "TestSFTPOpenssh:",
+		NilObject:  (*sftp.Object)(nil),
+	})
+}
+
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSFTPRclone:",
 		NilObject:  (*sftp.Object)(nil),
 	})
 }

backend/swift/swift_test.go

@@ -20,7 +20,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestSwift:",
+		RemoteName: "TestSwiftAIO:",
 		NilObject:  (*Object)(nil),
 	})
 }

backend/webdav/webdav_test.go

@@ -5,13 +5,36 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/webdav"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestWebdav:",
+		RemoteName: "TestWebdavNexcloud:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdavOwncloud:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdavRclone:",
 		NilObject:  (*webdav.Object)(nil),
 	})
 }

fstest/fstests/fstests.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/testserver"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
@@ -306,6 +307,10 @@ func Run(t *testing.T, opt *Opt) {
 		ctx           = context.Background()
 	)
 
+	if strings.HasSuffix(os.Getenv("RCLONE_CONFIG"), "/notfound") && *fstest.RemoteName == "" {
+		t.Skip("quicktest only")
+	}
+
 	// Skip the test if the remote isn't configured
 	skipIfNotOk := func(t *testing.T) {
 		if remote == nil {
@@ -352,7 +357,11 @@ func Run(t *testing.T, opt *Opt) {
 	if *fstest.RemoteName != "" {
 		remoteName = *fstest.RemoteName
 	}
+	oldFstestRemoteName := fstest.RemoteName
 	fstest.RemoteName = &remoteName
+	defer func() {
+		fstest.RemoteName = oldFstestRemoteName
+	}()
 	t.Logf("Using remote %q", remoteName)
 	var err error
 	if remoteName == "" {
@@ -361,6 +370,11 @@ func Run(t *testing.T, opt *Opt) {
 		isLocalRemote = true
 	}
 
+	// Start any test servers if required
+	finish, err := testserver.Start(remoteName)
+	require.NoError(t, err)
+	defer finish()
+
 	// Make the Fs we are testing with, initialising the local variables
 	// subRemoteName - name of the remote after the TestRemote:
 	// subRemoteLeaf - a subdirectory to use under that

fstest/test_all/config.yaml

@@ -144,13 +144,19 @@ backends:
    remote:   "TestS3Alibaba:"
    fastlist: true
  - backend:  "sftp"
-   remote:   "TestSftp:"
+   remote:   "TestSFTPOpenssh:"
+   fastlist: false
+ - backend:  "sftp"
+   remote:   "TestSFTPRclone:"
    fastlist: false
  - backend:  "sugarsync"
    remote:   "TestSugarSync:Test"
    fastlist: false
    ignore:
      - TestIntegration/FsMkdir/FsPutFiles/PublicLink
+ - backend:  "swift"
+   remote:   "TestSwiftAIO:"
+   fastlist: true
  - backend:  "swift"
    remote:   "TestSwift:"
    fastlist: true
@@ -163,10 +169,27 @@ backends:
    remote:   "TestYandex:"
    fastlist: false
  - backend:  "ftp"
-   remote:   "TestFTP:"
+   remote:   "TestFTPProftpd:"
    ignore:
      - TestIntegration/FsMkdir/FsEncoding/punctuation
    fastlist: false
+ # - backend:  "ftp"
+ #   remote:   "TestFTPVsftpd:"
+ #   ignore:
+ #     - TestIntegration/FsMkdir/FsEncoding/punctuation
+ #   fastlist: false
+ - backend:  "ftp"
+   remote:   "TestFTPPureftpd:"
+   ignore:
+     - TestIntegration/FsMkdir/FsEncoding/punctuation
+   fastlist: false
+ - backend:  "ftp"
+   remote:   "TestFTPRclone:"
+   ignore:
+     - "TestMultithreadCopy/{size:131071_streams:2}"
+     - "TestMultithreadCopy/{size:131072_streams:2}"
+     - "TestMultithreadCopy/{size:131073_streams:2}"
+   fastlist: false
  - backend:  "box"
    remote:   "TestBox:"
    fastlist: false
@@ -184,10 +207,25 @@ backends:
    remote:   "TestPcloud:"
    fastlist: false
  - backend:  "webdav"
-   remote:   "TestWebdav:"
+   remote:   "TestWebdavNextcloud:"
+   ignore:
+     - TestIntegration/FsMkdir/FsEncoding/punctuation
+     - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
+   fastlist: false
+ - backend:  "webdav"
+   remote:   "TestWebdavOwncloud:"
    ignore:
      - TestIntegration/FsMkdir/FsEncoding/punctuation
      - TestIntegration/FsMkdir/FsEncoding/invalid_UTF-8
+     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
+     - TestCopyFileCopyDest
+     - TestServerSideCopy
+     - TestSyncCopyDest
+   fastlist: false
+ - backend:  "webdav"
+   remote:   "TestWebdavRclone:"
+   ignore:
+     - TestFileReadAtZeroLength
    fastlist: false
  - backend:  "cache"
    remote:   "TestCache:"

fstest/test_all/run.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/testserver"
 )
 
 // Control concurrency per backend if required
@@ -213,6 +214,16 @@ func (r *Run) trial() {
 		return
 	}
 
+	// Start the test server if required
+	finish, err := testserver.Start(r.Remote)
+	if err != nil {
+		log.Printf("%s: Failed to start test server: %v", r.Remote, err)
+		_, _ = fmt.Fprintf(out, "%s: Failed to start test server: %v\n", r.Remote, err)
+		r.err = err
+		return
+	}
+	defer finish()
+
 	// Internal buffer
 	var b bytes.Buffer
 	multiOut := io.MultiWriter(out, &b)

fstest/testserver/images/test-sftp-openssh/Dockerfile

@@ -0,0 +1,11 @@
+# A very minimal sftp server for integration testing rclone
+FROM alpine:latest
+
+# User rclone, password password
+RUN \
+    apk add openssh && \
+    ssh-keygen -A && \
+    adduser -D rclone && \
+    echo "rclone:password" | chpasswd 
+
+ENTRYPOINT [ "/usr/sbin/sshd", "-D" ]

fstest/testserver/images/test-sftp-openssh/README.md

@@ -0,0 +1,17 @@
+# Test SFTP Openssh
+
+This is a docker image for rclone's integration tests which runs an
+openssh server in a docker image.
+
+## Build
+
+```
+docker build --rm -t rclone/test-sftp-openssh .
+docker push rclone/test-sftp-openssh
+```
+
+# Test
+
+```
+rclone lsf -R --sftp-host 172.17.0.2 --sftp-user rclone --sftp-pass $(rclone obscure password) :sftp:
+```

fstest/testserver/init.d/README.md

@@ -0,0 +1,31 @@
+This directory contains scripts to start and stop servers for testing.
+
+The commands are named after the remotes in use.  They should be
+executable files with the following parameters:
+
+    start  - starts the server
+    stop   - stops the server
+    status - returns non-zero exit code if the server is not running
+
+These will be called automatically by test_all if that remote is
+required.
+
+When start is run it should output config parameters for that remote.
+If a `_connect` parameter is output then that will be used for a
+connection test.  For example if `_connect=127.0.0.1:80` then a TCP
+connection will be made to `127.0.0.1:80` and only when that succeeds
+will the test continue.
+
+`run.bash` contains boilerplate to be included in a bash script for
+interpreting the command line parameters.
+
+`docker.bash` contains library functions to help with docker
+implementations.
+
+## TODO
+
+- sftpd - https://github.com/panubo/docker-sshd ?
+- openstack swift - https://github.com/bouncestorage/docker-swift
+- ceph - https://github.com/ceph/cn
+- other ftp servers
+

fstest/testserver/init.d/TestFTPProftpd

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+NAME=proftpd
+USER=rclone
+PASS=RaidedBannedPokes5
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           -e "FTP_USERNAME=rclone" \
+           -e "FTP_PASSWORD=$PASS" \
+           hauptmedia/proftpd
+
+    echo type=ftp
+    echo host=$(docker_ip)
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):21
+}
+
+. $(dirname "$0")/run.bash

fstest/testserver/init.d/TestFTPPureftpd

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+NAME=pureftpd
+USER=rclone
+PASS=AcridSpiesBooks2
+
+. $(dirname "$0")/docker.bash
+
+start() {
+    docker run --rm -d --name $NAME \
+           -e "FTP_USER_NAME=rclone" \
+           -e "FTP_USER_PASS=$PASS" \
+           -e "FTP_USER_HOME=/data" \
+           -e "FTP_MAX_CLIENTS=50" \
+           -e "FTP_MAX_CONNECTIONS=50" \
+           -e "FTP_PASSIVE_PORTS=30000:40000" \
+           stilliard/pure-ftpd
+
+    echo type=ftp
+    echo host=$(docker_ip)
+    echo user=$USER
+    echo pass=$(rclone obscure $PASS)
+    echo _connect=$(docker_ip):21
+}
+
+. $(dirname "$0")/run.bash