engines/http: Add support for Swift storage backend

This also converts two configuration variables to use "posval" instead
of string parsing or integers.

HOWTO

@@ -2137,14 +2137,16 @@ with the caveat that when used on the command line, they must come after the
 
 	Password for HTTP authentication.
 
-.. option:: https=bool : [http]
+.. option:: https=str : [http]
 
-	Enable HTTPS instead of http. Default is **0**
+	Enable HTTPS instead of http. *on* enables HTTPS; *insecure*
+	will enable HTTPS, but disable SSL peer verification (use with
+	caution!). Default is **off**
 
-.. option:: http_s3=bool : [http]
+.. option:: http_mode=str : [http]
 
-	Enable S3 specific HTTP headers such as authenticating requests
-	with AWS Signature Version 4. Default is **0**
+	Which HTTP access mode to use: *webdav*, *swift*, or *s3*.
+	Default is **webdav**
 
 .. option:: http_s3_region=str : [http]
 
@@ -2159,6 +2161,11 @@ with the caveat that when used on the command line, they must come after the
 
 	The S3 key/access id.
 
+.. option:: http_swift_auth_token=str : [http]
+
+	The Swift auth token. See the example configuration file on how
+	to retrieve this.
+
 .. option:: http_verbose=int : [http]
 
 	Enable verbose requests from libcurl. Useful for debugging. 1

engines/http.c

@@ -25,25 +25,37 @@
 #include <curl/curl.h>
 #include <openssl/hmac.h>
 #include <openssl/sha.h>
+#include <openssl/md5.h>
 #include "fio.h"
 #include "../optgroup.h"
 
 
+enum {
+	FIO_HTTP_WEBDAV	    = 0,
+	FIO_HTTP_S3	    = 1,
+	FIO_HTTP_SWIFT	    = 2,
+
+	FIO_HTTPS_OFF	    = 0,
+	FIO_HTTPS_ON	    = 1,
+	FIO_HTTPS_INSECURE  = 2,
+};
+
 struct http_data {
 	CURL *curl;
 };
 
 struct http_options {
 	void *pad;
-	int  https;
+	unsigned int https;
 	char *host;
 	char *user;
 	char *pass;
 	char *s3_key;
 	char *s3_keyid;
 	char *s3_region;
+	char *swift_auth_token;
 	int verbose;
-	int s3;
+	unsigned int mode;
 };
 
 struct http_curl_stream {
@@ -56,10 +68,24 @@ static struct fio_option options[] = {
 	{
 		.name     = "https",
 		.lname    = "https",
-		.type     = FIO_OPT_BOOL,
+		.type     = FIO_OPT_STR,
 		.help     = "Enable https",
 		.off1     = offsetof(struct http_options, https),
-		.def      = "0",
+		.def      = "off",
+		.posval = {
+			  { .ival = "off",
+			    .oval = FIO_HTTPS_OFF,
+			    .help = "No HTTPS",
+			  },
+			  { .ival = "on",
+			    .oval = FIO_HTTPS_ON,
+			    .help = "Enable HTTPS",
+			  },
+			  { .ival = "insecure",
+			    .oval = FIO_HTTPS_INSECURE,
+			    .help = "Enable HTTPS, disable peer verification",
+			  },
+		},
 		.category = FIO_OPT_C_ENGINE,
 		.group    = FIO_OPT_G_HTTP,
 	},
@@ -111,6 +137,16 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group    = FIO_OPT_G_HTTP,
 	},
+	{
+		.name     = "http_swift_auth_token",
+		.lname    = "Swift auth token",
+		.type     = FIO_OPT_STR_STORE,
+		.help     = "OpenStack Swift auth token",
+		.off1     = offsetof(struct http_options, swift_auth_token),
+		.def	  = "",
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
 	{
 		.name     = "http_s3_region",
 		.lname    = "S3 region",
@@ -122,18 +158,32 @@ static struct fio_option options[] = {
 		.group    = FIO_OPT_G_HTTP,
 	},
 	{
-		.name     = "http_s3",
-		.lname    = "S3 extensions",
-		.type     = FIO_OPT_BOOL,
-		.help     = "Whether to enable S3 specific headers",
-		.off1     = offsetof(struct http_options, s3),
-		.def	  = "0",
+		.name     = "http_mode",
+		.lname    = "Request mode to use",
+		.type     = FIO_OPT_STR,
+		.help     = "Whether to use WebDAV, Swift, or S3",
+		.off1     = offsetof(struct http_options, mode),
+		.def	  = "webdav",
+		.posval = {
+			  { .ival = "webdav",
+			    .oval = FIO_HTTP_WEBDAV,
+			    .help = "WebDAV server",
+			  },
+			  { .ival = "s3",
+			    .oval = FIO_HTTP_S3,
+			    .help = "S3 storage backend",
+			  },
+			  { .ival = "swift",
+			    .oval = FIO_HTTP_SWIFT,
+			    .help = "OpenStack Swift storage",
+			  },
+		},
 		.category = FIO_OPT_C_ENGINE,
 		.group    = FIO_OPT_G_HTTP,
 	},
 	{
 		.name     = "http_verbose",
-		.lname    = "CURL verbosity",
+		.lname    = "HTTP verbosity level",
 		.type     = FIO_OPT_INT,
 		.help     = "increase http engine verbosity",
 		.off1     = offsetof(struct http_options, verbose),
@@ -204,6 +254,14 @@ static char *_gen_hex_sha256(const char *p, size_t len)
 	return _conv_hex(hash, SHA256_DIGEST_LENGTH);
 }
 
+static char *_gen_hex_md5(const char *p, size_t len)
+{
+	unsigned char hash[MD5_DIGEST_LENGTH];
+
+	MD5((unsigned char*)p, len, hash);
+	return _conv_hex(hash, MD5_DIGEST_LENGTH);
+}
+
 static void _hmac(unsigned char *md, void *key, int key_len, char *data) {
 	HMAC_CTX *ctx;
 	unsigned int hmac_len;
@@ -338,6 +396,29 @@ static void _add_aws_auth_header(CURL *curl, struct curl_slist *slist, struct ht
 	free(signature);
 }
 
+static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_options *o,
+		int op, const char *uri, char *buf, size_t len)
+{
+	char *dsha = NULL;
+	char s[512];
+
+	if (op == DDIR_WRITE) {
+		dsha = _gen_hex_md5(buf, len);
+	}
+	/* Surpress automatic Accept: header */
+	slist = curl_slist_append(slist, "Accept:");
+
+	snprintf(s, sizeof(s), "etag: %s", dsha);
+	slist = curl_slist_append(slist, s);
+
+	snprintf(s, sizeof(s), "x-auth-token: %s", o->swift_auth_token);
+	slist = curl_slist_append(slist, s);
+
+	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist);
+
+	free(dsha);
+}
+
 static void fio_http_cleanup(struct thread_data *td)
 {
 	struct http_data *http = td->io_ops_data;
@@ -402,17 +483,24 @@ static enum fio_q_status fio_http_queue(struct thread_data *td,
 
 	fio_ro_check(td, io_u);
 	memset(&_curl_stream, 0, sizeof(_curl_stream));
-	snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name, io_u->offset, io_u->xfer_buflen);
-	snprintf(url, sizeof(url), "%s://%s%s", o->https ? "https" : "http", o->host, object);
+	snprintf(object, sizeof(object), "%s_%llu_%llu", td->files[0]->file_name,
+		io_u->offset, io_u->xfer_buflen);
+	if (o->https == FIO_HTTPS_OFF)
+		snprintf(url, sizeof(url), "http://%s%s", o->host, object);
+	else
+		snprintf(url, sizeof(url), "https://%s%s", o->host, object);
 	curl_easy_setopt(http->curl, CURLOPT_URL, url);
 	_curl_stream.buf = io_u->xfer_buf;
 	_curl_stream.max = io_u->xfer_buflen;
 	curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
 	curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
 
-	if (o->s3)
+	if (o->mode == FIO_HTTP_S3)
 		_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
 			io_u->xfer_buf, io_u->xfer_buflen);
+	else if (o->mode == FIO_HTTP_SWIFT)
+		_add_swift_header(http->curl, slist, o, io_u->ddir, object,
+			io_u->xfer_buf, io_u->xfer_buflen);
 
 	if (io_u->ddir == DDIR_WRITE) {
 		curl_easy_setopt(http->curl, CURLOPT_READDATA, &_curl_stream);
@@ -503,6 +591,10 @@ static int fio_http_setup(struct thread_data *td)
 	curl_easy_setopt(http->curl, CURLOPT_NOPROGRESS, 1L);
 	curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L);
 	curl_easy_setopt(http->curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP|CURLPROTO_HTTPS);
+	if (o->https == FIO_HTTPS_INSECURE) {
+		curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L);
+		curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L);
+	}
 	curl_easy_setopt(http->curl, CURLOPT_READFUNCTION, _http_read);
 	curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, _http_write);
 	curl_easy_setopt(http->curl, CURLOPT_SEEKFUNCTION, _http_seek);

examples/http-s3.fio

@@ -9,8 +9,8 @@ name=test
 direct=1
 filename=/larsmb-fio-test/object
 http_verbose=0
-https=1
-http_s3=1
+https=on
+http_mode=s3
 http_s3_key=${S3_KEY}
 http_s3_keyid=${S3_ID}
 http_host=s3.eu-central-1.amazonaws.com

examples/http-swift.fio

@@ -0,0 +1,32 @@
+[global]
+ioengine=http
+rw=randwrite
+name=test
+direct=1
+http_verbose=0
+http_mode=swift
+https=on
+# This is the hostname and port portion of the public access link for
+# the container:
+http_host=swift.srv.openstack.local:8081
+filename_format=/swift/v1/fio-test/bucket.$jobnum
+group_reporting
+bs=64k
+size=1M
+# Currently, fio cannot yet generate the Swift Auth-Token itself.
+# You need to set this prior to running fio via
+# eval $(openstack token issue -f shell --prefix SWIFT_) ; export SWIFT_id
+http_swift_auth_token=${SWIFT_id}
+
+[create]
+numjobs=1
+rw=randwrite
+io_size=256k
+verify=sha256
+
+# This will delete all created objects again
+[trim]
+stonewall
+numjobs=1
+rw=trim
+io_size=64k

examples/http-webdav.fio

@@ -4,8 +4,8 @@ rw=randwrite
 name=test
 direct=1
 http_verbose=0
-http_s3=0
-https=0
+http_mode=webdav
+https=off
 http_host=localhost
 filename_format=/dav/bucket.$jobnum
 group_reporting

fio.1

@@ -1829,12 +1829,14 @@ Username for HTTP authentication.
 .BI (http)http_pass \fR=\fPstr
 Password for HTTP authentication.
 .TP
-.BI (http)https \fR=\fPbool
-Whether to use HTTPS instead of plain HTTP. Default is \fB0\fR.
+.BI (http)https \fR=\fPstr
+Whether to use HTTPS instead of plain HTTP. \fRon\fP enables HTTPS;
+\fRinsecure\fP will enable HTTPS, but disable SSL peer verification (use
+with caution!).  Default is \fBoff\fR.
 .TP
-.BI (http)http_s3 \fR=\fPbool
-Include S3 specific HTTP headers such as authenticating requests with
-AWS Signature Version 4. Default is \fB0\fR.
+.BI (http)http_mode \fR=\fPstr
+Which HTTP access mode to use: webdav, swift, or s3. Default is
+\fBwebdav\fR.
 .TP
 .BI (http)http_s3_region \fR=\fPstr
 The S3 region/zone to include in the request. Default is \fBus-east-1\fR.
@@ -1845,6 +1847,10 @@ The S3 secret key.
 .BI (http)http_s3_keyid \fR=\fPstr
 The S3 key/access id.
 .TP
+.BI (http)http_swift_auth_token \fR=\fPstr
+The Swift auth token. See the example configuration file on how to
+retrieve this.
+.TP
 .BI (http)http_verbose \fR=\fPint
 Enable verbose requests from libcurl. Useful for debugging. 1 turns on
 verbose logging from libcurl, 2 additionally enables HTTP IO tracing.