From e4573b9d2024fd2f1be41090846406fea3fc02cf Mon Sep 17 00:00:00 2001 From: Connor Manning Date: Tue, 20 Feb 2018 13:15:45 -0600 Subject: [PATCH] Update arbiter bundle. --- vendor/arbiter/arbiter.cpp | 749 +++++++++++++++++++++++++++++++------ vendor/arbiter/arbiter.hpp | 227 +++++++++-- 2 files changed, 834 insertions(+), 142 deletions(-) diff --git a/vendor/arbiter/arbiter.cpp b/vendor/arbiter/arbiter.cpp index 797358fdca..5c28ad0a3a 100644 --- a/vendor/arbiter/arbiter.cpp +++ b/vendor/arbiter/arbiter.cpp @@ -86,13 +86,22 @@ namespace if (!b.isNull()) { - for (const auto& key : b.getMemberNames()) + if (b.isObject()) { - // If A doesn't have this key, then set it to B's value. - // If A has the key but it's an object, then recursively merge. - // Otherwise A already has a value here that we won't overwrite. - if (!out.isMember(key)) out[key] = b[key]; - else if (out[key].isObject()) merge(out[key], b[key]); + for (const auto& key : b.getMemberNames()) + { + // If A doesn't have this key, then set it to B's value. + // If A has the key but it's an object, then recursively + // merge. + // Otherwise A already has a value here that we won't + // overwrite. + if (!out.isMember(key)) out[key] = b[key]; + else if (out[key].isObject()) merge(out[key], b[key]); + } + } + else + { + out = b; } } @@ -142,24 +151,19 @@ Arbiter::Arbiter(const Json::Value& in) auto https(Https::create(*m_pool, json["http"])); if (https) m_drivers[https->type()] = std::move(https); - if (json["s3"].isArray()) - { - for (const auto& sub : json["s3"]) - { - auto s3(S3::create(*m_pool, sub)); - m_drivers[s3->type()] = std::move(s3); - } - } - else - { - auto s3(S3::create(*m_pool, json["s3"])); - if (s3) m_drivers[s3->type()] = std::move(s3); - } + auto s3(S3::create(*m_pool, json["s3"])); + for (auto& s : s3) m_drivers[s->type()] = std::move(s); // Credential-based drivers should probably all do something similar to the // S3 driver to support multiple profiles. auto dropbox(Dropbox::create(*m_pool, json["dropbox"])); if (dropbox) m_drivers[dropbox->type()] = std::move(dropbox); + +#ifdef ARBITER_OPENSSL + auto google(Google::create(*m_pool, json["gs"])); + if (google) m_drivers[google->type()] = std::move(google); +#endif + #endif } @@ -308,7 +312,7 @@ void Arbiter::copy( { std::cout << ++i << " / " << paths.size() << ": " << - path << " -> " << dstEndpoint.fullPath(subpath) << + path << " -> " << dstEndpoint.prefixedFullPath(subpath) << std::endl; } @@ -775,6 +779,41 @@ void Endpoint::put( getHttpDriver().put(path, data, headers, query); } +http::Response Endpoint::httpGet( + std::string path, + http::Headers headers, + http::Query query, + const std::size_t reserve) const +{ + return getHttpDriver().internalGet(fullPath(path), headers, query, reserve); +} + +http::Response Endpoint::httpPut( + std::string path, + const std::vector& data, + http::Headers headers, + http::Query query) const +{ + return getHttpDriver().internalPut(fullPath(path), data, headers, query); +} + +http::Response Endpoint::httpHead( + std::string path, + http::Headers headers, + http::Query query) const +{ + return getHttpDriver().internalHead(fullPath(path), headers, query); +} + +http::Response Endpoint::httpPost( + std::string path, + const std::vector& data, + http::Headers headers, + http::Query query) const +{ + return getHttpDriver().internalPost(fullPath(path), data, headers, query); +} + std::string Endpoint::fullPath(const std::string& subpath) const { return m_root + subpath; @@ -1347,13 +1386,38 @@ void Http::put( } } +void Http::post( + const std::string path, + const std::string& data, + const Headers h, + const Query q) const +{ + return post(path, std::vector(data.begin(), data.end()), h, q); +} + +void Http::post( + const std::string path, + const std::vector& data, + const Headers headers, + const Query query) const +{ + auto http(m_pool.acquire()); + auto res(http.post(typedPath(path), data, headers, query)); + + if (!res.ok()) + { + std::cout << res.str() << std::endl; + throw ArbiterError("Couldn't HTTP POST to " + path); + } +} + Response Http::internalGet( const std::string path, const Headers headers, const Query query, const std::size_t reserve) const { - return m_pool.acquire().get(path, headers, query, reserve); + return m_pool.acquire().get(typedPath(path), headers, query, reserve); } Response Http::internalPut( @@ -1362,7 +1426,7 @@ Response Http::internalPut( const Headers headers, const Query query) const { - return m_pool.acquire().put(path, data, headers, query); + return m_pool.acquire().put(typedPath(path), data, headers, query); } Response Http::internalHead( @@ -1370,16 +1434,26 @@ Response Http::internalHead( const Headers headers, const Query query) const { - return m_pool.acquire().head(path, headers, query); + return m_pool.acquire().head(typedPath(path), headers, query); } Response Http::internalPost( const std::string path, const std::vector& data, - const Headers headers, + Headers headers, const Query query) const { - return m_pool.acquire().post(path, data, headers, query); + if (!headers.count("Content-Length")) + { + headers["Content-Length"] = std::to_string(data.size()); + } + return m_pool.acquire().post(typedPath(path), data, headers, query); +} + +std::string Http::typedPath(const std::string& p) const +{ + if (Arbiter::getType(p) != "file") return p; + else return type() + "://" + p; } } // namespace drivers @@ -1405,7 +1479,6 @@ Response Http::internalPost( // ////////////////////////////////////////////////////////////////////// #ifndef ARBITER_IS_AMALGAMATION -#include #include #endif @@ -1449,7 +1522,7 @@ namespace // See: // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html - const std::string credIp("http://169.254.169.254/"); + const std::string credIp("169.254.169.254/"); const std::string credBase( credIp + "latest/meta-data/iam/security-credentials/"); @@ -1516,30 +1589,51 @@ S3::S3( , m_config(std::move(config)) { } -std::unique_ptr S3::create(Pool& pool, const Json::Value& json) +std::vector> S3::create(Pool& pool, const Json::Value& json) +{ + std::vector> result; + + if (json.isArray()) + { + for (const auto& curr : json) + { + if (auto s = createOne(pool, curr)) + { + result.push_back(std::move(s)); + } + } + } + else if (auto s = createOne(pool, json)) + { + result.push_back(std::move(s)); + } + + return result; +} + +std::unique_ptr S3::createOne(Pool& pool, const Json::Value& json) { const std::string profile(extractProfile(json)); auto auth(Auth::create(json, profile)); if (!auth) return std::unique_ptr(); - auto config(Config::create(json, profile)); - if (!config) return std::unique_ptr(); - + std::unique_ptr config(new Config(json, profile)); return makeUnique(pool, profile, std::move(auth), std::move(config)); } std::string S3::extractProfile(const Json::Value& json) { - if (auto p = util::env("AWS_PROFILE")) return *p; - else if (auto p = util::env("AWS_DEFAULT_PROFILE")) return *p; - else if ( + if ( !json.isNull() && json.isMember("profile") && json["profile"].asString().size()) { return json["profile"].asString(); } + + if (auto p = util::env("AWS_PROFILE")) return *p; + if (auto p = util::env("AWS_DEFAULT_PROFILE")) return *p; else return "default"; } @@ -1547,38 +1641,41 @@ std::unique_ptr S3::Auth::create( const Json::Value& json, const std::string profile) { - // Try environment settings first. + // Try explicit JSON configuration first. + if ( + !json.isNull() && + json.isMember("access") && + (json.isMember("secret") || json.isMember("hidden"))) + { + return makeUnique( + json["access"].asString(), + json.isMember("secret") ? + json["secret"].asString() : + json["hidden"].asString(), + json["token"].asString()); + } + + // Try environment settings next. { auto access(util::env("AWS_ACCESS_KEY_ID")); auto hidden(util::env("AWS_SECRET_ACCESS_KEY")); + auto token(util::env("AWS_SESSION_TOKEN")); if (access && hidden) { - return makeUnique(*access, *hidden); + return makeUnique(*access, *hidden, token ? *token : ""); } access = util::env("AMAZON_ACCESS_KEY_ID"); hidden = util::env("AMAZON_SECRET_ACCESS_KEY"); + token = util::env("AMAZON_SESSION_TOKEN"); if (access && hidden) { - return makeUnique(*access, *hidden); + return makeUnique(*access, *hidden, token ? *token : ""); } } - // Try explicit JSON configuration next. - if ( - !json.isNull() && - json.isMember("access") && - (json.isMember("secret") || json.isMember("hidden"))) - { - return makeUnique( - json["access"].asString(), - json.isMember("secret") ? - json["secret"].asString() : - json["hidden"].asString()); - } - const std::string credPath( util::env("AWS_CREDENTIAL_FILE") ? *util::env("AWS_CREDENTIAL_FILE") : "~/.aws/credentials"); @@ -1627,31 +1724,21 @@ std::unique_ptr S3::Auth::create( } S3::Config::Config( - const std::string region, - const std::string baseUrl, - const bool sse, - const bool precheck) - : m_region(region) - , m_baseUrl(baseUrl) - , m_precheck(precheck) -{ - if (sse) - { - // This could grow to support other SSE schemes, like KMS and customer- - // supplied keys. - m_baseHeaders["x-amz-server-side-encryption"] = "AES256"; - } -} - -std::unique_ptr S3::Config::create( const Json::Value& json, const std::string profile) + : m_region(extractRegion(json, profile)) + , m_baseUrl(extractBaseUrl(json, m_region)) + , m_precheck(json["precheck"].asBool()) { - const auto region(extractRegion(json, profile)); - const auto baseUrl(extractBaseUrl(json, region)); - const bool sse(json["sse"].asBool()); - const bool precheck(json["precheck"].asBool()); - return makeUnique(region, baseUrl, sse, precheck); + if (json["sse"].asBool()) + { + m_baseHeaders["x-amz-server-side-encryption"] = "AES256"; + } + + if (json["requesterPays"].asBool()) + { + m_baseHeaders["x-amz-request-payer"] = "requester"; + } } std::string S3::Config::extractRegion( @@ -1769,7 +1856,10 @@ S3::AuthFields S3::Auth::fields() const m_access = creds["AccessKeyId"].asString(); m_hidden = creds["SecretAccessKey"].asString(); m_token = creds["Token"].asString(); - m_expiration.reset(new Time(creds["Expiration"].asString(), arbiter::Time::iso8601)); + m_expiration.reset( + new Time( + creds["Expiration"].asString(), + arbiter::Time::iso8601)); if (*m_expiration - now < reauthSeconds) { @@ -1806,7 +1896,8 @@ std::unique_ptr S3::tryGetSize(std::string rawPath) const Headers(), empty); - Response res(Http::internalHead(resource.url(), apiV4.headers())); + drivers::Http http(m_pool); + Response res(http.internalHead(resource.url(), apiV4.headers())); if (res.ok() && res.headers().count("Content-Length")) { @@ -1820,9 +1911,13 @@ std::unique_ptr S3::tryGetSize(std::string rawPath) const bool S3::get( const std::string rawPath, std::vector& data, - const Headers headers, + const Headers userHeaders, const Query query) const { + Headers headers(m_config->baseHeaders()); + headers.erase("x-amz-server-side-encryption"); + headers.insert(userHeaders.begin(), userHeaders.end()); + std::unique_ptr size( m_config->precheck() && !headers.count("Range") ? tryGetSize(rawPath) : nullptr); @@ -1837,8 +1932,9 @@ bool S3::get( headers, empty); + drivers::Http http(m_pool); Response res( - Http::internalGet( + http.internalGet( resource.url(), apiV4.headers(), apiV4.query(), @@ -1851,8 +1947,7 @@ bool S3::get( } else { - std::cout << std::string(res.data().data(), res.data().size()) << - std::endl; + std::cout << res.code() << ": " << res.str() << std::endl; return false; } } @@ -1877,8 +1972,9 @@ void S3::put( headers, data); + drivers::Http http(m_pool); Response res( - Http::internalPut( + http.internalPut( resource.url(), data, apiV4.headers(), @@ -1968,7 +2064,8 @@ std::vector S3::glob(std::string path, bool verbose) const // beyond the prefix if recursive is true. if (recursive || !isSubdir) { - results.push_back("s3://" + bucket + "/" + key); + results.push_back( + type() + "://" + bucket + "/" + key); } if (more) @@ -2147,8 +2244,8 @@ std::string S3::ApiV4::getAuthHeader( "Signature=" + signature; } -S3::Resource::Resource(std::string baseUrl, std::string fullPath) - : m_baseUrl(baseUrl) +S3::Resource::Resource(std::string base, std::string fullPath) + : m_baseUrl(base) , m_bucket() , m_object() , m_virtualHosted(true) @@ -2159,15 +2256,29 @@ S3::Resource::Resource(std::string baseUrl, std::string fullPath) m_bucket = fullPath.substr(0, split); if (split != std::string::npos) m_object = fullPath.substr(split + 1); - m_virtualHosted = m_bucket.find_first_of('.') == std::string::npos; + // Always use virtual-host style paths. We'll use HTTP for our back-end + // calls to allow this. If we were to use HTTPS on the back-end, then we + // would have to use non-virtual-hosted paths if the bucket name contained + // '.' characters. + // + // m_virtualHosted = m_bucket.find_first_of('.') == std::string::npos; +} + +std::string S3::Resource::baseUrl() const +{ + return m_baseUrl; +} + +std::string S3::Resource::bucket() const +{ + return m_virtualHosted ? m_bucket : ""; } std::string S3::Resource::url() const { - // We can't use virtual-host style paths if the bucket contains dots. if (m_virtualHosted) { - return "https://" + m_bucket + "." + m_baseUrl + m_object; + return "http://" + m_bucket + "." + m_baseUrl + m_object; } else { @@ -2212,6 +2323,389 @@ std::string S3::Resource::host() const +// ////////////////////////////////////////////////////////////////////// +// Beginning of content of file: arbiter/drivers/google.cpp +// ////////////////////////////////////////////////////////////////////// + +#ifndef ARBITER_IS_AMALGAMATION +#include +#endif + +#include + +#ifdef ARBITER_OPENSSL +#include +#include +#include +#include +#include +#endif + +#ifndef ARBITER_IS_AMALGAMATION +#include +#include +#include +#endif + +#ifdef ARBITER_CUSTOM_NAMESPACE +namespace ARBITER_CUSTOM_NAMESPACE +{ +#endif + +namespace arbiter +{ + +namespace +{ + std::mutex sslMutex; + + const std::string baseGoogleUrl("www.googleapis.com/storage/v1/"); + const std::string uploadUrl("www.googleapis.com/upload/storage/v1/"); + const http::Query altMediaQuery{ { "alt", "media" } }; + + class GResource + { + public: + GResource(std::string path) + { + const std::size_t split(path.find("/")); + m_bucket = path.substr(0, split) + "/"; + if (split != std::string::npos) m_object = path.substr(split + 1); + } + + const std::string& bucket() const { return m_bucket; } + const std::string& object() const { return m_object; } + std::string endpoint() const + { + // https://cloud.google.com/storage/docs/json_api/#encoding + static const std::string exclusions("!$&'()*+,;=:@"); + + // https://cloud.google.com/storage/docs/json_api/v1/ + return + baseGoogleUrl + "b/" + bucket() + + "o/" + http::sanitize(object(), exclusions); + } + + std::string uploadEndpoint() const + { + return uploadUrl + "b/" + bucket() + "o"; + } + + std::string listEndpoint() const + { + return baseGoogleUrl + "b/" + bucket() + "o"; + } + + private: + std::string m_bucket; + std::string m_object; + + }; +} // unnamed namespace + +namespace drivers +{ + +Google::Google(http::Pool& pool, std::unique_ptr auth) + : Https(pool) + , m_auth(std::move(auth)) +{ } + +std::unique_ptr Google::create( + http::Pool& pool, + const Json::Value& json) +{ + if (auto auth = Auth::create(json)) + { + return util::makeUnique(pool, std::move(auth)); + } + + return std::unique_ptr(); +} + +std::unique_ptr Google::tryGetSize(const std::string path) const +{ + http::Headers headers(m_auth->headers()); + const GResource resource(path); + + drivers::Https https(m_pool); + const auto res( + https.internalHead(resource.endpoint(), headers, altMediaQuery)); + + if (res.ok() && res.headers().count("Content-Length")) + { + const auto& s(res.headers().at("Content-Length")); + return util::makeUnique(std::stoul(s)); + } + + return std::unique_ptr(); +} + +bool Google::get( + const std::string path, + std::vector& data, + const http::Headers userHeaders, + const http::Query query) const +{ + http::Headers headers(m_auth->headers()); + headers.insert(userHeaders.begin(), userHeaders.end()); + const GResource resource(path); + + drivers::Https https(m_pool); + const auto res( + https.internalGet(resource.endpoint(), headers, altMediaQuery)); + + if (res.ok()) + { + data = res.data(); + return true; + } + else + { + std::cout << + "Failed get - " << res.code() << ": " << res.str() << std::endl; + return false; + } +} + +void Google::put( + const std::string path, + const std::vector& data, + const http::Headers userHeaders, + const http::Query userQuery) const +{ + const GResource resource(path); + const std::string url(resource.uploadEndpoint()); + + http::Headers headers(m_auth->headers()); + headers["Expect"] = ""; + headers.insert(userHeaders.begin(), userHeaders.end()); + + http::Query query(userQuery); + query["uploadType"] = "media"; + query["name"] = resource.object(); + + drivers::Https https(m_pool); + const auto res(https.internalPost(url, data, headers, query)); +} + +std::vector Google::glob(std::string path, bool verbose) const +{ + std::vector results; + + path.pop_back(); + const bool recursive(path.back() == '*'); + if (recursive) path.pop_back(); + + const GResource resource(path); + const std::string url(resource.listEndpoint()); + std::string pageToken; + + drivers::Https https(m_pool); + http::Query query; + + // When the delimiter is set to "/", then the response will contain a + // "prefixes" key in addition to the "items" key. The "prefixes" key will + // contain the directories found, which we will ignore. + if (!recursive) query["delimiter"] = "/"; + if (resource.object().size()) query["prefix"] = resource.object(); + + do + { + if (pageToken.size()) query["pageToken"] = pageToken; + + const auto res(https.internalGet(url, m_auth->headers(), query)); + + if (!res.ok()) + { + throw ArbiterError(std::to_string(res.code()) + ": " + res.str()); + } + + const Json::Value json(util::parse(res.str())); + for (const auto& item : json["items"]) + { + results.push_back( + type() + "://" + + resource.bucket() + item["name"].asString()); + } + + pageToken = json["nextPageToken"].asString(); + } while (pageToken.size()); + + return results; +} + +/////////////////////////////////////////////////////////////////////////////// + +std::unique_ptr Google::Auth::create(const Json::Value& json) +{ + if (auto path = util::env("GOOGLE_APPLICATION_CREDENTIALS")) + { + if (const auto file = drivers::Fs().tryGet(*path)) + { + return util::makeUnique(util::parse(*file)); + } + } + else if (json.isString()) + { + const auto path = json.asString(); + if (const auto file = drivers::Fs().tryGet(path)) + { + return util::makeUnique(util::parse(*file)); + } + } + else if (json.isObject()) + { + return util::makeUnique(json); + } + + return std::unique_ptr(); +} + +Google::Auth::Auth(const Json::Value& creds) + : m_creds(creds) +{ + maybeRefresh(); +} + +http::Headers Google::Auth::headers() const +{ + std::lock_guard lock(m_mutex); + maybeRefresh(); + return m_headers; +} + +void Google::Auth::maybeRefresh() const +{ + using namespace crypto; + + const auto now(Time().asUnix()); + if (m_expiration - now > 120) return; // Refresh when under 2 mins left. + + // https://developers.google.com/identity/protocols/OAuth2ServiceAccount + Json::Value h; + h["alg"] = "RS256"; + h["typ"] = "JWT"; + + Json::Value c; + c["iss"] = m_creds["client_email"].asString(); + c["scope"] = "https://www.googleapis.com/auth/devstorage.read_write"; + c["aud"] = "https://www.googleapis.com/oauth2/v4/token"; + c["iat"] = Json::Int64(now); + c["exp"] = Json::Int64(now + 3600); + + const std::string header(encodeBase64(util::toFastString(h))); + const std::string claims(encodeBase64(util::toFastString(c))); + + const std::string key(m_creds["private_key"].asString()); + const std::string signature( + http::sanitize(encodeBase64(sign(header + '.' + claims, key)))); + + const std::string assertion(header + '.' + claims + '.' + signature); + const std::string sbody = + "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&" + "assertion=" + assertion; + const std::vector body(sbody.begin(), sbody.end()); + + const http::Headers headers { { "Expect", "" } }; + const std::string tokenRequestUrl("www.googleapis.com/oauth2/v4/token"); + + http::Pool pool; + drivers::Https https(pool); + const auto res(https.internalPost(tokenRequestUrl, body, headers)); + + if (!res.ok()) throw ArbiterError("Failed to get token: " + res.str()); + + const Json::Value token(util::parse(res.str())); + m_headers["Authorization"] = "Bearer " + token["access_token"].asString(); + m_expiration = now + token["expires_in"].asInt64(); +} + +std::string Google::Auth::sign( + const std::string data, + const std::string pkey) const +{ + std::string signature; + +#ifdef ARBITER_OPENSSL + std::lock_guard lock(sslMutex); + + auto loadKey([](std::string s, bool isPublic)->EVP_PKEY* + { + // BIO_new_mem_buf needs non-const char*, so use a vector. + std::vector vec(s.data(), s.data() + s.size()); + + EVP_PKEY* key(nullptr); + if (BIO* bio = BIO_new_mem_buf(vec.data(), vec.size())) + { + if (isPublic) + { + key = PEM_read_bio_PUBKEY(bio, &key, nullptr, nullptr); + } + else + { + key = PEM_read_bio_PrivateKey(bio, &key, nullptr, nullptr); + } + + BIO_free(bio); + + if (!key) + { + std::vector err(256, 0); + ERR_error_string(ERR_get_error(), err.data()); + throw ArbiterError( + std::string("Could not load key: ") + err.data()); + } + } + + return key; + }); + + + EVP_PKEY* key(loadKey(pkey, false)); + + EVP_MD_CTX ctx; + EVP_MD_CTX_init(&ctx); + EVP_DigestSignInit(&ctx, nullptr, EVP_sha256(), nullptr, key); + + if (EVP_DigestSignUpdate(&ctx, data.data(), data.size()) == 1) + { + std::size_t size(0); + if (EVP_DigestSignFinal(&ctx, nullptr, &size) == 1) + { + std::vector v(size, 0); + if (EVP_DigestSignFinal(&ctx, v.data(), &size) == 1) + { + signature.assign(reinterpret_cast(v.data()), size); + } + } + } + + EVP_MD_CTX_cleanup(&ctx); + if (signature.empty()) throw ArbiterError("Could not sign JWT"); + return signature; +#else + throw ArbiterError("Cannot use google driver without OpenSSL"); +#endif +} + +} // namespace drivers +} // namespace arbiter + +#ifdef ARBITER_CUSTOM_NAMESPACE +} +#endif + + +// ////////////////////////////////////////////////////////////////////// +// End of content of file: arbiter/drivers/google.cpp +// ////////////////////////////////////////////////////////////////////// + + + + + + // ////////////////////////////////////////////////////////////////////// // Beginning of content of file: arbiter/drivers/dropbox.cpp // ////////////////////////////////////////////////////////////////////// @@ -2253,8 +2747,9 @@ namespace arbiter namespace { - const std::string baseGetUrl("https://content.dropboxapi.com/"); - const std::string getUrl(baseGetUrl + "2/files/download"); + const std::string baseDropboxUrl("https://content.dropboxapi.com/"); + const std::string getUrl(baseDropboxUrl + "2/files/download"); + const std::string putUrl(baseDropboxUrl + "2/files/upload"); const std::string listUrl("https://api.dropboxapi.com/2/files/list_folder"); const std::string metaUrl("https://api.dropboxapi.com/2/files/get_metadata"); @@ -2281,6 +2776,7 @@ namespace drivers { using namespace http; +using namespace util; Dropbox::Dropbox(Pool& pool, const Dropbox::Auth& auth) : Http(pool) @@ -2289,14 +2785,19 @@ Dropbox::Dropbox(Pool& pool, const Dropbox::Auth& auth) std::unique_ptr Dropbox::create(Pool& pool, const Json::Value& json) { - std::unique_ptr dropbox; - - if (!json.isNull() && json.isMember("token")) + if (!json.isNull()) { - dropbox.reset(new Dropbox(pool, Auth(json["token"].asString()))); + if (json.isObject() && json.isMember("token")) + { + return makeUnique(pool, Auth(json["token"].asString())); + } + else if (json.isString()) + { + return makeUnique(pool, Auth(json.asString())); + } } - return dropbox; + return std::unique_ptr(); } Headers Dropbox::httpGetHeaders() const @@ -2432,10 +2933,23 @@ bool Dropbox::get( void Dropbox::put( const std::string rawPath, const std::vector& data, - const Headers headers, + const Headers userHeaders, const Query query) const { - throw ArbiterError("PUT not yet supported for " + type()); + const std::string path(sanitize(rawPath)); + + Headers headers(httpGetHeaders()); + + Json::Value json; + json["path"] = std::string("/" + path); + headers["Dropbox-API-Arg"] = toSanitizedString(json); + headers["Content-Type"] = "application/octet-stream"; + + headers.insert(userHeaders.begin(), userHeaders.end()); + + const Response res(Http::internalPost(putUrl, data, headers, query)); + + if (!res.ok()) throw ArbiterError(res.str()); } std::string Dropbox::continueFileInfo(std::string cursor) const @@ -2464,19 +2978,22 @@ std::string Dropbox::continueFileInfo(std::string cursor) const return std::string(""); } -std::vector Dropbox::glob(std::string rawPath, bool verbose) const +std::vector Dropbox::glob(std::string path, bool verbose) const { std::vector results; - const std::string path(sanitize(rawPath.substr(0, rawPath.size() - 2))); + path.pop_back(); + const bool recursive(path.back() == '*'); + if (recursive) path.pop_back(); + if (path.back() == '/') path.pop_back(); - auto listPath = [this](std::string path)->std::string + auto listPath = [this, recursive](std::string path)->std::string { Headers headers(httpPostHeaders()); Json::Value request; request["path"] = std::string("/" + path); - request["recursive"] = false; + request["recursive"] = recursive; request["include_media_info"] = false; request["include_deleted"] = false; @@ -2507,7 +3024,8 @@ std::vector Dropbox::glob(std::string rawPath, bool verbose) const bool more(false); std::string cursor(""); - auto processPath = [verbose, &results, &more, &cursor](std::string data) + auto processPath = + [this, verbose, &results, &more, &cursor](std::string data) { if (data.empty()) return; @@ -2540,7 +3058,7 @@ std::vector Dropbox::glob(std::string rawPath, bool verbose) const if (std::equal(tag.begin(), tag.end(), fileTag.begin(), ins)) { // Results already begin with a slash. - results.push_back("dropbox:/" + v["path_lower"].asString()); + results.push_back(type() + ":/" + v["path_lower"].asString()); } } }; @@ -3816,7 +4334,7 @@ namespace const std::string hexVals("0123456789abcdef"); } // unnamed namespace -std::string encodeBase64(const std::vector& data) +std::string encodeBase64(const std::vector& data, const bool pad) { std::vector input; for (std::size_t i(0); i < data.size(); ++i) @@ -3825,17 +4343,18 @@ std::string encodeBase64(const std::vector& data) input.push_back(*reinterpret_cast(&c)); } - std::size_t fullSteps(input.size() / 3); + const std::size_t fullSteps(data.size() / 3); + const std::size_t remainder(data.size() % 3); + while (input.size() % 3) input.push_back(0); uint8_t* pos(input.data()); - uint8_t* end(input.data() + fullSteps * 3); std::string output(fullSteps * 4, '_'); std::size_t outIndex(0); const uint32_t mask(0x3F); - while (pos != end) + for (std::size_t i(0); i < fullSteps; ++i) { uint32_t chunk((*pos) << 16 | *(pos + 1) << 8 | *(pos + 2)); @@ -3847,24 +4366,26 @@ std::string encodeBase64(const std::vector& data) pos += 3; } - if (end != input.data() + input.size()) + if (remainder) { - const std::size_t num(pos - end == 1 ? 2 : 3); uint32_t chunk(*(pos) << 16 | *(pos + 1) << 8 | *(pos + 2)); output.push_back(base64Vals[(chunk >> 18) & mask]); output.push_back(base64Vals[(chunk >> 12) & mask]); - if (num == 3) output.push_back(base64Vals[(chunk >> 6) & mask]); - } + if (remainder == 2) output.push_back(base64Vals[(chunk >> 6) & mask]); - while (output.size() % 4) output.push_back('='); + if (pad) + { + while (output.size() % 4) output.push_back('='); + } + } return output; } -std::string encodeBase64(const std::string& input) +std::string encodeBase64(const std::string& input, const bool pad) { - return encodeBase64(std::vector(input.begin(), input.end())); + return encodeBase64(std::vector(input.begin(), input.end()), pad); } std::string encodeAsHex(const std::vector& input) @@ -4017,6 +4538,12 @@ int64_t Time::operator-(const Time& other) const return std::difftime(m_time, other.m_time); } +int64_t Time::asUnix() const +{ + static const Time epoch("1970-01-01T00:00:00Z"); + return *this - epoch; +} + } // namespace arbiter #ifdef ARBITER_CUSTOM_NAMESPACE diff --git a/vendor/arbiter/arbiter.hpp b/vendor/arbiter/arbiter.hpp index a67730dae2..3710ae6edd 100644 --- a/vendor/arbiter/arbiter.hpp +++ b/vendor/arbiter/arbiter.hpp @@ -1,7 +1,7 @@ /// Arbiter amalgamated header (https://github.com/connormanning/arbiter). /// It is intended to be used with #include "arbiter.hpp" -// Git SHA: 311b81e3dcb5e8c55b2148f7eba46a12cff66914 +// Git SHA: e3fead2f855dc2d3193cd3bb784f7d90b5d5daff // ////////////////////////////////////////////////////////////////////// // Beginning of content of file: LICENSE @@ -153,6 +153,10 @@ class Response std::vector data() const { return m_data; } const Headers& headers() const { return m_headers; } + std::string str() const + { + return std::string(data().data(), data().size()); + } private: int m_code; @@ -210,6 +214,8 @@ class Response #ifdef ARBITER_CURL #include +#else +typedef void CURL; #endif struct curl_slist; @@ -268,7 +274,7 @@ class ARBITER_DLL Curl Curl(const Curl&); Curl& operator=(const Curl&); - void* m_curl = nullptr; + CURL* m_curl = nullptr; curl_slist* m_headers = nullptr; bool m_verbose = false; @@ -523,6 +529,7 @@ class ARBITER_DLL Time // Return value is in seconds. int64_t operator-(const Time& other) const; + int64_t asUnix() const; private: std::time_t m_time; @@ -803,6 +810,8 @@ class ARBITER_DLL Fs : public Driver public: Fs() { } + using Driver::get; + static std::unique_ptr create(const Json::Value& json); virtual std::string type() const override { return "file"; } @@ -950,13 +959,14 @@ class ARBITER_DLL Http : public Driver http::Headers headers, http::Query query) const; -protected: - /** HTTP-derived Drivers should override this version of GET to allow for - * custom headers and query parameters. - */ - virtual bool get( + void post( std::string path, - std::vector& data, + const std::string& data, + http::Headers headers, + http::Query query) const; + void post( + std::string path, + const std::vector& data, http::Headers headers, http::Query query) const; @@ -986,6 +996,18 @@ class ARBITER_DLL Http : public Driver http::Headers headers = http::Headers(), http::Query query = http::Query()) const; +protected: + /** HTTP-derived Drivers should override this version of GET to allow for + * custom headers and query parameters. + */ + virtual bool get( + std::string path, + std::vector& data, + http::Headers headers, + http::Query query) const; + + http::Pool& m_pool; + private: virtual bool get( std::string path, @@ -994,12 +1016,7 @@ class ARBITER_DLL Http : public Driver return get(path, data, http::Headers(), http::Query()); } - std::string typedPath(const std::string& p) const - { - return type() + "://" + p; - } - - http::Pool& m_pool; + std::string typedPath(const std::string& p) const; }; /** @brief HTTPS driver. Identical to the HTTP driver except for its type @@ -3850,8 +3867,8 @@ namespace arbiter namespace crypto { -std::string encodeBase64(const std::vector& data); -std::string encodeBase64(const std::string& data); +std::string encodeBase64(const std::vector& data, bool pad = true); +std::string encodeBase64(const std::string& data, bool pad = true); std::string encodeAsHex(const std::vector& data); std::string encodeAsHex(const std::string& data); @@ -3887,6 +3904,11 @@ std::string encodeAsHex(const std::string& data); #ifndef ARBITER_IS_AMALGAMATION #include + +#ifndef ARBITER_EXTERNAL_JSON +#include +#endif + #endif @@ -4056,6 +4078,26 @@ namespace util if (t) return makeUnique(*t); else return std::unique_ptr(); } + + inline Json::Value parse(const std::string& s) + { + Json::Reader reader; + Json::Value json; + if (!reader.parse(s, json)) + { + throw std::runtime_error( + "Parse failure: " + reader.getFormattedErrorMessages()); + } + return json; + } + + inline std::string toFastString(const Json::Value& json) + { + std::string s = Json::FastWriter().write(json); + s.pop_back(); // Strip trailing newline. + return s; + } + } // namespace util } // namespace arbiter @@ -4124,7 +4166,11 @@ class S3 : public Http * `~/.aws/credentials` or the file at AWS_CREDENTIAL_FILE. * - EC2 instance profile. */ - static std::unique_ptr create( + static std::vector> create( + http::Pool& pool, + const Json::Value& json); + + static std::unique_ptr createOne( http::Pool& pool, const Json::Value& json); @@ -4189,9 +4235,10 @@ class S3::AuthFields class S3::Auth { public: - Auth(std::string access, std::string hidden) + Auth(std::string access, std::string hidden, std::string token = "") : m_access(access) , m_hidden(hidden) + , m_token(token) { } Auth(std::string iamRole) @@ -4217,11 +4264,7 @@ class S3::Auth class S3::Config { public: - Config(std::string region, std::string baseUrl, bool sse, bool precheck); - - static std::unique_ptr create( - const Json::Value& json, - std::string profile); + Config(const Json::Value& json, std::string profile); const std::string& region() const { return m_region; } const std::string& baseUrl() const { return m_baseUrl; } @@ -4252,8 +4295,8 @@ class S3::Resource std::string url() const; std::string host() const; - std::string baseUrl() const { return m_baseUrl; } - std::string bucket() const { return m_bucket; } + std::string baseUrl() const; + std::string bucket() const; std::string object() const; private: @@ -4327,6 +4370,103 @@ class S3::ApiV4 +// ////////////////////////////////////////////////////////////////////// +// Beginning of content of file: arbiter/drivers/google.hpp +// ////////////////////////////////////////////////////////////////////// + +#pragma once + +#ifndef ARBITER_IS_AMALGAMATION +#include +#endif + +#include + +#ifdef ARBITER_CUSTOM_NAMESPACE +namespace ARBITER_CUSTOM_NAMESPACE +{ +#endif + +namespace arbiter +{ + +namespace drivers +{ + +class Google : public Https +{ + class Auth; +public: + Google(http::Pool& pool, std::unique_ptr auth); + + static std::unique_ptr create( + http::Pool& pool, + const Json::Value& json); + + // Overrides. + virtual std::string type() const override { return "gs"; } // Match gsutil. + + virtual std::unique_ptr tryGetSize( + std::string path) const override; + + /** Inherited from Drivers::Http. */ + virtual void put( + std::string path, + const std::vector& data, + http::Headers headers, + http::Query query) const override; + +private: + /** Inherited from Drivers::Http. */ + virtual bool get( + std::string path, + std::vector& data, + http::Headers headers, + http::Query query) const override; + + virtual std::vector glob( + std::string path, + bool verbose) const override; + + std::unique_ptr m_auth; +}; + +class Google::Auth +{ +public: + Auth(const Json::Value& creds); + static std::unique_ptr create(const Json::Value& json); + + http::Headers headers() const; + +private: + void maybeRefresh() const; + std::string sign(std::string data, std::string privateKey) const; + + const Json::Value m_creds; + mutable int64_t m_expiration = 0; // Unix time. + mutable http::Headers m_headers; + + mutable std::mutex m_mutex; +}; + +} // namespace drivers +} // namespace arbiter + +#ifdef ARBITER_CUSTOM_NAMESPACE +} +#endif + + +// ////////////////////////////////////////////////////////////////////// +// End of content of file: arbiter/drivers/google.hpp +// ////////////////////////////////////////////////////////////////////// + + + + + + // ////////////////////////////////////////////////////////////////////// // Beginning of content of file: arbiter/drivers/dropbox.hpp // ////////////////////////////////////////////////////////////////////// @@ -4652,6 +4792,29 @@ class Endpoint /** Get a further nested subpath relative to this Endpoint's root. */ Endpoint getSubEndpoint(std::string subpath) const; + http::Response httpGet( + std::string path, + http::Headers headers = http::Headers(), + http::Query query = http::Query(), + std::size_t reserve = 0) const; + + http::Response httpPut( + std::string path, + const std::vector& data, + http::Headers headers = http::Headers(), + http::Query query = http::Query()) const; + + http::Response httpHead( + std::string path, + http::Headers headers = http::Headers(), + http::Query query = http::Query()) const; + + http::Response httpPost( + std::string path, + const std::vector& data, + http::Headers headers = http::Headers(), + http::Query query = http::Query()) const; + private: Endpoint(const Driver& driver, std::string root); @@ -4696,15 +4859,17 @@ class Endpoint #endif #ifndef ARBITER_IS_AMALGAMATION -#include -#include #include +#include +#include #include -#include +#include #include #include -#include +#include +#include #include +#include #ifndef ARBITER_EXTERNAL_JSON #include @@ -4849,8 +5014,8 @@ class ARBITER_DLL Arbiter /** Copy the single file @p file to the destination @p to. If @p to ends * with a `/` or '\' character, then @p file will be copied into the - * directory @p to with the basename of @p file. If @p does not end with a - * slash character, then @p to will be interpreted as a file path. + * directory @p to with the basename of @p file. If @p to does not end + * with a slash character, then @p to will be interpreted as a file path. * * If @p to is a local filesystem path, then `fs::mkdirp` will be called * prior to copying.