From dbb311a0fca4bf5ae9acd2648bdc3bc8ad07e1cf Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Sun, 15 Jul 2018 15:10:53 +0200 Subject: [PATCH 1/8] Add plumbing necessary for RFC3230 digest requests --- src/XrdHttp/XrdHttpProtocol.cc | 14 +++++++++++++ src/XrdHttp/XrdHttpProtocol.hh | 3 ++- src/XrdHttp/XrdHttpReq.cc | 37 +++++++++++++++++++++++++--------- src/XrdHttp/XrdHttpReq.hh | 3 +++ 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/XrdHttp/XrdHttpProtocol.cc b/src/XrdHttp/XrdHttpProtocol.cc index 86b3d6d2f22..c8ae6ae3ff6 100644 --- a/src/XrdHttp/XrdHttpProtocol.cc +++ b/src/XrdHttp/XrdHttpProtocol.cc @@ -2657,6 +2657,20 @@ int XrdHttpProtocol::doStat(char *fname) { } +int XrdHttpProtocol::doChksum(const XrdOucString &fname) { + size_t length; + memset(&CurrentReq.xrdreq, 0, sizeof (ClientRequest)); + CurrentReq.xrdreq.query.requestid = htons(kXR_query); + CurrentReq.xrdreq.query.infotype = htons(kXR_Qcksum); + memset(CurrentReq.xrdreq.query.reserved1, '\0', sizeof(CurrentReq.xrdreq.query.reserved1)); + memset(CurrentReq.xrdreq.query.fhandle, '\0', sizeof(CurrentReq.xrdreq.query.fhandle)); + memset(CurrentReq.xrdreq.query.reserved2, '\0', sizeof(CurrentReq.xrdreq.query.reserved2)); + length = fname.length() + 1; + CurrentReq.xrdreq.query.dlen = htonl(length); + + return Bridge->Run(reinterpret_cast(&CurrentReq.xrdreq), const_cast(fname.c_str()), length) ? 0 : -1; +} + static XrdVERSIONINFODEF(compiledVer, XrdHttpProtocolTest, XrdVNUMBER, XrdVERSION); diff --git a/src/XrdHttp/XrdHttpProtocol.hh b/src/XrdHttp/XrdHttpProtocol.hh index 766df9b13dd..64558a8eaea 100644 --- a/src/XrdHttp/XrdHttpProtocol.hh +++ b/src/XrdHttp/XrdHttpProtocol.hh @@ -107,7 +107,8 @@ public: /// Perform a Stat request int doStat(char *fname); - + /// Perform a checksum request + int doChksum(const XrdOucString &fname); /// Ctor, dtors and copy ctor XrdHttpProtocol operator =(const XrdHttpProtocol &rhs); diff --git a/src/XrdHttp/XrdHttpReq.cc b/src/XrdHttp/XrdHttpReq.cc index a696f2278ba..d876ee11cb1 100644 --- a/src/XrdHttp/XrdHttpReq.cc +++ b/src/XrdHttp/XrdHttpReq.cc @@ -185,6 +185,9 @@ int XrdHttpReq::parseLine(char *line, int len) { } else if (!strcmp(key, "Destination")) { destination.assign(val, line+len-val); trim(destination); + } else if (!strcmp(key, "Want-Digest")) { + m_req_digest.assign(val, line + len - val); + trim(m_req_digest); } else if (!strcmp(key, "Depth")) { depth = -1; if (strcmp(val, "infinity")) @@ -1469,9 +1472,11 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { } case XrdHttpReq::rtHEAD: { - - if (xrdresp == kXR_ok) { - + if (xrdresp != kXR_ok) { + prot->SendSimpleResp(httpStatusCode, NULL, NULL, + httpStatusText.c_str(), httpStatusText.length()); + return -1; + } else if (reqstate == 0) { if (iovN > 0) { // Now parse the stat info @@ -1484,18 +1489,32 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { &fileflags, &filemodtime); - prot->SendSimpleResp(200, NULL, NULL, NULL, filesize); - return 1; + if (m_req_digest.size()) { + if (prot->doChksum(resourceplusopaque) < 0) { + prot->SendSimpleResp(500, NULL, NULL, "Failed to route checksum request", 0); + return -1; + } + return 0; + } else { + prot->SendSimpleResp(200, NULL, NULL, NULL, filesize); + return 1; + } } prot->SendSimpleResp(httpStatusCode, NULL, NULL, httpStatusText.c_str(), httpStatusText.length()); reset(); return 1; - } else { - prot->SendSimpleResp(httpStatusCode, NULL, NULL, - httpStatusText.c_str(), httpStatusText.length()); - return -1; + } else { // We requested a checksum and now have its response. + if (iovN > 0) { + TRACEI(REQ, "Checksum for HEAD " << resource << " value=" << reinterpret_cast(iovP[0].iov_base)); + + std::string digest_response = "Digest: "; + digest_response += m_req_digest; + digest_response += "="; + digest_response += reinterpret_cast(iovP[0].iov_base); + prot->SendSimpleResp(200, NULL, digest_response.c_str(), NULL, filesize); + } } } case XrdHttpReq::rtGET: diff --git a/src/XrdHttp/XrdHttpReq.hh b/src/XrdHttp/XrdHttpReq.hh index 83f7900960f..eada343646d 100644 --- a/src/XrdHttp/XrdHttpReq.hh +++ b/src/XrdHttp/XrdHttpReq.hh @@ -202,6 +202,9 @@ public: /// The destination field specified in the req std::string destination; + /// The requested digest type + std::string m_req_digest; + /// Additional opaque info that may come from the hdr2cgi directive std::string hdr2cgistr; From 7c0aa3130adc81d07016f71b81d621c1558e17a0 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Sun, 15 Jul 2018 15:14:18 +0200 Subject: [PATCH 2/8] Move internal function to private. --- src/XrdHttp/XrdHttpReq.hh | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/XrdHttp/XrdHttpReq.hh b/src/XrdHttp/XrdHttpReq.hh index eada343646d..b30c0776cd2 100644 --- a/src/XrdHttp/XrdHttpReq.hh +++ b/src/XrdHttp/XrdHttpReq.hh @@ -93,7 +93,12 @@ private: void getfhandle(); - + /// Cook and send the response after the bridge did something + /// Return values: + /// 0->everything OK, additionsl steps may be required + /// 1->request processed completely + /// -1->error + int PostProcessHTTPReq(bool final = false); // Parse a resource string, typically a filename, setting the resource field and the opaque data void parseResource(char *url); @@ -258,19 +263,6 @@ public: /// -1->error int ProcessHTTPReq(); - /// Cook and send the response after the bridge did something - /// Return values: - /// 0->everything OK, additionsl steps may be required - /// 1->request processed completely - /// -1->error - int PostProcessHTTPReq(bool final = false); - - - - - - - // ------------ // Items inherited from the Bridge class From 4cf1ab6fbc955321ae06efa115e8a716cb81c5b1 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Sun, 15 Jul 2018 08:30:16 -0500 Subject: [PATCH 3/8] Prevent a HEAD request from returning a body. --- src/XrdHttp/XrdHttpReq.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/XrdHttp/XrdHttpReq.cc b/src/XrdHttp/XrdHttpReq.cc index d876ee11cb1..0a504b359b2 100644 --- a/src/XrdHttp/XrdHttpReq.cc +++ b/src/XrdHttp/XrdHttpReq.cc @@ -1473,8 +1473,8 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { case XrdHttpReq::rtHEAD: { if (xrdresp != kXR_ok) { - prot->SendSimpleResp(httpStatusCode, NULL, NULL, - httpStatusText.c_str(), httpStatusText.length()); + // NOTE that HEAD MUST NOT return a body, even in the case of failure. + prot->SendSimpleResp(httpStatusCode, NULL, NULL, NULL, 0); return -1; } else if (reqstate == 0) { if (iovN > 0) { @@ -1491,7 +1491,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { if (m_req_digest.size()) { if (prot->doChksum(resourceplusopaque) < 0) { - prot->SendSimpleResp(500, NULL, NULL, "Failed to route checksum request", 0); + prot->SendSimpleResp(500, NULL, NULL, NULL, 0); return -1; } return 0; @@ -1501,8 +1501,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { } } - prot->SendSimpleResp(httpStatusCode, NULL, NULL, - httpStatusText.c_str(), httpStatusText.length()); + prot->SendSimpleResp(httpStatusCode, NULL, NULL, NULL, 0); reset(); return 1; } else { // We requested a checksum and now have its response. From bcb0f303e834c71fee8f55f044d631f298cc2885 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 16 Jul 2018 09:08:37 -0500 Subject: [PATCH 4/8] Fix checksumming on filesystems that don't support fattr. --- src/XrdXrootd/XrdXrootdXeq.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/XrdXrootd/XrdXrootdXeq.cc b/src/XrdXrootd/XrdXrootdXeq.cc index 9928986b5f7..37979743f73 100644 --- a/src/XrdXrootd/XrdXrootdXeq.cc +++ b/src/XrdXrootd/XrdXrootdXeq.cc @@ -432,11 +432,13 @@ int XrdXrootdProtocol::do_CKsum(char *algT, const char *Path, char *Opaque) // Diagnose any hard errors // - if (rc) return fsError(rc, 0, myError, Path, Opaque); + if (rc && (myError.getErrInfo() != ENOTSUP)) { + return fsError(rc, 0, myError, Path, Opaque); + } // Return result if it is actually available // - if (*csData) + if (!rc && *csData) {if (*csData == '!') return Response.Send(csData+1); struct iovec iov[4] = {{0,0}, {algT, (size_t)CKTLen}, {&Space, 1}, {(char *)csData, strlen(csData)+1}}; From f312ef878394d243ae2ecf11a524ff2da2b8e6d7 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Tue, 17 Jul 2018 04:13:47 -0500 Subject: [PATCH 5/8] Correctly handle the Want-Digest header by computing the checksum. --- src/XrdHttp/XrdHttpReq.cc | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/XrdHttp/XrdHttpReq.cc b/src/XrdHttp/XrdHttpReq.cc index 0a504b359b2..89e4191e138 100644 --- a/src/XrdHttp/XrdHttpReq.cc +++ b/src/XrdHttp/XrdHttpReq.cc @@ -865,14 +865,21 @@ int XrdHttpReq::ProcessHTTPReq() { } case XrdHttpReq::rtHEAD: { - - // Do a Stat - if (prot->doStat((char *) resourceplusopaque.c_str())) { - prot->SendSimpleResp(404, NULL, NULL, (char *) "Could not run request.", 0); - return -1; + if (reqstate == 0) { + // Always start with Stat; in the case of a checksum request, we'll have a follow-up query + if (prot->doStat((char *) resourceplusopaque.c_str())) { + prot->SendSimpleResp(404, NULL, NULL, (char *) "Could not run request.", 0); + return -1; + } + return 0; + } else { + if (prot->doChksum(resourceplusopaque) < 0) { + // In this case, the Want-Digest header was set and PostProcess gave the go-ahead to do a checksum. + prot->SendSimpleResp(500, NULL, NULL, NULL, 0); + return -1; + } + return 1; } - - return 1; } case XrdHttpReq::rtGET: { @@ -1490,10 +1497,6 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { &filemodtime); if (m_req_digest.size()) { - if (prot->doChksum(resourceplusopaque) < 0) { - prot->SendSimpleResp(500, NULL, NULL, NULL, 0); - return -1; - } return 0; } else { prot->SendSimpleResp(200, NULL, NULL, NULL, filesize); @@ -1506,13 +1509,17 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { return 1; } else { // We requested a checksum and now have its response. if (iovN > 0) { - TRACEI(REQ, "Checksum for HEAD " << resource << " value=" << reinterpret_cast(iovP[0].iov_base)); + TRACEI(REQ, "Checksum for HEAD " << resource << " " << reinterpret_cast(iovP[0].iov_base) << "=" << reinterpret_cast(iovP[iovN-1].iov_base)); std::string digest_response = "Digest: "; digest_response += m_req_digest; digest_response += "="; - digest_response += reinterpret_cast(iovP[0].iov_base); + digest_response += reinterpret_cast(iovP[iovN-1].iov_base); prot->SendSimpleResp(200, NULL, digest_response.c_str(), NULL, filesize); + return 1; + } else { + prot->SendSimpleResp(500, NULL, NULL, NULL, 0); + return -1; } } } From 9501f2a202a97ade5f7769886f1fcea15bf369dd Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Tue, 17 Jul 2018 12:04:07 -0500 Subject: [PATCH 6/8] Convert algorithm outputs to base64 encoding. Handle encoding of different digests according to: - https://tools.ietf.org/html/rfc3230#ref-11 - https://www.iana.org/assignments/http-dig-alg/http-dig-alg.xhtml#http-dig-alg-1 --- src/XrdHttp/XrdHttpReq.cc | 80 ++++++++++++++++++++++++++++--------- src/XrdHttp/XrdHttpReq.hh | 4 ++ src/XrdHttp/XrdHttpUtils.cc | 27 +++++++++++++ src/XrdHttp/XrdHttpUtils.hh | 4 ++ 4 files changed, 97 insertions(+), 18 deletions(-) diff --git a/src/XrdHttp/XrdHttpReq.cc b/src/XrdHttp/XrdHttpReq.cc index 89e4191e138..aa2e4cf1aaa 100644 --- a/src/XrdHttp/XrdHttpReq.cc +++ b/src/XrdHttp/XrdHttpReq.cc @@ -66,24 +66,42 @@ #define TRACELINK prot->Link +static XrdOucString convert_digest_name(const std::string &rfc_name) +{ + if (!strcasecmp(rfc_name.c_str(), "md5")) { + return "md5"; + } else if (!strcasecmp(rfc_name.c_str(), "adler32")) { + return "adler32"; + } else if (strcasecmp(rfc_name.c_str(), "SHA")) { + return "sha1"; + } else if (strcasecmp(rfc_name.c_str(), "SHA-256")) { + return "sha256"; + } else if (strcasecmp(rfc_name.c_str(), "SHA-512")) { + return "sha512"; + } else if (strcasecmp(rfc_name.c_str(), "UNIXcksum")) { + return "cksum"; + } + return "unknown"; +} - - - - - - - - - - - - - - - - +static bool needs_base64_padding(const std::string &rfc_name) +{ + if (!strcasecmp(rfc_name.c_str(), "md5")) { + return true; + } else if (!strcasecmp(rfc_name.c_str(), "adler32")) { + return false; + } else if (strcasecmp(rfc_name.c_str(), "SHA")) { + return true; + } else if (strcasecmp(rfc_name.c_str(), "SHA-256")) { + return true; + } else if (strcasecmp(rfc_name.c_str(), "SHA-512")) { + return true; + } else if (strcasecmp(rfc_name.c_str(), "UNIXcksum")) { + return false; + } + return false; +} void trim(std::string &str) @@ -873,7 +891,17 @@ int XrdHttpReq::ProcessHTTPReq() { } return 0; } else { - if (prot->doChksum(resourceplusopaque) < 0) { + const char *opaque = strchr(resourceplusopaque.c_str(), '?'); + // Note that doChksum requires that the memory stays alive until the callback is invoked. + m_resource_with_digest = resourceplusopaque; + if (!opaque) { + m_resource_with_digest += "?cks.type="; + m_resource_with_digest += convert_digest_name(m_req_digest); + } else { + m_resource_with_digest += "&cks.type="; + m_resource_with_digest += convert_digest_name(m_req_digest); + } + if (prot->doChksum(m_resource_with_digest) < 0) { // In this case, the Want-Digest header was set and PostProcess gave the go-ahead to do a checksum. prot->SendSimpleResp(500, NULL, NULL, NULL, 0); return -1; @@ -1511,10 +1539,26 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { if (iovN > 0) { TRACEI(REQ, "Checksum for HEAD " << resource << " " << reinterpret_cast(iovP[0].iov_base) << "=" << reinterpret_cast(iovP[iovN-1].iov_base)); + bool convert_to_base64 = needs_base64_padding(m_req_digest); + char *digest_value = reinterpret_cast(iovP[iovN-1].iov_base); + if (convert_to_base64) { + size_t digest_length = strlen(digest_value); + unsigned char *digest_binary_value = (unsigned char *)malloc(digest_length); + if (!Fromhexdigest(reinterpret_cast(digest_value), digest_length, digest_binary_value)) { + prot->SendSimpleResp(500, NULL, NULL, NULL, 0); + free(digest_binary_value); + } + char *digest_base64_value = (char *)malloc(digest_length); + Tobase64(digest_binary_value, digest_length/2, digest_base64_value); + free(digest_binary_value); + digest_value = digest_base64_value; + } + std::string digest_response = "Digest: "; digest_response += m_req_digest; digest_response += "="; - digest_response += reinterpret_cast(iovP[iovN-1].iov_base); + digest_response += digest_value; + if (convert_to_base64) {free(digest_value);} prot->SendSimpleResp(200, NULL, digest_response.c_str(), NULL, filesize); return 1; } else { diff --git a/src/XrdHttp/XrdHttpReq.hh b/src/XrdHttp/XrdHttpReq.hh index b30c0776cd2..fff98d04b9c 100644 --- a/src/XrdHttp/XrdHttpReq.hh +++ b/src/XrdHttp/XrdHttpReq.hh @@ -209,6 +209,10 @@ public: /// The requested digest type std::string m_req_digest; + /// The checksum algorithm is specified as part of the opaque data in the URL. + /// Hence, when a digest is generated to satisfy a request, we cache the tweaked + /// URL in this data member. + XrdOucString m_resource_with_digest; /// Additional opaque info that may come from the hdr2cgi directive std::string hdr2cgistr; diff --git a/src/XrdHttp/XrdHttpUtils.cc b/src/XrdHttp/XrdHttpUtils.cc index bfcac258392..2f8396465d4 100644 --- a/src/XrdHttp/XrdHttpUtils.cc +++ b/src/XrdHttp/XrdHttpUtils.cc @@ -149,7 +149,34 @@ void Tobase64(const unsigned char *input, int length, char *out) { } +static int +char_to_int(int c) +{ + if (isdigit(c)) { + return c - '0'; + } else { + c = tolower(c); + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + return -1; + } +} + +// Decode a hex digest array to raw bytes. +// +bool Fromhexdigest(const unsigned char *input, int length, unsigned char *out) { + for (int idx=0; idx < length; idx += 2) { + int upper = char_to_int(input[idx]); + int lower = char_to_int(input[idx+1]); + if ((upper < 0) || (lower < 0)) { + return false; + } + out[idx/2] = (upper << 4) + lower; + } + return true; +} // Simple itoa function diff --git a/src/XrdHttp/XrdHttpUtils.hh b/src/XrdHttp/XrdHttpUtils.hh index ab5acbcf80d..d1392cf0b72 100644 --- a/src/XrdHttp/XrdHttpUtils.hh +++ b/src/XrdHttp/XrdHttpUtils.hh @@ -71,6 +71,10 @@ int compareHash( const char *h2); +bool Fromhexdigest(const unsigned char *input, int length, unsigned char *out); + +void Tobase64(const unsigned char *input, int length, char *out); + // Create a new quoted string char *quote(const char *str); From 8987840a97deef96bab3ba529ebdcecf68eb102f Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 19 Jul 2018 09:31:30 -0500 Subject: [PATCH 7/8] [XrdHttp] Reset the digest information when request is reset. --- src/XrdHttp/XrdHttpReq.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/XrdHttp/XrdHttpReq.cc b/src/XrdHttp/XrdHttpReq.cc index aa2e4cf1aaa..8c9a84b339c 100644 --- a/src/XrdHttp/XrdHttpReq.cc +++ b/src/XrdHttp/XrdHttpReq.cc @@ -2397,6 +2397,10 @@ void XrdHttpReq::reset() { resource = ""; allheaders.clear(); + // Reset the state of the request's digest request. + m_req_digest.clear(); + m_resource_with_digest = ""; + headerok = false; keepalive = true; length = 0; From b126f31d4f7e4dd1a030c45eefada3afd8f838f4 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Sat, 21 Jul 2018 00:26:46 -0500 Subject: [PATCH 8/8] Revert "Fix checksumming on filesystems that don't support fattr." This reverts commit bcb0f303e834c71fee8f55f044d631f298cc2885. It appears locally-computed checksums are not meant to be supported on filesystems without extended attributes. In that case, the admin write their own scripts for checksum calculation. --- src/XrdXrootd/XrdXrootdXeq.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/XrdXrootd/XrdXrootdXeq.cc b/src/XrdXrootd/XrdXrootdXeq.cc index 37979743f73..9928986b5f7 100644 --- a/src/XrdXrootd/XrdXrootdXeq.cc +++ b/src/XrdXrootd/XrdXrootdXeq.cc @@ -432,13 +432,11 @@ int XrdXrootdProtocol::do_CKsum(char *algT, const char *Path, char *Opaque) // Diagnose any hard errors // - if (rc && (myError.getErrInfo() != ENOTSUP)) { - return fsError(rc, 0, myError, Path, Opaque); - } + if (rc) return fsError(rc, 0, myError, Path, Opaque); // Return result if it is actually available // - if (!rc && *csData) + if (*csData) {if (*csData == '!') return Response.Send(csData+1); struct iovec iov[4] = {{0,0}, {algT, (size_t)CKTLen}, {&Space, 1}, {(char *)csData, strlen(csData)+1}};