XrdHttp: Refactored HTTP checksum algorithm handling code

A new class XrdHttpChecksumHandler was created allowing the HTTP checksum-matching logic to be put in one place

src/XrdHttp.cmake

@@ -26,7 +26,9 @@ if( BUILD_HTTP )
     XrdHttp/XrdHttpExtHandler.cc      XrdHttp/XrdHttpExtHandler.hh
                                       XrdHttp/XrdHttpStatic.hh
                                       XrdHttp/XrdHttpTrace.hh
-    XrdHttp/XrdHttpUtils.cc           XrdHttp/XrdHttpUtils.hh )
+    XrdHttp/XrdHttpUtils.cc           XrdHttp/XrdHttpUtils.hh
+    XrdHttp/XrdHttpChecksumHandler.cc XrdHttp/XrdHttpChecksumHandler.hh
+    XrdHttp/XrdHttpChecksum.cc        XrdHttp/XrdHttpChecksum.hh)
 
   # Note this is marked as a shared library as XrdHttp plugins are expected to
   # link against this for the XrdHttpExt class implementations.

src/XrdHttp/XrdHttpChecksum.cc

@@ -0,0 +1,47 @@
+//------------------------------------------------------------------------------
+// This file is part of XrdHTTP: A pragmatic implementation of the
+// HTTP/WebDAV protocol for the Xrootd framework
+//
+// Copyright (c) 2013 by European Organization for Nuclear Research (CERN)
+// Author: Cedric Caffy <ccaffy@cern.ch>
+// File Date: Mar 2023
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see <http://www.gnu.org/licenses/>.
+//------------------------------------------------------------------------------
+
+#include "XrdHttpChecksum.hh"
+#include <algorithm>
+
+XrdHttpChecksum::XrdHttpChecksum(const std::string & xrootdName, const std::string & httpName, bool needsBase64Padding):
+        mXRootDName(xrootdName),
+        mHTTPName(httpName),
+        mNeedsBase64Padding(needsBase64Padding){}
+
+std::string XrdHttpChecksum::getHttpName() const {
+    return mHTTPName;
+}
+
+std::string XrdHttpChecksum::getHttpNameLowerCase() const {
+    std::string ret = getHttpName();
+    std::transform(ret.begin(),ret.end(),ret.begin(),::tolower);
+    return ret;
+}
+
+std::string XrdHttpChecksum::getXRootDName() const {
+    return mXRootDName;
+}
+
+bool XrdHttpChecksum::needsBase64Padding() const {
+    return mNeedsBase64Padding;
+}

src/XrdHttp/XrdHttpChecksum.hh

@@ -0,0 +1,54 @@
+//------------------------------------------------------------------------------
+// This file is part of XrdHTTP: A pragmatic implementation of the
+// HTTP/WebDAV protocol for the Xrootd framework
+//
+// Copyright (c) 2013 by European Organization for Nuclear Research (CERN)
+// Author: Cedric Caffy <ccaffy@cern.ch>
+// File Date: Mar 2023
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see <http://www.gnu.org/licenses/>.
+//------------------------------------------------------------------------------
+
+#ifndef XROOTD_XRDHTTPCHECKSUM_HH
+#define XROOTD_XRDHTTPCHECKSUM_HH
+
+#include <string>
+
+/**
+ * Simple object containing information about
+ * a checksum
+ */
+class XrdHttpChecksum {
+public:
+    /**
+     * Constructor
+     * @param xrootdName the name that will be used by XRootD server to run the checksum
+     * @param httpName the HTTP RFC compliant name of the checksum
+     * @param needsBase64Padding sets to true if the checksum needs to be base64 encoded before being sent, false otherwise
+     */
+    XrdHttpChecksum(const std::string & xrootdName, const std::string & httpName, bool needsBase64Padding);
+
+    std::string getXRootDName() const;
+    std::string getHttpName() const;
+    std::string getHttpNameLowerCase() const;
+    bool needsBase64Padding() const;
+
+private:
+    std::string mXRootDName;
+    std::string mHTTPName;
+    bool mNeedsBase64Padding;
+};
+
+
+#endif //XROOTD_XRDHTTPCHECKSUM_HH

src/XrdHttp/XrdHttpChecksumHandler.cc

@@ -0,0 +1,135 @@
+//------------------------------------------------------------------------------
+// This file is part of XrdHTTP: A pragmatic implementation of the
+// HTTP/WebDAV protocol for the Xrootd framework
+//
+// Copyright (c) 2013 by European Organization for Nuclear Research (CERN)
+// Author: Cedric Caffy <ccaffy@cern.ch>
+// File Date: Mar 2023
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see <http://www.gnu.org/licenses/>.
+//------------------------------------------------------------------------------
+
+#include "XrdHttpChecksumHandler.hh"
+#include "XrdOuc/XrdOucTUtils.hh"
+#include "XrdOuc/XrdOucUtils.hh"
+#include <exception>
+#include <algorithm>
+#include <sstream>
+
+std::map<std::string,XrdHttpChecksumHandlerImpl::XrdHttpChecksumPtr> XrdHttpChecksumHandlerImpl::XROOTD_TO_HTTP_CKSUMS;
+XrdHttpChecksumHandlerImpl::XrdHttpChecksumPtr XrdHttpChecksumHandlerImpl::NONE_CONFIGURED_CKSUM = std::make_shared<XrdHttpChecksum>("none_configured", "none_configured", false);
+
+void XrdHttpChecksumHandlerImpl::initializeCksumsMaps() {
+    addChecksumToMaps(std::make_shared<XrdHttpChecksum>("md5","md5",true));
+    addChecksumToMaps(std::make_shared<XrdHttpChecksum>("adler32","adler32",false));
+    addChecksumToMaps(std::make_shared<XrdHttpChecksum>("sha1","sha",true));
+    addChecksumToMaps(std::make_shared<XrdHttpChecksum>("sha256","sha-256",true));
+    addChecksumToMaps(std::make_shared<XrdHttpChecksum>("sha512","sha-512",true));
+    addChecksumToMaps(std::make_shared<XrdHttpChecksum>("cksum","UNIXcksum",false));
+    addChecksumToMaps(std::make_shared<XrdHttpChecksum>("crc32","crc32",false));
+    addChecksumToMaps(std::make_shared<XrdHttpChecksum>("crc32c","crc32c",true));
+}
+
+void XrdHttpChecksumHandlerImpl::addChecksumToMaps(XrdHttpChecksumHandlerImpl::XrdHttpChecksumPtr checksum) {
+    XROOTD_TO_HTTP_CKSUMS[checksum->getXRootDName()] = checksum;
+}
+
+XrdHttpChecksumHandlerImpl::XrdHttpChecksumPtr XrdHttpChecksumHandlerImpl::getChecksumToRun(const std::string &userDigest) const {
+    if(mConfiguredChecksums.empty()) {
+        // This happens when no checksum is configured on the server
+        // this will anyway result in a METHOD NOT ALLOWED HTTP error sent to the user
+        // if a checksum have been requested
+        return NONE_CONFIGURED_CKSUM;
+    }
+    std::vector<std::string> userDigests = getUserDigests(userDigest);
+    //Loop over the user digests and find the corresponding checksum
+    for(auto userDigest: userDigests) {
+        auto httpCksum = std::find_if(mConfiguredChecksums.begin(), mConfiguredChecksums.end(),[userDigest](const XrdHttpChecksumPtr & cksum){
+            return userDigest == cksum->getHttpNameLowerCase();
+        });
+        if(httpCksum != mConfiguredChecksums.end()) {
+            return (*httpCksum);
+        }
+    }
+    return mConfiguredChecksums[0];
+}
+
+const std::vector<std::string> &XrdHttpChecksumHandlerImpl::getIncompatibleChecksums() const {
+    return mIncompatibleConfiguredChecksums;
+}
+
+const std::vector<XrdHttpChecksumHandler::XrdHttpChecksumPtr> & XrdHttpChecksumHandlerImpl::getConfiguredChecksums() const {
+    return mConfiguredChecksums;
+}
+
+
+void XrdHttpChecksumHandlerImpl::configure(const char *csList) {
+    initializeCksumsMaps();
+    if(csList != nullptr) {
+        initializeXRootDConfiguredCksums(csList);
+        if(mConfiguredChecksums.empty()) {
+            std::stringstream ss;
+            ss << "No HTTP-compatible checksums have been configured. Please configure at least one " <<
+            "of the following checksums [";
+            std::string supportedCksums = "";
+            for(auto xrdToHttpCksum : XROOTD_TO_HTTP_CKSUMS){
+                supportedCksums += xrdToHttpCksum.first + ",";
+            }
+            supportedCksums.erase(supportedCksums.size() - 1);
+            ss << supportedCksums << "]";
+            throw std::runtime_error(ss.str());
+        }
+    }
+}
+
+/**
+ * Initializes the checksums from the csList parameter passed
+ * @param csList the list of the configured checksum under the format 0:adler32,1:sha1,2:sha512
+ */
+void XrdHttpChecksumHandlerImpl::initializeXRootDConfiguredCksums(const char *csList) {
+    std::vector<std::string> splittedCslist;
+    XrdOucTUtils::splitString(splittedCslist,csList,",");
+    for(auto csElt: splittedCslist) {
+        auto csName = getElement(csElt,":",1);
+        auto csNameLowerCase = csName;
+        std::transform(csNameLowerCase.begin(),csNameLowerCase.end(),csNameLowerCase.begin(),::tolower);
+        auto checksumItor = XROOTD_TO_HTTP_CKSUMS.find(csNameLowerCase);
+        if(checksumItor != XROOTD_TO_HTTP_CKSUMS.end()) {
+            mConfiguredChecksums.push_back(checksumItor->second);
+        } else {
+            mIncompatibleConfiguredChecksums.push_back(csName);
+        }
+    }
+}
+
+std::string XrdHttpChecksumHandlerImpl::getElement(const std::string &input, const std::string & delimiter,
+                                               const size_t position) {
+    std::vector<std::string> elementsAfterSplit;
+    XrdOucTUtils::splitString(elementsAfterSplit,input,delimiter);
+    return elementsAfterSplit[position];
+}
+
+std::vector<std::string> XrdHttpChecksumHandlerImpl::getUserDigests(const std::string &userDigests) {
+    //userDigest is a comma-separated list with q-values
+    std::vector<std::string> userDigestsRet;
+    std::vector<std::string> userDigestsWithQValues;
+    XrdOucTUtils::splitString(userDigestsWithQValues,userDigests,",");
+    for(auto & userDigestWithQValue: userDigestsWithQValues){
+        std::transform(userDigestWithQValue.begin(),userDigestWithQValue.end(),userDigestWithQValue.begin(),::tolower);
+        auto userDigest = getElement(userDigestWithQValue,";",0);
+        XrdOucUtils::trim(userDigest);
+        userDigestsRet.push_back(userDigest);
+    }
+    return userDigestsRet;
+}

src/XrdHttp/XrdHttpChecksumHandler.hh

@@ -0,0 +1,107 @@
+//------------------------------------------------------------------------------
+// This file is part of XrdHTTP: A pragmatic implementation of the
+// HTTP/WebDAV protocol for the Xrootd framework
+//
+// Copyright (c) 2013 by European Organization for Nuclear Research (CERN)
+// Author: Cedric Caffy <ccaffy@cern.ch>
+// File Date: Mar 2023
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see <http://www.gnu.org/licenses/>.
+//------------------------------------------------------------------------------
+#ifndef XROOTD_XRDHTTPCHECKSUMHANDLER_HH
+#define XROOTD_XRDHTTPCHECKSUMHANDLER_HH
+
+#include <string>
+#include <map>
+#include <vector>
+#include "XrdHttpChecksum.hh"
+#include <memory>
+
+/**
+ * Implementation class of the XrdHttpChecksumHandler
+ *
+ * Is useful for unit testing
+ */
+class XrdHttpChecksumHandlerImpl {
+public:
+    using XrdHttpChecksumPtr = std::shared_ptr<XrdHttpChecksum>;
+
+    XrdHttpChecksumHandlerImpl() = default;
+
+    void configure(const char * csList);
+    XrdHttpChecksumPtr getChecksumToRun(const std::string & userDigest) const;
+    /**
+     * For testing purposes
+     */
+    const std::vector<std::string> & getIncompatibleChecksums() const;
+    /**
+     * For testing purposes
+     */
+    const std::vector<XrdHttpChecksumPtr> & getConfiguredChecksums() const;
+private:
+    void initializeXRootDConfiguredCksums(const char *csList);
+    /**
+     * Modify this if new checksums have to be supported or
+     * if some don't require base64 padding anymore
+     */
+    static void initializeCksumsMaps();
+    static void addChecksumToMaps(XrdHttpChecksumPtr checksum);
+    static std::string getElement(const std::string & input, const std::string & delimiter, const size_t position);
+    /**
+     * Returns a vector of user digests (lower-cased) extracted from the userDigests string passed in parameter
+     * @param userDigests the string containing a quality-valued checksum list e.g: adler32, md5;q=0.4, md5
+     * @return the lower-cased user digests vector
+     */
+    static std::vector<std::string> getUserDigests(const std::string & userDigests);
+
+    //The map that containing all possible HTTP-compatible xrootd checksums
+    static std::map<std::string,XrdHttpChecksumPtr> XROOTD_TO_HTTP_CKSUMS;
+    //Default checksum to return if no checksums have been configured, it's just something
+    //to avoid dealing with nullptr when calling getChecksumToRun()
+    static XrdHttpChecksumPtr NONE_CONFIGURED_CKSUM;
+    // The vector of HTTP-compatible configured checksum
+    std::vector<XrdHttpChecksumPtr> mConfiguredChecksums;
+    // The vector of non-HTTP configured checksum names (for testing purposes)
+    std::vector<std::string> mIncompatibleConfiguredChecksums;
+};
+
+/**
+ * This class allows to handle xrd http checksum algorithm selection
+ * based on what the user provided as a digest
+ */
+class XrdHttpChecksumHandler {
+public:
+    using XrdHttpChecksumPtr = XrdHttpChecksumHandlerImpl::XrdHttpChecksumPtr;
+
+    XrdHttpChecksumHandler() = default;
+    /**
+     * Configure this handler.
+     * @throws runtime_exception if no algorithm in the csList is compatible with HTTP
+     * @param csList the list coming from the server configuration. Should be under the format 0:adler32,1:sha512
+     */
+    void configure(const char * csList) { pImpl.configure(csList); }
+    /**
+     * Returns the checksum to run from the user "Want-Digest" provided string
+     * @param userDigest the digest string under the format "sha-512,sha-256;q=0.8,sha;q=0.6,md5;q=0.4,adler32;q=0.2"
+     * @return the checksum to run depending on the userDigest provided string
+     * The logic behind it is simple: returns the first userDigest provided that matches the one configured.
+     * If none is matched, the first algorithm configured on the server side will be returned.
+     */
+    XrdHttpChecksumPtr getChecksumToRun(const std::string & userDigest) const { return pImpl.getChecksumToRun(userDigest); }
+private:
+    XrdHttpChecksumHandlerImpl pImpl;
+};
+
+
+#endif //XROOTD_XRDHTTPCHECKSUMHANDLER_HH

src/XrdHttp/XrdHttpProtocol.cc

@@ -109,6 +109,7 @@ XrdSecService *XrdHttpProtocol::CIA = 0; // Authentication Server
 int XrdHttpProtocol::m_bio_type = 0; // BIO type identifier for our custom BIO.
 BIO_METHOD *XrdHttpProtocol::m_bio_method = NULL; // BIO method constructor.
 char *XrdHttpProtocol::xrd_cslist = nullptr;
+XrdHttpChecksumHandler XrdHttpProtocol::cksumHandler = XrdHttpChecksumHandler();
 
 XrdSysTrace XrdHttpTrace("http");
 
@@ -948,6 +949,15 @@ int XrdHttpProtocol::Config(const char *ConfigFN, XrdOucEnv *myEnv) {
   char *var;
   int cfgFD, GoNo, NoGo = 0, ismine;
 
+  try {
+      cksumHandler.configure(xrd_cslist);
+  } catch (const std::runtime_error &ex) {
+      // We prevent the server to start if no HTTP-compatible checksums are configured
+      std::string errMsg = "Config failure: " + std::string(ex.what());
+      eDest.Say(errMsg.c_str());
+      return 1;
+  }
+
 
   // Initialize our custom BIO type.
   if (!m_bio_type) {