From 4515360181f2e3503d748fa7c4ad498a7fb96fcd Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 25 Sep 2017 15:22:09 -0500 Subject: [PATCH 01/29] Initial commit --- .gitignore | 32 +++++++++ LICENSE | 201 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + 3 files changed, 235 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000000..259148fa18f --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000000..8dada3edaf5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000000..9dcad933bbd --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# xrootd-tpc +A HTTPS third-party-copy implementation for Xrootd From 6b7a95a1d0eee73863125231e6668a15a4cbdc0c Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 25 Sep 2017 15:23:12 -0500 Subject: [PATCH 02/29] Initial functional version of the third-party-copy library. Only a minimal proof-of-concept to show we can get COPY intercepted and working. --- CMakeLists.txt | 40 ++++ cmake/FindXrootd.cmake | 41 ++++ configs/export-lib-symbols | 7 + src/tpc.cpp | 384 +++++++++++++++++++++++++++++++++++++ 4 files changed, 472 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 cmake/FindXrootd.cmake create mode 100644 configs/export-lib-symbols create mode 100644 src/tpc.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000000..d62dcc98b26 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,40 @@ + +cmake_minimum_required( VERSION 2.8 ) +project( xrootd-tpc ) + +set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ) + +find_package( Xrootd REQUIRED ) + +macro(use_cxx11) + if (CMAKE_VERSION VERSION_LESS "3.1") + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "-std=gnu++11 ${CMAKE_CXX_FLAGS}") + endif () + else () + set (CMAKE_CXX_STANDARD 11) + endif () +endmacro(use_cxx11) +use_cxx11() + +if( CMAKE_COMPILER_IS_GNUCXX ) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror" ) +endif() +SET( CMAKE_SHARED_LINKER_FLAGS "-Wl,--no-undefined") +SET( CMAKE_MODULE_LINKER_FLAGS "-Wl,--no-undefined") + +include (FindPkgConfig) +pkg_check_modules(CURL REQUIRED libcurl) + +include_directories(${XROOTD_INCLUDES} ${CURL_INCLUDE_DIRS}) + +add_library(XrdHttpTPC SHARED src/tpc.cpp) +target_link_libraries(XrdHttpTPC -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${XROOTD_HTTP_LIB} ${CURL_LIBRARIES}) +set_target_properties(XrdHttpTPC PROPERTIES OUTPUT_NAME "XrdHttpTPC-4" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols") + +SET(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib" CACHE PATH "Install path for libraries") + +install( + TARGETS XrdHttpTPC + LIBRARY DESTINATION ${LIB_INSTALL_DIR} +) diff --git a/cmake/FindXrootd.cmake b/cmake/FindXrootd.cmake new file mode 100644 index 00000000000..4cb4b526abb --- /dev/null +++ b/cmake/FindXrootd.cmake @@ -0,0 +1,41 @@ + +FIND_PATH(XROOTD_INCLUDES XrdHttp/XrdHttpExtHandler.hh + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd/ + PATH_SUFFIXES include/xrootd + PATHS /opt/xrootd +) + +FIND_LIBRARY(XROOTD_UTILS_LIB XrdUtils + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd/ + PATH_SUFFIXES lib +) + +FIND_LIBRARY(XROOTD_SERVER_LIB XrdServer + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd/ + PATH_SUFFIXES lib +) + +FIND_LIBRARY(XROOTD_HTTP_LIB XrdHttp-4 + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt/xrootd/ + PATH_SUFFIXES lib +) + +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Xrootd DEFAULT_MSG XROOTD_UTILS_LIB XROOTD_HTTP_LIB XROOTD_INCLUDES) + diff --git a/configs/export-lib-symbols b/configs/export-lib-symbols new file mode 100644 index 00000000000..4e6f0d221d8 --- /dev/null +++ b/configs/export-lib-symbols @@ -0,0 +1,7 @@ +{ +global: + XrdHttpGetExtHandler*; + +local: + *; +}; diff --git a/src/tpc.cpp b/src/tpc.cpp new file mode 100644 index 00000000000..e1a618c0077 --- /dev/null +++ b/src/tpc.cpp @@ -0,0 +1,384 @@ + +#include "XrdHttp/XrdHttpExtHandler.hh" +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucPinPath.hh" +#include "XrdOuc/XrdOucStream.hh" +#include "XrdSec/XrdSecEntity.hh" +#include "XrdSfs/XrdSfsInterface.hh" +#include "XrdVersion.hh" + +#include + +#include +#include + +#include +#include +#include + +XrdVERSIONINFO(XrdHttpGetExtHandler, HttpTPC); +extern XrdSfsFileSystem *XrdSfsGetDefaultFileSystem(XrdSfsFileSystem *native_fs, + XrdSysLogger *lp, + const char *configfn, + XrdOucEnv *EnvInfo); + + +static char *quote(const char *str) { + int l = strlen(str); + char *r = (char *) malloc(l*3 + 1); + r[0] = '\0'; + int i, j = 0; + + for (i = 0; i < l; i++) { + char c = str[i]; + + switch (c) { + case ' ': + strcpy(r + j, "%20"); + j += 3; + break; + case '[': + strcpy(r + j, "%5B"); + j += 3; + break; + case ']': + strcpy(r + j, "%5D"); + j += 3; + break; + case ':': + strcpy(r + j, "%3A"); + j += 3; + break; + case '/': + strcpy(r + j, "%2F"); + j += 3; + break; + default: + r[j++] = c; + } + } + + r[j] = '\0'; + + return r; +} + + +static XrdSfsFileSystem *load_sfs(void *handle, bool alt, XrdSysError &log, const std::string &libpath, const char *configfn, XrdOucEnv &myEnv, XrdSfsFileSystem *prior_sfs) { + XrdSfsFileSystem *sfs = nullptr; + if (alt) { + auto ep = (XrdSfsFileSystem *(*)(XrdSfsFileSystem *, XrdSysLogger *, const char *, XrdOucEnv *)) + (dlsym(handle, "XrdSfsGetFileSystem2")); + if (ep == nullptr) { + log.Emsg("Config", "Failed to load XrdSfsGetFileSystem2 from library ", libpath.c_str(), dlerror()); + return nullptr; + } + sfs = ep(prior_sfs, log.logger(), configfn, &myEnv); + } else { + auto ep = (XrdSfsFileSystem *(*)(XrdSfsFileSystem *, XrdSysLogger *, const char *)) + (dlsym(nullptr, "XrdSfsGetFileSystem")); + if (ep == nullptr) { + log.Emsg("Config", "Failed to load XrdSfsGetFileSystem from library ", libpath.c_str(), dlerror()); + return nullptr; + } + sfs = ep(prior_sfs, log.logger(), configfn); + } + if (!sfs) { + log.Emsg("Config", "Failed to initialize filesystem library for XrdHttpTPC from ", libpath.c_str()); + return nullptr; + } + return sfs; +} + +class XrdHttpTPCState { +public: + XrdHttpTPCState (XrdSfsFile &fh, CURL *curl) : + m_fh(fh) + { + InstallHandlers(curl); + } + + bool InstallHandlers(CURL *curl) { + curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/0.1"); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &XrdHttpTPCState::HeaderCB); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, this); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &XrdHttpTPCState::WriteCB); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); + return true; + } + + int GetStatus() const {return m_status_code;} + +private: + static size_t HeaderCB(char *buffer, size_t size, size_t nitems, void *userdata) { + XrdHttpTPCState *obj = static_cast(userdata); + std::string header(buffer, size*nitems); + return obj->Header(header); + } + + int Header(const std::string &header) { + // TODO: Handle status codes appropriately. + //printf("Recieved remote header: %s\n", header.c_str()); + return header.size(); + } + + static size_t WriteCB(void *buffer, size_t size, size_t nitems, void *userdata) { + XrdHttpTPCState *obj = static_cast(userdata); + return obj->Write(static_cast(buffer), size*nitems); + } + + int Write(char *buffer, size_t size) { + int retval = m_fh.write(m_offset, buffer, size); + if (retval == SFS_ERROR) { + return -1; + } + m_offset += retval; + return retval; + } + + XrdSfsXferSize m_offset{0}; + int m_status_code{-1}; + XrdSfsFile &m_fh; +}; + + +class XrdHttpTPC : public XrdHttpExtHandler { +public: + virtual bool MatchesPath(const char *verb, const char *path) { + return !strcmp(verb, "COPY"); + } + + virtual int ProcessReq(XrdHttpExtReq &req) { + auto header = req.headers.find("Source"); + if (header != req.headers.end()) { + return ProcessPullReq(header->second, req); + } + header = req.headers.find("Destination"); + if (header != req.headers.end()) { + return ProcessPushReq(header->second, req); + } + return req.SendSimpleResp(400, NULL, NULL, (char *)"No Source or Destination specified", 0); + } + + /** + * Abstract method in the base class, but does not seem to be used. + */ + virtual int Init(const char *cfgfile) { + return 0; + } + + virtual ~XrdHttpTPC() { + m_sfs = nullptr; // NOTE: must delete the SFS here as we may unload the destructor from memory below! + if (m_handle_base) { + dlclose(m_handle_base); + m_handle_base = nullptr; + } + if (m_handle_chained) { + dlclose(m_handle_chained); + m_handle_chained = nullptr; + } + } + + XrdHttpTPC(XrdSysError *log, const char *config, XrdOucEnv *myEnv) : + m_log(*log) + { + if (!Configure(config, myEnv)) { + throw std::runtime_error("Failed to configure the HTTP third-party-copy handler."); + } + } + +private: + int ProcessPushReq(const std::string & /* Resource */, XrdHttpExtReq &req) { + char msg[] = "Push mode for COPY not implemented"; + return req.SendSimpleResp(501, nullptr, nullptr, msg, 0); + } + + int ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) { + CURL *curl = curl_easy_init(); + if (!curl) { + char msg[] = "Failed to initialize internal transfer resources"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + XrdSfsFile *fh = m_sfs->newFile(req.GetSecEntity().name, m_monid++); + if (!fh) { + char msg[] = "Failed to initialize internal transfer file handle"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + std::string authz; + auto authz_header = req.headers.find("Authorization"); + if (authz_header != req.headers.end()) { + char * quoted_url = quote(authz_header->second.c_str()); + std::stringstream ss; + ss << "authz=" << quoted_url; + free(quoted_url); + authz = ss.str(); + } + if (SFS_OK != fh->open(req.resource.c_str(), SFS_O_CREAT, 0600, &(req.GetSecEntity()), authz.empty() ? nullptr : authz.c_str())) { + char msg[] = "Failed to open local resource"; + return req.SendSimpleResp(400, nullptr, nullptr, msg, 0); + } + + CURLcode res; + curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); + + XrdHttpTPCState(*fh, curl); + res = curl_easy_perform(curl); + if (res) { + m_log.Emsg("ProcessPullReq", "Curl failed", curl_easy_strerror(res)); + char msg[] = "Unknown internal transfer failure"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } else { + char msg[] = "Created"; + return req.SendSimpleResp(201, nullptr, nullptr, msg, 0); + } + } + + bool ConfigureFSLib(XrdOucStream &Config, std::string &path1, bool &path1_alt, std::string &path2, bool &path2_alt) { + char *val; + if (!(val = Config.GetWord())) { + m_log.Emsg("Config", "fslib not specified"); + return false; + } + if (!strcmp("throttle", val)) { + path2 = "libXrdThrottle.so"; + if (!(val = Config.GetWord())) { + m_log.Emsg("Config", "fslib throttle target library not specified"); + return false; + } + } + else if (!strcmp("-2", val)) { + path2_alt = true; + if (!(val = Config.GetWord())) { + m_log.Emsg("Config", "fslib library not specified"); + return false; + } + path2 = val; + } + else { + path2 = val; + } + if (!(val = Config.GetWord()) || !strcmp("default", val)) { + if (path2 == "libXrdThrottle.so") { + path1 = "default"; + } else if (!path2.empty()) { + path1 = path2; + path2 = ""; + path1_alt = path2_alt; + } + } else if (!strcmp("-2", val)) { + path1_alt = true; + if (!(val = Config.GetWord())) { + m_log.Emsg("Config", "fslib base library not specified"); + return false; + } + path1 = val; + } else { + path2 = val; + } + return true; + } + + bool Configure(const char *configfn, XrdOucEnv *myEnv) { + + XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), myEnv, "=====> "); + + std::string authLib; + std::string authLibParms; + int cfgFD = open(configfn, O_RDONLY, 0); + if (cfgFD < 0) { + m_log.Emsg("Config", errno, "open config file", configfn); + return false; + } + Config.Attach(cfgFD); + const char *val; + std::string path2, path1 = "default"; + bool path1_alt = false, path2_alt = false; + while ((val = Config.GetMyFirstWord())) { + if (!strcmp("xrootd.fslib", val)) { + if (!ConfigureFSLib(Config, path1, path1_alt, path2, path2_alt)) { + Config.Close(); + m_log.Emsg("Config", "Failed to parse the xrootd.fslib directive"); + return false; + } + m_log.Emsg("Config", "xrootd.fslib line successfully processed by XrdHttpTPC"); + } + } + Config.Close(); + + XrdSfsFileSystem *base_sfs = nullptr; + if (path1 == "default") { + m_log.Emsg("Config", "Loading the default filesystem"); + base_sfs = XrdSfsGetDefaultFileSystem(nullptr, m_log.logger(), configfn, myEnv); + m_log.Emsg("Config", "Finished loading the default filesystem"); + } else { + char resolvePath[2048]; + bool usedAltPath{true}; + if (!XrdOucPinPath(path1.c_str(), usedAltPath, resolvePath, 2048)) { + m_log.Emsg("Config", "Failed to locate appropriately versioned base filesystem library for ", path1.c_str()); + return false; + } + m_handle_base = dlopen(resolvePath, RTLD_LOCAL|RTLD_NOW); + if (m_handle_base == nullptr) { + m_log.Emsg("Config", "Failed to base plugin ", resolvePath, dlerror()); + return false; + } + base_sfs = load_sfs(m_handle_base, path1_alt, m_log, path1, configfn, *myEnv, nullptr); + } + if (!base_sfs) { + m_log.Emsg("Config", "Failed to initialize filesystem library for XrdHttpTPC from ", path1.c_str()); + return false; + } + XrdSfsFileSystem *chained_sfs = nullptr; + if (!path2.empty()) { + char resolvePath[2048]; + bool usedAltPath{true}; + if (!XrdOucPinPath(path2.c_str(), usedAltPath, resolvePath, 2048)) { + m_log.Emsg("Config", "Failed to locate appropriately versioned chained filesystem library for ", path2.c_str()); + return false; + } + m_handle_chained = dlopen(resolvePath, RTLD_LOCAL|RTLD_NOW); + if (m_handle_chained == nullptr) { + m_log.Emsg("Config", "Failed to chained plugin ", resolvePath, dlerror()); + return false; + } + chained_sfs = load_sfs(m_handle_chained, path2_alt, m_log, path2, configfn, *myEnv, base_sfs); + } + m_sfs.reset(chained_sfs ? chained_sfs : base_sfs); + m_log.Emsg("Config", "Successfully configured the filesystem object for XrdHttpTPC"); + return true; + } + + static std::atomic m_monid; + XrdSysError &m_log; + std::unique_ptr m_sfs; + void *m_handle_base{nullptr}; + void *m_handle_chained{nullptr}; +}; + +std::atomic XrdHttpTPC::m_monid{0}; + +extern "C" { + +XrdHttpExtHandler *XrdHttpGetExtHandler(XrdSysError *log, const char * config, const char * /*parms*/, XrdOucEnv *myEnv) { + if (curl_global_init(CURL_GLOBAL_DEFAULT)) { + log->Emsg("Initialize", "libcurl failed to initialize"); + return nullptr; + } + + XrdHttpTPC *retval{nullptr}; + if (!config) { + log->Emsg("Initialize", "XrdHttpTPC requires a config filename in order to load"); + return nullptr; + } + try { + log->Emsg("Initialize", "Will load configuration for XrdHttpTPC from", config); + retval = new XrdHttpTPC(log, config, myEnv); + } catch (std::runtime_error &re) { + log->Emsg("Initialize", "Encountered a runtime failure when loading ", re.what()); + printf("Provided env vars: %p, XrdInet*: %p\n", myEnv, myEnv->GetPtr("XrdInet*")); + } + return retval; +} + +} + From 4bd2126488832c44ba528a10dbe335a30238f3de Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Sat, 11 Nov 2017 10:00:40 -0600 Subject: [PATCH 03/29] Working version of push and pull mode for COPY. This provides a working implementation of the push mode for COPY. Additionally, this now compiles against the latest Xrootd build, implements overwrite mode, and correctly copies headers into the destination transfer. As of this tag, a Xrootd server can copy files to and from itself. --- CMakeLists.txt | 2 +- cmake/FindXrootd.cmake | 14 +++- src/XrdTpcVersion.hh | 3 + src/tpc.cpp | 183 +++++++++++++++++++++++++++++++++++------ tools/xrootd-test-tpc | 85 +++++++++++++++++++ 5 files changed, 260 insertions(+), 27 deletions(-) create mode 100644 src/XrdTpcVersion.hh create mode 100755 tools/xrootd-test-tpc diff --git a/CMakeLists.txt b/CMakeLists.txt index d62dcc98b26..ec9201fefbd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ SET( CMAKE_MODULE_LINKER_FLAGS "-Wl,--no-undefined") include (FindPkgConfig) pkg_check_modules(CURL REQUIRED libcurl) -include_directories(${XROOTD_INCLUDES} ${CURL_INCLUDE_DIRS}) +include_directories(${XROOTD_INCLUDES} ${XROOTD_PRIVATE_INCLUDES} ${CURL_INCLUDE_DIRS}) add_library(XrdHttpTPC SHARED src/tpc.cpp) target_link_libraries(XrdHttpTPC -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${XROOTD_HTTP_LIB} ${CURL_LIBRARIES}) diff --git a/cmake/FindXrootd.cmake b/cmake/FindXrootd.cmake index 4cb4b526abb..e2e61d30160 100644 --- a/cmake/FindXrootd.cmake +++ b/cmake/FindXrootd.cmake @@ -1,5 +1,5 @@ -FIND_PATH(XROOTD_INCLUDES XrdHttp/XrdHttpExtHandler.hh +FIND_PATH(XROOTD_INCLUDES XrdVersion.hh HINTS ${XROOTD_DIR} $ENV{XROOTD_DIR} @@ -9,6 +9,16 @@ FIND_PATH(XROOTD_INCLUDES XrdHttp/XrdHttpExtHandler.hh PATHS /opt/xrootd ) +FIND_PATH(XROOTD_PRIVATE_INCLUDES XrdHttp/XrdHttpExtHandler.hh + HINTS + ${XROOTD_DIR}/private + $ENV{XROOTD_DIR}/private + /usr + /opt/xrootd/ + PATH_SUFFIXES include/xrootd/private + PATHS /opt/xrootd +) + FIND_LIBRARY(XROOTD_UTILS_LIB XrdUtils HINTS ${XROOTD_DIR} @@ -37,5 +47,5 @@ FIND_LIBRARY(XROOTD_HTTP_LIB XrdHttp-4 ) INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(Xrootd DEFAULT_MSG XROOTD_UTILS_LIB XROOTD_HTTP_LIB XROOTD_INCLUDES) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Xrootd DEFAULT_MSG XROOTD_UTILS_LIB XROOTD_HTTP_LIB XROOTD_INCLUDES XROOTD_PRIVATE_INCLUDES) diff --git a/src/XrdTpcVersion.hh b/src/XrdTpcVersion.hh new file mode 100644 index 00000000000..64d6a3cae1b --- /dev/null +++ b/src/XrdTpcVersion.hh @@ -0,0 +1,3 @@ + +#define XRDTPC_VERSION "0.2" + diff --git a/src/tpc.cpp b/src/tpc.cpp index e1a618c0077..035e5e7c4d1 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -16,6 +16,8 @@ #include #include +#include "XrdTpcVersion.hh" + XrdVERSIONINFO(XrdHttpGetExtHandler, HttpTPC); extern XrdSfsFileSystem *XrdSfsGetDefaultFileSystem(XrdSfsFileSystem *native_fs, XrdSysLogger *lp, @@ -92,21 +94,58 @@ static XrdSfsFileSystem *load_sfs(void *handle, bool alt, XrdSysError &log, cons class XrdHttpTPCState { public: - XrdHttpTPCState (XrdSfsFile &fh, CURL *curl) : - m_fh(fh) + XrdHttpTPCState (XrdSfsFile &fh, CURL *curl, bool push) : + m_push(push), + m_fh(fh), + m_curl(curl) { InstallHandlers(curl); } + ~XrdHttpTPCState() { + if (m_headers) { + curl_slist_free_all(m_headers); + m_headers = nullptr; + curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, m_headers); + } + } + bool InstallHandlers(CURL *curl) { - curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/0.1"); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/" XRDTPC_VERSION); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &XrdHttpTPCState::HeaderCB); curl_easy_setopt(curl, CURLOPT_HEADERDATA, this); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &XrdHttpTPCState::WriteCB); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); + if (m_push) { + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, &XrdHttpTPCState::ReadCB); + curl_easy_setopt(curl, CURLOPT_READDATA, this); + struct stat buf; + if (SFS_OK == m_fh.stat(&buf)) { + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, buf.st_size); + } + } else { + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &XrdHttpTPCState::WriteCB); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); + } + curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); return true; } + /** + * Handle the 'Copy-Headers' feature + */ + void CopyHeaders(XrdHttpExtReq &req) { + struct curl_slist *list = NULL; + for (auto &hdr : req.headers) { + if (hdr.first == "Copy-Header") { + list = curl_slist_append(list, hdr.second.c_str()); + } + } + if (list != nullptr) { + curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, list); + m_headers = list; + } + } + int GetStatus() const {return m_status_code;} private: @@ -118,7 +157,7 @@ class XrdHttpTPCState { int Header(const std::string &header) { // TODO: Handle status codes appropriately. - //printf("Recieved remote header: %s\n", header.c_str()); + printf("Recieved remote header: %s\n", header.c_str()); return header.size(); } @@ -136,19 +175,39 @@ class XrdHttpTPCState { return retval; } + static size_t ReadCB(void *buffer, size_t size, size_t nitems, void *userdata) { + XrdHttpTPCState *obj = static_cast(userdata); + return obj->Read(static_cast(buffer), size*nitems); + } + + int Read(char *buffer, size_t size) { + int retval = m_fh.read(m_offset, buffer, size); + if (retval == SFS_ERROR) { + return -1; + } + m_offset += retval; + return retval; + } + + bool m_push{true}; XrdSfsXferSize m_offset{0}; int m_status_code{-1}; XrdSfsFile &m_fh; + CURL *m_curl{nullptr}; + struct curl_slist *m_headers{nullptr}; }; class XrdHttpTPC : public XrdHttpExtHandler { public: virtual bool MatchesPath(const char *verb, const char *path) { - return !strcmp(verb, "COPY"); + return !strcmp(verb, "COPY") || !strcmp(verb, "OPTIONS"); } virtual int ProcessReq(XrdHttpExtReq &req) { + if (req.verb == "OPTIONS") { + return ProcessOptionsReq(req); + } auto header = req.headers.find("Source"); if (header != req.headers.end()) { return ProcessPullReq(header->second, req); @@ -188,9 +247,72 @@ class XrdHttpTPC : public XrdHttpExtHandler { } private: - int ProcessPushReq(const std::string & /* Resource */, XrdHttpExtReq &req) { - char msg[] = "Push mode for COPY not implemented"; - return req.SendSimpleResp(501, nullptr, nullptr, msg, 0); + + /** + * Handle the OPTIONS verb as we have added a new one... + */ + int ProcessOptionsReq(XrdHttpExtReq &req) { + return req.SendSimpleResp(200, NULL, (char *) "DAV: 1\r\nDAV: \r\nAllow: HEAD,GET,PUT,PROPFIND,DELETE,OPTIONS,COPY", NULL, 0); + } + + static std::string GetAuthz(XrdHttpExtReq &req) { + std::string authz; + auto authz_header = req.headers.find("Authorization"); + if (authz_header != req.headers.end()) { + char * quoted_url = quote(authz_header->second.c_str()); + std::stringstream ss; + ss << "authz=" << quoted_url; + free(quoted_url); + authz = ss.str(); + } + return authz; + } + + int ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) { + CURL *curl = curl_easy_init(); + if (!curl) { + char msg[] = "Failed to initialize internal transfer resources"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + char *name = req.GetSecEntity().name; + XrdSfsFile *fh = m_sfs->newFile(name, m_monid++); + if (!fh) { + char msg[] = "Failed to initialize internal transfer file handle"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + std::string authz = GetAuthz(req); + + if (SFS_OK != fh->open(req.resource.c_str(), SFS_O_RDONLY, 0600, &(req.GetSecEntity()), authz.empty() ? nullptr : authz.c_str())) { + int code; + char msg_generic[] = "Failed to open local resource"; + const char *msg = fh->error.getErrText(code); + if (msg == nullptr) msg = msg_generic; + int status_code = 400; + if (code == EACCES) status_code = 401; + fh->close(); + delete fh; + return req.SendSimpleResp(status_code, nullptr, nullptr, const_cast(msg), 0); + } + + CURLcode res; + curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); + + XrdHttpTPCState state(*fh, curl, true); + state.CopyHeaders(req); + res = curl_easy_perform(curl); + fh->close(); + delete fh; + if (res == CURLE_HTTP_RETURNED_ERROR) { + m_log.Emsg("ProcessPushReq", "Remote server failed request", curl_easy_strerror(res)); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); + } else if (res) { + m_log.Emsg("ProcessPushReq", "Curl failed", curl_easy_strerror(res)); + char msg[] = "Unknown internal transfer failure"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } else { + char msg[] = "Created"; + return req.SendSimpleResp(201, nullptr, nullptr, msg, 0); + } } int ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) { @@ -199,31 +321,44 @@ class XrdHttpTPC : public XrdHttpExtHandler { char msg[] = "Failed to initialize internal transfer resources"; return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } - XrdSfsFile *fh = m_sfs->newFile(req.GetSecEntity().name, m_monid++); + char *name = req.GetSecEntity().name; + XrdSfsFile *fh = m_sfs->newFile(name, m_monid++); if (!fh) { char msg[] = "Failed to initialize internal transfer file handle"; return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } - std::string authz; - auto authz_header = req.headers.find("Authorization"); - if (authz_header != req.headers.end()) { - char * quoted_url = quote(authz_header->second.c_str()); - std::stringstream ss; - ss << "authz=" << quoted_url; - free(quoted_url); - authz = ss.str(); + std::string authz = GetAuthz(req); + XrdSfsFileOpenMode mode = SFS_O_CREAT; + auto overwrite_header = req.headers.find("Overwrite"); + if ((overwrite_header == req.headers.end()) || (overwrite_header->second == "T")) { + mode = SFS_O_TRUNC|SFS_O_POSC; } - if (SFS_OK != fh->open(req.resource.c_str(), SFS_O_CREAT, 0600, &(req.GetSecEntity()), authz.empty() ? nullptr : authz.c_str())) { - char msg[] = "Failed to open local resource"; - return req.SendSimpleResp(400, nullptr, nullptr, msg, 0); + + if (SFS_OK != fh->open(req.resource.c_str(), mode, 0600, &(req.GetSecEntity()), authz.empty() ? nullptr : authz.c_str())) { + int code; + char msg_generic[] = "Failed to open local resource"; + const char *msg = fh->error.getErrText(code); + if (msg == nullptr) msg = msg_generic; + int status_code = 400; + if (code == EACCES) status_code = 401; + if (code == EEXIST) status_code = 412; + fh->close(); + delete fh; + return req.SendSimpleResp(status_code, nullptr, nullptr, const_cast(msg), 0); } CURLcode res; curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - XrdHttpTPCState(*fh, curl); + XrdHttpTPCState state(*fh, curl, false); + state.CopyHeaders(req); res = curl_easy_perform(curl); - if (res) { + fh->close(); + delete fh; + if (res == CURLE_HTTP_RETURNED_ERROR) { + m_log.Emsg("ProcessPullReq", "Remote server failed request", curl_easy_strerror(res)); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); + } else if (res) { m_log.Emsg("ProcessPullReq", "Curl failed", curl_easy_strerror(res)); char msg[] = "Unknown internal transfer failure"; return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); diff --git a/tools/xrootd-test-tpc b/tools/xrootd-test-tpc new file mode 100755 index 00000000000..5dce2abb62b --- /dev/null +++ b/tools/xrootd-test-tpc @@ -0,0 +1,85 @@ +#!/usr/bin/python + +import os +import sys +import argparse + +import requests + +def parse_args(): + parser = argparse.ArgumentParser(description='Drive test TPC transfers') + parser.add_argument("-t", "--token", help="Bearer token for transfer") + parser.add_argument("--dest-token", help="Bearer token for destination host") + parser.add_argument("--src-token", help="Bearer token for source host") + parser.add_argument("--push", dest="mode", action="store_const", const="push", help="Use push-mode for transfer (source manages transfer)", default="auto") + parser.add_argument("--pull", dest="mode", action="store_const", const="pull", help="Use pull-mode for transfer (destination manages transfer)") + parser.add_argument("--no-overwrite", dest="overwrite", action="store_false", default=True, help="Disable overwrite of existing files.") + parser.add_argument("src") + parser.add_argument("dest") + + args = parser.parse_args() + + if not args.token and (not args.src_token or not args.dest_token): + if 'SCITOKEN' in os.environ and os.path.exists(os.environ['SCITOKEN']): + args.token = os.environ['SCITOKEN'] + elif os.path.exists(os.path.expanduser("~/.scitokens/token")): + args.token = os.path.expanduser("~/.scitokens/token") + elif os.path.exists('/tmp/scitoken_u%d' % os.geteuid()): + args.token = '/tmp/scitoken_u%d' % os.geteuid() + else: + print >> sys.stderr, "No token file found in user environment." + sys.exit(1) + + if not args.src_token: + args.src_token = args.token + if not args.dest_token: + args.dest_token = args.token + + return args + +def get_token(fname): + with open(fname, "r") as fp: + for line in fp: + if line.startswith("#"): + continue + return line.strip() + raise Exception("No token found in specified file (%s)" % fname) + +def determine_mode(args): + verbs = requests.options(args.dest) + if 'allow' in verbs.headers: + if 'COPY' in verbs.headers['allow'].split(","): + return 'pull' + return 'push' + +def main(): + args = parse_args() + + src_token = get_token(args.src_token) + dest_token = get_token(args.dest_token) + + headers = {} + if args.overwrite: + headers['Overwrite'] = 'T' + else: + headers['Overwrite'] = 'F' + mode = args.mode + if mode == "auto": + mode = determine_mode(args) + print "Auto detect determined %s mode" % mode + if mode == 'pull': + headers['Authorization'] = 'Bearer %s' % dest_token + headers['Source'] = args.src + headers['Copy-Header'] = 'Authorization: Bearer %s' % src_token + resp = requests.request('COPY', args.dest, headers=headers) + else: # Push + headers['Authorization'] = 'Bearer %s' % src_token + headers['Destination'] = args.dest + headers['Copy-Header'] = 'Authorization: Bearer %s' % dest_token + resp = requests.request('COPY', args.src, headers=headers) + + print resp.status_code + print resp.text + +if __name__ == '__main__': + main() From 2afed061f896f28076bbcb0f1d527c8e0cbca27a Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 13 Nov 2017 02:43:19 -0600 Subject: [PATCH 04/29] Add support for redirections. With this, both push and pull mode can involve redirections. --- src/tpc.cpp | 132 ++++++++++++++++++++++++++++++++++-------- tools/xrootd-test-tpc | 31 ++++++---- 2 files changed, 126 insertions(+), 37 deletions(-) diff --git a/src/tpc.cpp b/src/tpc.cpp index 035e5e7c4d1..85d6af43674 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -94,9 +94,9 @@ static XrdSfsFileSystem *load_sfs(void *handle, bool alt, XrdSysError &log, cons class XrdHttpTPCState { public: - XrdHttpTPCState (XrdSfsFile &fh, CURL *curl, bool push) : + XrdHttpTPCState (std::unique_ptr fh, CURL *curl, bool push) : m_push(push), - m_fh(fh), + m_fh(std::move(fh)), m_curl(curl) { InstallHandlers(curl); @@ -108,6 +108,7 @@ class XrdHttpTPCState { m_headers = nullptr; curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, m_headers); } + m_fh->close(); } bool InstallHandlers(CURL *curl) { @@ -119,14 +120,14 @@ class XrdHttpTPCState { curl_easy_setopt(curl, CURLOPT_READFUNCTION, &XrdHttpTPCState::ReadCB); curl_easy_setopt(curl, CURLOPT_READDATA, this); struct stat buf; - if (SFS_OK == m_fh.stat(&buf)) { + if (SFS_OK == m_fh->stat(&buf)) { curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, buf.st_size); } } else { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &XrdHttpTPCState::WriteCB); curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); } - curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + //curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); return true; } @@ -146,7 +147,7 @@ class XrdHttpTPCState { } } - int GetStatus() const {return m_status_code;} + int GetStatusCode() const {return m_status_code;} private: static size_t HeaderCB(char *buffer, size_t size, size_t nitems, void *userdata) { @@ -158,16 +159,37 @@ class XrdHttpTPCState { int Header(const std::string &header) { // TODO: Handle status codes appropriately. printf("Recieved remote header: %s\n", header.c_str()); + if (m_recv_all_headers) { // This is the second request -- maybe processed a redirect? + m_recv_all_headers = false; + m_recv_status_line = false; + } + if (!m_recv_status_line) { + std::stringstream ss(header); + std::string item; + if (!std::getline(ss, item, ' ')) return 0; + m_resp_protocol = item; + printf("Response protocol: %s\n", m_resp_protocol.c_str()); + if (!std::getline(ss, item, ' ')) return 0; + try { + m_status_code = std::stol(item); + } catch (...) { + return 0; + } + m_recv_status_line = true; + } + if (header.size() == 0) {m_recv_all_headers = true;} return header.size(); } static size_t WriteCB(void *buffer, size_t size, size_t nitems, void *userdata) { XrdHttpTPCState *obj = static_cast(userdata); + if (obj->GetStatusCode() < 0) {return 0;} // malformed request - got body before headers. + if (obj->GetStatusCode() >= 400) {return 0;} // Status indicates failure. return obj->Write(static_cast(buffer), size*nitems); } int Write(char *buffer, size_t size) { - int retval = m_fh.write(m_offset, buffer, size); + int retval = m_fh->write(m_offset, buffer, size); if (retval == SFS_ERROR) { return -1; } @@ -177,11 +199,13 @@ class XrdHttpTPCState { static size_t ReadCB(void *buffer, size_t size, size_t nitems, void *userdata) { XrdHttpTPCState *obj = static_cast(userdata); + if (obj->GetStatusCode() < 0) {return 0;} // malformed request - got body before headers. + if (obj->GetStatusCode() >= 400) {return 0;} // Status indicates failure. return obj->Read(static_cast(buffer), size*nitems); } int Read(char *buffer, size_t size) { - int retval = m_fh.read(m_offset, buffer, size); + int retval = m_fh->read(m_offset, buffer, size); if (retval == SFS_ERROR) { return -1; } @@ -190,11 +214,14 @@ class XrdHttpTPCState { } bool m_push{true}; + bool m_recv_status_line{false}; + bool m_recv_all_headers{false}; XrdSfsXferSize m_offset{0}; int m_status_code{-1}; - XrdSfsFile &m_fh; + std::unique_ptr m_fh; CURL *m_curl{nullptr}; struct curl_slist *m_headers{nullptr}; + std::string m_resp_protocol; }; @@ -268,6 +295,33 @@ class XrdHttpTPC : public XrdHttpExtHandler { return authz; } + int RedirectTransfer(XrdHttpExtReq &req, XrdOucErrInfo &error) { + int port; + const char *host = error.getErrText(port); + if ((host == nullptr) || (*host == '\0') || (port == 0)) { + char msg[] = "Internal error: redirect without hostname"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + std::stringstream ss; + ss << "Location: http" << (m_desthttps ? "s" : "") << "://" << host << ":" << port << "/" << req.resource; + return req.SendSimpleResp(307, nullptr, const_cast(ss.str().c_str()), nullptr, 0); + } + + int OpenWaitStall(XrdSfsFile &fh, const std::string &resource, int mode, int openMode, const XrdSecEntity &sec, + const std::string &authz) { + int open_result; + while (1) { + open_result = fh.open(resource.c_str(), mode, openMode, &sec, authz.empty() ? nullptr : authz.c_str()); + if ((open_result == SFS_STALL) || (open_result == SFS_STARTED)) { + int secs_to_stall = fh.error.getErrInfo(); + if (open_result == SFS_STARTED) {secs_to_stall = secs_to_stall/2 + 5;} + sleep(secs_to_stall); + } + break; + } + return open_result; + } + int ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) { CURL *curl = curl_easy_init(); if (!curl) { @@ -275,36 +329,42 @@ class XrdHttpTPC : public XrdHttpExtHandler { return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } char *name = req.GetSecEntity().name; - XrdSfsFile *fh = m_sfs->newFile(name, m_monid++); - if (!fh) { + std::unique_ptr fh(m_sfs->newFile(name, m_monid++)); + if (!fh.get()) { char msg[] = "Failed to initialize internal transfer file handle"; return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } std::string authz = GetAuthz(req); - if (SFS_OK != fh->open(req.resource.c_str(), SFS_O_RDONLY, 0600, &(req.GetSecEntity()), authz.empty() ? nullptr : authz.c_str())) { + int open_results = OpenWaitStall(*fh, req.resource, SFS_O_RDONLY, 0644, req.GetSecEntity(), authz); + if (SFS_REDIRECT == open_results) { + return RedirectTransfer(req, fh->error); + } else if (SFS_OK != open_results) { int code; char msg_generic[] = "Failed to open local resource"; const char *msg = fh->error.getErrText(code); if (msg == nullptr) msg = msg_generic; int status_code = 400; if (code == EACCES) status_code = 401; + int resp_result = req.SendSimpleResp(status_code, nullptr, nullptr, const_cast(msg), 0); fh->close(); - delete fh; - return req.SendSimpleResp(status_code, nullptr, nullptr, const_cast(msg), 0); + return resp_result; } CURLcode res; curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - XrdHttpTPCState state(*fh, curl, true); + XrdHttpTPCState state(std::move(fh), curl, true); state.CopyHeaders(req); res = curl_easy_perform(curl); - fh->close(); - delete fh; if (res == CURLE_HTTP_RETURNED_ERROR) { m_log.Emsg("ProcessPushReq", "Remote server failed request", curl_easy_strerror(res)); return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); + } else if (state.GetStatusCode() >= 400) { + std::stringstream ss; + ss << "Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg("ProcessPushReq", "Remote server failed request", ss.str().c_str()); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); } else if (res) { m_log.Emsg("ProcessPushReq", "Curl failed", curl_easy_strerror(res)); char msg[] = "Unknown internal transfer failure"; @@ -322,8 +382,8 @@ class XrdHttpTPC : public XrdHttpExtHandler { return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } char *name = req.GetSecEntity().name; - XrdSfsFile *fh = m_sfs->newFile(name, m_monid++); - if (!fh) { + std::unique_ptr fh(m_sfs->newFile(name, m_monid++)); + if (!fh.get()) { char msg[] = "Failed to initialize internal transfer file handle"; return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } @@ -334,30 +394,36 @@ class XrdHttpTPC : public XrdHttpExtHandler { mode = SFS_O_TRUNC|SFS_O_POSC; } - if (SFS_OK != fh->open(req.resource.c_str(), mode, 0600, &(req.GetSecEntity()), authz.empty() ? nullptr : authz.c_str())) { + int open_result = OpenWaitStall(*fh, req.resource, mode|SFS_O_WRONLY, 0644, req.GetSecEntity(), authz); + if (SFS_REDIRECT == open_result) { + return RedirectTransfer(req, fh->error); + } else if (SFS_OK != open_result) { int code; char msg_generic[] = "Failed to open local resource"; const char *msg = fh->error.getErrText(code); - if (msg == nullptr) msg = msg_generic; + if ((msg == nullptr) || (*msg == '\0')) msg = msg_generic; int status_code = 400; if (code == EACCES) status_code = 401; if (code == EEXIST) status_code = 412; + int resp_result = req.SendSimpleResp(status_code, nullptr, nullptr, const_cast(msg), 0); fh->close(); - delete fh; - return req.SendSimpleResp(status_code, nullptr, nullptr, const_cast(msg), 0); + return resp_result; } CURLcode res; curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - XrdHttpTPCState state(*fh, curl, false); + XrdHttpTPCState state(std::move(fh), curl, false); state.CopyHeaders(req); res = curl_easy_perform(curl); - fh->close(); - delete fh; if (res == CURLE_HTTP_RETURNED_ERROR) { m_log.Emsg("ProcessPullReq", "Remote server failed request", curl_easy_strerror(res)); return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); + } else if (state.GetStatusCode() >= 400) { + std::stringstream ss; + ss << "Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg("ProcessPushReq", "Remote server failed request", ss.str().c_str()); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); } else if (res) { m_log.Emsg("ProcessPullReq", "Curl failed", curl_easy_strerror(res)); char msg[] = "Unknown internal transfer failure"; @@ -436,6 +502,21 @@ class XrdHttpTPC : public XrdHttpExtHandler { return false; } m_log.Emsg("Config", "xrootd.fslib line successfully processed by XrdHttpTPC"); + } else if (!strcmp("http.desthttps", val)) { + if (!(val = Config.GetWord())) { + Config.Close(); + m_log.Emsg("Config", "http.desthttps value not specified"); + return false; + } + if (!strcmp("1", val) || !strcasecmp("yes", val) || !strcasecmp("true", val)) { + m_desthttps = true; + } else if (!strcmp("1", val) || !strcasecmp("yes", val) || !strcasecmp("true", val)) { + m_desthttps = false; + } else { + Config.Close(); + m_log.Emsg("Config", "https.dests value is invalid", val); + return false; + } } } Config.Close(); @@ -483,6 +564,7 @@ class XrdHttpTPC : public XrdHttpExtHandler { return true; } + bool m_desthttps{false}; static std::atomic m_monid; XrdSysError &m_log; std::unique_ptr m_sfs; diff --git a/tools/xrootd-test-tpc b/tools/xrootd-test-tpc index 5dce2abb62b..851618ab390 100755 --- a/tools/xrootd-test-tpc +++ b/tools/xrootd-test-tpc @@ -67,19 +67,26 @@ def main(): if mode == "auto": mode = determine_mode(args) print "Auto detect determined %s mode" % mode - if mode == 'pull': - headers['Authorization'] = 'Bearer %s' % dest_token - headers['Source'] = args.src - headers['Copy-Header'] = 'Authorization: Bearer %s' % src_token - resp = requests.request('COPY', args.dest, headers=headers) - else: # Push - headers['Authorization'] = 'Bearer %s' % src_token - headers['Destination'] = args.dest - headers['Copy-Header'] = 'Authorization: Bearer %s' % dest_token - resp = requests.request('COPY', args.src, headers=headers) - print resp.status_code - print resp.text + with requests.Session() as session: + session.verify = '/etc/grid-security/certificates' + if mode == 'pull': + headers['Authorization'] = 'Bearer %s' % dest_token + headers['Source'] = args.src + headers['Copy-Header'] = 'Authorization: Bearer %s' % src_token + url = args.dest + else: # Push + headers['Authorization'] = 'Bearer %s' % src_token + headers['Destination'] = args.dest + headers['Copy-Header'] = 'Authorization: Bearer %s' % dest_token + url = args.src + try_again = True + while try_again: + resp = session.request('COPY', url, headers=headers, allow_redirects = True) + print resp.status_code + print resp.headers + print resp.text + break if __name__ == '__main__': main() From 81646717112753a323ba9713bc5d7bf63949edfc Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 13 Nov 2017 03:31:48 -0600 Subject: [PATCH 05/29] Add packaging support. --- rpm/xrootd-tpc.spec | 48 ++++++++++++++++++++++++++++++++++++++++++++ src/XrdTpcVersion.hh | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 rpm/xrootd-tpc.spec diff --git a/rpm/xrootd-tpc.spec b/rpm/xrootd-tpc.spec new file mode 100644 index 00000000000..d2153509974 --- /dev/null +++ b/rpm/xrootd-tpc.spec @@ -0,0 +1,48 @@ + +Name: xrootd-tpc +Version: 0.3 +Release: 1%{?dist} +Summary: HTTP Third Party Copy plugin for XRootD + +Group: System Environment/Daemons +License: BSD +URL: https://github.com/bbockelm/xrootd-tpc +# Generated from: +# git archive v%{version} --prefix=%{name}-%{version}/ | gzip -7 > ~/rpmbuild/SOURCES/%{name}-%{version}.tar.gz +Source0: %{name}-%{version}.tar.gz +BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) +BuildRequires: xrootd-devel +BuildRequires: xrootd-server-libs +BuildRequires: xrootd-server-devel +BuildRequires: xrootd-private-devel +BuildRequires: cmake +BuildRequires: gcc-c++ +BuildRequires: libcurl-devel + +%description +%{summary} + +%prep +%setup -q + +sed -i 's|".*"|"%{version}"|' src/XrdTpcVersion.hh + +%build +%cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo . +make VERBOSE=1 %{?_smp_mflags} + +%install +rm -rf $RPM_BUILD_ROOT +make install DESTDIR=$RPM_BUILD_ROOT + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%{_libdir}/libXrdHttpTPC-4.so + +%changelog +* Mon Nov 13 2017 Brian Bockelman - 0.3-1 +- Add support for redirections in COPY requests. +- Add RPM packaging. diff --git a/src/XrdTpcVersion.hh b/src/XrdTpcVersion.hh index 64d6a3cae1b..8f792c5138c 100644 --- a/src/XrdTpcVersion.hh +++ b/src/XrdTpcVersion.hh @@ -1,3 +1,3 @@ -#define XRDTPC_VERSION "0.2" +#define XRDTPC_VERSION "devel" From fead56d49476a2853f1493822b4f6cd029f5babb Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 13 Nov 2017 04:05:01 -0600 Subject: [PATCH 06/29] Add minor README lines. --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9dcad933bbd..6db604d9a7f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,10 @@ -# xrootd-tpc -A HTTPS third-party-copy implementation for Xrootd +# HTTPS Third Party Copy for XRootD + +The `xrootd-tpc` module provides an implementation of HTTPS third-party-copy +for the HTTPS implementation inside XRootD. + +To enable, set the following in the configuration file: + +``` +http.exthandler xrdtpc libXrdHttpTPC.so +``` From 7ee438af81e10897233b71a6d4d42be9b985b0a9 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Wed, 20 Dec 2017 14:37:25 -0600 Subject: [PATCH 07/29] Document the COPY protocol we intend to follow. Primarily from the documentation here: https://svnweb.cern.ch/trac/lcgdm/wiki/Dpm/WebDAV/Extensions#ThirdPartyCopies and the contents of various personal email threads. --- README.md | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/README.md b/README.md index 6db604d9a7f..a0845d23646 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,114 @@ To enable, set the following in the configuration file: ``` http.exthandler xrdtpc libXrdHttpTPC.so ``` + + +## HTTPS TPC technical details. + +Third-party-copy is done using the standard WebDAV copy verb. The actors involved are: + +1. The client orchestrating the transfer. +2. The source server where the resource is read from. +3. The destination server where the resource is written to. + +The client initiates the third party copy by issuing a COPY request to _either_ the source or destination +endpoint. Typically, when this is done, the initial endpoint redirects the client to a second service +(the disk server) that will performs the actual transfer. + +When the client contacts the source server, it should set the `Destination` header so the source knows +where to put the data. Analogously, if the client first contacts the destination server, it should set +the `Source` header. + +For the former case, the interaction looks like this: + +``` +-> COPY /sfn/of/source/replica HTTP/1.1 + Destination: https:///pfn/of/dest/replica +<- HTTP/1.1 302 Found + Location: https:///pfn/of/source/replica +``` + +where `` is the service that will actually perform the transfer. + +The client would follow the redirection and retry the `COPY`: + +``` +-> COPY /sfn/of/source/replica HTTP/1.1 + Destination: https:///pfn/of/dest/replica +<- HTTP/1.1 201 Created + Transfer-Encoding: chunked +``` + +The source server *should* respond with an appropriate status code (such as 201) in a timely manner. +As a TPC can take a significant amount of time, the source server SHOULD NOT wait until the transfer is +finished before responding with a success. In the case when a transfer can be started, it is recommended +that the response be started as soon as possible. + +In the case of a transfer being started (or queued) by the source server, it SHOULD use chunked encoding +and send a multipart response. + +Next `` will connect to the destination endpoint and actually perform the `PUT`: + +``` +-> PUT /pfn/of/destination/replica HTTP/1.1 +<- HTTP/1.1 201 Created +``` + +As the PUT is ongoing, the source disk server should send back a periodic transfer chunk of the following +form: + +``` +Perf Marker + Timestamp: 1360578938 + Stripe Index: 0 + Stripe Bytes Transferred: 49397760 + Total Stripe Count: 1 +End +``` + +Timestamps should be seconds from Unix epoch. It is recommended that the time period between chunks be +less than 30 seconds. + +If the transfer ultimately succeeds, then the last chunk should be of the following form: + +``` +success: Created +``` + +Here, `success` indicates the transfer status: subsequent text is informational for the user. + +Failure of the transfer can be indicated with the prefix `failed` or `failure`: + +``` +failure: Network connection unexpectedly closed. +``` + +Finally, if the source disk server decides to cancel or abort the transfer, use the `aborted` prefix: + +``` +aborted: Transfer took too long. +``` + +There is an analogous case when the client contacts the destination server to perform the `COPY` and +sets the `Source` header. In such a case, the response to the client looks identical but the destination +server will perform a `GET` instead of a `PUT`. + +In some cases, the user may want the server performing the transfer to append additional headers +(such as an HTTP `Authorization` header) to the transfer it initiates. In such case, the client should +utilize the `TransferHeader` mechanism. Add a header of the form: + +``` +TransferHeader
: +``` + +For example, if the client sends the following to the source server as part of its `COPY` request: + +``` +TransferHeaderAuthorization: Basic cGF1bDpwYXNzd29yZA== +``` + +then the source server should set the following header as part of its `PUT` request to the destination: + +``` +Authorization: Basic cGF1bDpwYXNzd29yZA== +``` From 6ea5749bf9a6eecaf4bde3961e5fe3ba8e55d397 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Wed, 20 Dec 2017 20:00:54 -0600 Subject: [PATCH 08/29] Detect the presence of chunked response support. With the chunked transfer-encoding in the Xrootd external handler framework, we can return HTTP performance markers to the client. --- CMakeLists.txt | 19 +++++++++++++++++++ cmake/FindXrootd.cmake | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec9201fefbd..d3a8ed67434 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,21 @@ macro(use_cxx11) endmacro(use_cxx11) use_cxx11() +# Chunked responses were introduced after Xrootd 4.8.0. Check to see if the symbol exists. +#include (CheckCXXSymbolExists) +SET( CMAKE_REQUIRED_INCLUDES "${XROOTD_PRIVATE_INCLUDES}" ) +SET( CMAKE_REQUIRED_LIBRARIES "${XROOTD_HTTP_LIB}" ) +SET( CMAKE_REQUIRED_FLAGS "" ) +include (CheckCXXSourceCompiles) +CHECK_CXX_SOURCE_COMPILES("#include + +int main(int argc, char** argv) +{ + (void)argv; + return ((int*)reinterpret_cast(&XrdHttpExtReq::ChunkResp))[argc]; +} +" XRD_CHUNK_RESP) + if( CMAKE_COMPILER_IS_GNUCXX ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror" ) endif() @@ -29,6 +44,10 @@ pkg_check_modules(CURL REQUIRED libcurl) include_directories(${XROOTD_INCLUDES} ${XROOTD_PRIVATE_INCLUDES} ${CURL_INCLUDE_DIRS}) add_library(XrdHttpTPC SHARED src/tpc.cpp) +if ( XRD_CHUNK_RESP ) + set_target_properties(XrdHttpTPC PROPERTIES COMPILE_DEFINITIONS "XRD_CHUNK_RESP" ) +endif () + target_link_libraries(XrdHttpTPC -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${XROOTD_HTTP_LIB} ${CURL_LIBRARIES}) set_target_properties(XrdHttpTPC PROPERTIES OUTPUT_NAME "XrdHttpTPC-4" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols") diff --git a/cmake/FindXrootd.cmake b/cmake/FindXrootd.cmake index e2e61d30160..ad951ba90c2 100644 --- a/cmake/FindXrootd.cmake +++ b/cmake/FindXrootd.cmake @@ -11,8 +11,8 @@ FIND_PATH(XROOTD_INCLUDES XrdVersion.hh FIND_PATH(XROOTD_PRIVATE_INCLUDES XrdHttp/XrdHttpExtHandler.hh HINTS - ${XROOTD_DIR}/private - $ENV{XROOTD_DIR}/private + ${XROOTD_DIR} + $ENV{XROOTD_DIR} /usr /opt/xrootd/ PATH_SUFFIXES include/xrootd/private From 7da01d330f53ca02721271adcb7650c9d89c0f1e Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Wed, 20 Dec 2017 20:04:54 -0600 Subject: [PATCH 09/29] Add initial performance marker support. If Xrootd supports chunked transfer encoding, then send performance markers along with the COPY request response. --- src/tpc.cpp | 226 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 184 insertions(+), 42 deletions(-) diff --git a/src/tpc.cpp b/src/tpc.cpp index 85d6af43674..e0e66e0875e 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -111,6 +111,8 @@ class XrdHttpTPCState { m_fh->close(); } + off_t BytesTransferred() const {return m_offset;} + bool InstallHandlers(CURL *curl) { curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/" XRDTPC_VERSION); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &XrdHttpTPCState::HeaderCB); @@ -127,6 +129,7 @@ class XrdHttpTPCState { curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &XrdHttpTPCState::WriteCB); curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); } + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); //curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); return true; } @@ -147,6 +150,11 @@ class XrdHttpTPCState { } } + /** + * Perform the curl-based transfer, responding periodically with transfer + * markers. + */ + int GetStatusCode() const {return m_status_code;} private: @@ -157,8 +165,7 @@ class XrdHttpTPCState { } int Header(const std::string &header) { - // TODO: Handle status codes appropriately. - printf("Recieved remote header: %s\n", header.c_str()); + printf("Recieved remote header (%d, %d): %s", m_recv_all_headers, m_recv_status_line, header.c_str()); if (m_recv_all_headers) { // This is the second request -- maybe processed a redirect? m_recv_all_headers = false; m_recv_status_line = false; @@ -168,7 +175,7 @@ class XrdHttpTPCState { std::string item; if (!std::getline(ss, item, ' ')) return 0; m_resp_protocol = item; - printf("Response protocol: %s\n", m_resp_protocol.c_str()); + printf("\n\nResponse protocol: %s\n", m_resp_protocol.c_str()); if (!std::getline(ss, item, ' ')) return 0; try { m_status_code = std::stol(item); @@ -177,7 +184,7 @@ class XrdHttpTPCState { } m_recv_status_line = true; } - if (header.size() == 0) {m_recv_all_headers = true;} + if (header.size() == 0 || header == "\n") {m_recv_all_headers = true;} return header.size(); } @@ -322,6 +329,166 @@ class XrdHttpTPC : public XrdHttpExtHandler { return open_result; } +#ifdef XRD_CHUNK_RESP + int SendPerfMarker(XrdHttpExtReq &req, XrdHttpTPCState &state) { + std::stringstream ss; + const std::string crlf = "\r\n"; + ss << "Perf Marker" << crlf; + ss << " Timestamp: " << time(NULL) << crlf; + ss << " Stripe Index: 0" << crlf; + ss << " Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; + ss << " Total Stripe Count: 1" << crlf; + ss << "End" << crlf; + + return req.ChunkResp(ss.str().c_str(), 0); + } + + int RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, XrdHttpTPCState &state, + const char *log_prefix) { + + // Create the multi-handle and add in the current transfer to it. + CURLM *multi_handle = curl_multi_init(); + if (!multi_handle) { + m_log.Emsg(log_prefix, "Failed to initialize a libcurl multi-handle"); + char msg[] = "Failed to initialize internal server memory"; + curl_easy_cleanup(curl); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + + CURLMcode mres; + mres = curl_multi_add_handle(multi_handle, curl); + if (mres) { + m_log.Emsg(log_prefix, "Failed to add transfer to libcurl multi-handle", + curl_multi_strerror(mres)); + char msg[] = "Failed to initialize internal server handle"; + curl_easy_cleanup(curl); + curl_multi_cleanup(multi_handle); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + + // Start response to client prior to the first call to curl_multi_perform + int retval = req.StartChunkedResp(201, "Created", "Content-Type: text/plain"); + if (retval) { + curl_easy_cleanup(curl); + curl_multi_cleanup(multi_handle); + return retval; + } + + // Transfer loop: use curl to actually run the transfer, but periodically + // interrupt things to send back performance updates to the client. + int running_handles = 1; + time_t last_marker = 0; + do { + time_t now = time(NULL); + time_t next_marker = last_marker + m_marker_period; + if (now > next_marker) { + if (SendPerfMarker(req, state)) { + curl_multi_remove_handle(multi_handle, curl); + curl_easy_cleanup(curl); + curl_multi_cleanup(multi_handle); + return -1; + } + last_marker = now; + } + mres = curl_multi_perform(multi_handle, &running_handles); + if (mres == CURLM_CALL_MULTI_PERFORM) { + // curl_multi_perform should be called again immediately. On newer + // versions of curl, this is no longer used. + continue; + } else if (mres != CURLM_OK) { + break; + } else if (running_handles == 0) { + break; + } + + int64_t max_sleep_time = next_marker - time(NULL); + if (max_sleep_time < 0) { + continue; + } + int fd_count; + mres = curl_multi_wait(multi_handle, NULL, 0, max_sleep_time*1000, &fd_count); + if (mres != CURLM_OK) { + break; + } + } while (running_handles); + + if (mres != CURLM_OK) { + m_log.Emsg(log_prefix, "Internal libcurl multi-handle error", + curl_multi_strerror(mres)); + char msg[] = "Internal server error due to libcurl"; + curl_multi_remove_handle(multi_handle, curl); + curl_easy_cleanup(curl); + + curl_multi_cleanup(multi_handle); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + + // Harvest any messages, looking for CURLMSG_DONE. + CURLcode res = static_cast(-1); + CURLMsg *msg; + do { + int msgq = 0; + msg = curl_multi_info_read(multi_handle, &msgq); + if (msg && (msg->msg == CURLMSG_DONE)) { + CURL *easy_handle = msg->easy_handle; + res = msg->data.result; + curl_multi_remove_handle(multi_handle, easy_handle); + curl_easy_cleanup(easy_handle); + } + } while (msg); + + if (res == -1) { // No transfers returned?!? + curl_multi_remove_handle(multi_handle, curl); + curl_easy_cleanup(curl); + curl_multi_cleanup(multi_handle); + char msg[] = "Internal state error in libcurl"; + m_log.Emsg(log_prefix, msg); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + curl_multi_cleanup(multi_handle); + + // Generate the final response back to the client. + std::stringstream ss; + if (res != CURLE_OK) { + m_log.Emsg(log_prefix, "Remote server failed request", curl_easy_strerror(res)); + ss << "failure: " << curl_easy_strerror(res); + } else if (state.GetStatusCode() >= 400) { + ss << "failure: Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg(log_prefix, "Remote server failed request", ss.str().c_str()); + } else { + ss << "success: Created"; + } + + if ((retval = req.ChunkResp(ss.str().c_str(), 0))) { + return retval; + } + return req.ChunkResp(nullptr, 0); + } +#else + int RunCurlBasic(CURL *curl, XrdHttpExtReq &req, XrdHttpTPCState &state, + const char *log_prefix) { + CURLcode res; + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + if (res == CURLE_HTTP_RETURNED_ERROR) { + m_log.Emsg(log_prefix, "Remote server failed request", curl_easy_strerror(res)); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); + } else if (state.GetStatusCode() >= 400) { + std::stringstream ss; + ss << "Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg(log_prefix, "Remote server failed request", ss.str().c_str()); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); + } else if (res) { + m_log.Emsg(log_prefix, "Curl failed", curl_easy_strerror(res)); + char msg[] = "Unknown internal transfer failure"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } else { + char msg[] = "Created"; + return req.SendSimpleResp(201, nullptr, nullptr, msg, 0); + } + } +#endif + int ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) { CURL *curl = curl_easy_init(); if (!curl) { @@ -351,28 +518,16 @@ class XrdHttpTPC : public XrdHttpExtHandler { return resp_result; } - CURLcode res; curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); XrdHttpTPCState state(std::move(fh), curl, true); state.CopyHeaders(req); - res = curl_easy_perform(curl); - if (res == CURLE_HTTP_RETURNED_ERROR) { - m_log.Emsg("ProcessPushReq", "Remote server failed request", curl_easy_strerror(res)); - return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); - } else if (state.GetStatusCode() >= 400) { - std::stringstream ss; - ss << "Remote side failed with status code " << state.GetStatusCode(); - m_log.Emsg("ProcessPushReq", "Remote server failed request", ss.str().c_str()); - return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); - } else if (res) { - m_log.Emsg("ProcessPushReq", "Curl failed", curl_easy_strerror(res)); - char msg[] = "Unknown internal transfer failure"; - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } else { - char msg[] = "Created"; - return req.SendSimpleResp(201, nullptr, nullptr, msg, 0); - } + +#ifdef XRD_CHUNK_RESP + return RunCurlWithUpdates(curl, req, state, "ProcessPushReq"); +#else + return RunCurlBasic(curl, req, state, "ProcessPushReq"); +#endif } int ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) { @@ -409,29 +564,15 @@ class XrdHttpTPC : public XrdHttpExtHandler { fh->close(); return resp_result; } - - CURLcode res; curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - XrdHttpTPCState state(std::move(fh), curl, false); state.CopyHeaders(req); - res = curl_easy_perform(curl); - if (res == CURLE_HTTP_RETURNED_ERROR) { - m_log.Emsg("ProcessPullReq", "Remote server failed request", curl_easy_strerror(res)); - return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); - } else if (state.GetStatusCode() >= 400) { - std::stringstream ss; - ss << "Remote side failed with status code " << state.GetStatusCode(); - m_log.Emsg("ProcessPushReq", "Remote server failed request", ss.str().c_str()); - return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); - } else if (res) { - m_log.Emsg("ProcessPullReq", "Curl failed", curl_easy_strerror(res)); - char msg[] = "Unknown internal transfer failure"; - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } else { - char msg[] = "Created"; - return req.SendSimpleResp(201, nullptr, nullptr, msg, 0); - } + +#ifdef XRD_CHUNK_RESP + return RunCurlWithUpdates(curl, req, state, "ProcessPushReq"); +#else + return RunCurlBasic(curl, req, state, "ProcessPushReq"); +#endif } bool ConfigureFSLib(XrdOucStream &Config, std::string &path1, bool &path1_alt, std::string &path2, bool &path2_alt) { @@ -564,6 +705,7 @@ class XrdHttpTPC : public XrdHttpExtHandler { return true; } + static constexpr int m_marker_period = 20; bool m_desthttps{false}; static std::atomic m_monid; XrdSysError &m_log; From 6d8e54a7b01ba5c4614fd02a51ac4da0b18cc1b7 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Wed, 20 Dec 2017 22:18:20 -0600 Subject: [PATCH 10/29] First working version with gfal-copy. --- src/tpc.cpp | 52 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/src/tpc.cpp b/src/tpc.cpp index e0e66e0875e..033b349c6e5 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -165,7 +165,7 @@ class XrdHttpTPCState { } int Header(const std::string &header) { - printf("Recieved remote header (%d, %d): %s", m_recv_all_headers, m_recv_status_line, header.c_str()); + //printf("Recieved remote header (%d, %d): %s", m_recv_all_headers, m_recv_status_line, header.c_str()); if (m_recv_all_headers) { // This is the second request -- maybe processed a redirect? m_recv_all_headers = false; m_recv_status_line = false; @@ -175,7 +175,7 @@ class XrdHttpTPCState { std::string item; if (!std::getline(ss, item, ' ')) return 0; m_resp_protocol = item; - printf("\n\nResponse protocol: %s\n", m_resp_protocol.c_str()); + //printf("\n\nResponse protocol: %s\n", m_resp_protocol.c_str()); if (!std::getline(ss, item, ' ')) return 0; try { m_status_code = std::stol(item); @@ -201,6 +201,7 @@ class XrdHttpTPCState { return -1; } m_offset += retval; + //printf("Wrote a total of %ld bytes.\n", m_offset); return retval; } @@ -217,13 +218,14 @@ class XrdHttpTPCState { return -1; } m_offset += retval; + //printf("Read a total of %ld bytes.\n", m_offset); return retval; } bool m_push{true}; bool m_recv_status_line{false}; bool m_recv_all_headers{false}; - XrdSfsXferSize m_offset{0}; + off_t m_offset{0}; int m_status_code{-1}; std::unique_ptr m_fh; CURL *m_curl{nullptr}; @@ -250,7 +252,8 @@ class XrdHttpTPC : public XrdHttpExtHandler { if (header != req.headers.end()) { return ProcessPushReq(header->second, req); } - return req.SendSimpleResp(400, NULL, NULL, (char *)"No Source or Destination specified", 0); + m_log.Emsg("ProcessReq", "COPY verb requested but no source or destination specified."); + return req.SendSimpleResp(400, NULL, NULL, "No Source or Destination specified", 0); } /** @@ -332,12 +335,14 @@ class XrdHttpTPC : public XrdHttpExtHandler { #ifdef XRD_CHUNK_RESP int SendPerfMarker(XrdHttpExtReq &req, XrdHttpTPCState &state) { std::stringstream ss; - const std::string crlf = "\r\n"; + const std::string crlf = "\n"; ss << "Perf Marker" << crlf; - ss << " Timestamp: " << time(NULL) << crlf; - ss << " Stripe Index: 0" << crlf; - ss << " Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; - ss << " Total Stripe Count: 1" << crlf; + ss << "Timestamp: " << time(NULL) << crlf; + ss << "Stripe Index: 0" << crlf; + for (int i=0; i<30; i++) + ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; + ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; + ss << "Total Stripe Count: 1" << crlf; ss << "End" << crlf; return req.ChunkResp(ss.str().c_str(), 0); @@ -378,10 +383,11 @@ class XrdHttpTPC : public XrdHttpExtHandler { // interrupt things to send back performance updates to the client. int running_handles = 1; time_t last_marker = 0; + CURLcode res = static_cast(-1); do { time_t now = time(NULL); time_t next_marker = last_marker + m_marker_period; - if (now > next_marker) { + if (now >= next_marker) { if (SendPerfMarker(req, state)) { curl_multi_remove_handle(multi_handle, curl); curl_easy_cleanup(curl); @@ -400,9 +406,23 @@ class XrdHttpTPC : public XrdHttpExtHandler { } else if (running_handles == 0) { break; } + //printf("There are %d running handles\n", running_handles); + + // Harvest any messages, looking for CURLMSG_DONE. + CURLMsg *msg; + do { + int msgq = 0; + msg = curl_multi_info_read(multi_handle, &msgq); + if (msg && (msg->msg == CURLMSG_DONE)) { + CURL *easy_handle = msg->easy_handle; + res = msg->data.result; + curl_multi_remove_handle(multi_handle, easy_handle); + curl_easy_cleanup(easy_handle); + } + } while (msg); int64_t max_sleep_time = next_marker - time(NULL); - if (max_sleep_time < 0) { + if (max_sleep_time <= 0) { continue; } int fd_count; @@ -424,7 +444,6 @@ class XrdHttpTPC : public XrdHttpExtHandler { } // Harvest any messages, looking for CURLMSG_DONE. - CURLcode res = static_cast(-1); CURLMsg *msg; do { int msgq = 0; @@ -490,6 +509,7 @@ class XrdHttpTPC : public XrdHttpExtHandler { #endif int ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) { + m_log.Emsg("ProcessPushReq", "Starting a push request for resource", resource.c_str()); CURL *curl = curl_easy_init(); if (!curl) { char msg[] = "Failed to initialize internal transfer resources"; @@ -544,10 +564,10 @@ class XrdHttpTPC : public XrdHttpExtHandler { } std::string authz = GetAuthz(req); XrdSfsFileOpenMode mode = SFS_O_CREAT; - auto overwrite_header = req.headers.find("Overwrite"); + /*auto overwrite_header = req.headers.find("Overwrite"); if ((overwrite_header == req.headers.end()) || (overwrite_header->second == "T")) { mode = SFS_O_TRUNC|SFS_O_POSC; - } + }*/ int open_result = OpenWaitStall(*fh, req.resource, mode|SFS_O_WRONLY, 0644, req.GetSecEntity(), authz); if (SFS_REDIRECT == open_result) { @@ -705,7 +725,7 @@ class XrdHttpTPC : public XrdHttpExtHandler { return true; } - static constexpr int m_marker_period = 20; + static constexpr int m_marker_period = 5; bool m_desthttps{false}; static std::atomic m_monid; XrdSysError &m_log; @@ -734,7 +754,7 @@ XrdHttpExtHandler *XrdHttpGetExtHandler(XrdSysError *log, const char * config, c retval = new XrdHttpTPC(log, config, myEnv); } catch (std::runtime_error &re) { log->Emsg("Initialize", "Encountered a runtime failure when loading ", re.what()); - printf("Provided env vars: %p, XrdInet*: %p\n", myEnv, myEnv->GetPtr("XrdInet*")); + //printf("Provided env vars: %p, XrdInet*: %p\n", myEnv, myEnv->GetPtr("XrdInet*")); } return retval; } From cc839695baad18c633d78783542a70f7563ec80c Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 28 Dec 2017 21:05:18 -0600 Subject: [PATCH 11/29] Add support for dCache-style TransferHeader. --- src/tpc.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/tpc.cpp b/src/tpc.cpp index 033b349c6e5..544e04ed9c5 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -143,6 +143,12 @@ class XrdHttpTPCState { if (hdr.first == "Copy-Header") { list = curl_slist_append(list, hdr.second.c_str()); } + // Note: len("TransferHeader") == 14 + if (!hdr.first.compare(0, 14, "TransferHeader")) { + std::stringstream ss; + ss << hdr.first.substr(14) << ": " << hdr.second; + list = curl_slist_append(list, ss.str().c_str()); + } } if (list != nullptr) { curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, list); From 9563b8b7d196a19d94420cce04474ddd468756c3 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 28 Dec 2017 21:05:34 -0600 Subject: [PATCH 12/29] Update RPM spec for next release. --- rpm/xrootd-tpc.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rpm/xrootd-tpc.spec b/rpm/xrootd-tpc.spec index d2153509974..a07ff65a033 100644 --- a/rpm/xrootd-tpc.spec +++ b/rpm/xrootd-tpc.spec @@ -1,6 +1,6 @@ Name: xrootd-tpc -Version: 0.3 +Version: 0.3.1 Release: 1%{?dist} Summary: HTTP Third Party Copy plugin for XRootD @@ -43,6 +43,9 @@ rm -rf $RPM_BUILD_ROOT %{_libdir}/libXrdHttpTPC-4.so %changelog +* Fri Dec 29 2017 Brian Bockelman - 0.3.1-1 +- Add support for dCache-style TransferHeader. + * Mon Nov 13 2017 Brian Bockelman - 0.3-1 - Add support for redirections in COPY requests. - Add RPM packaging. From 9e1b31e16349a06f1ee9626b770254b0d35177bd Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 28 Dec 2017 22:36:14 -0600 Subject: [PATCH 13/29] Add support for overriding the CA directory. --- src/tpc.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/tpc.cpp b/src/tpc.cpp index 544e04ed9c5..4936efff4a5 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -543,7 +543,9 @@ class XrdHttpTPC : public XrdHttpExtHandler { fh->close(); return resp_result; } - + if (!m_cadir.empty()) { + curl_easy_setopt(curl, CURLOPT_CAPATH, m_cadir.c_str()); + } curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); XrdHttpTPCState state(std::move(fh), curl, true); @@ -590,6 +592,9 @@ class XrdHttpTPC : public XrdHttpExtHandler { fh->close(); return resp_result; } + if (!m_cadir.empty()) { + curl_easy_setopt(curl, CURLOPT_CAPATH, m_cadir.c_str()); + } curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); XrdHttpTPCState state(std::move(fh), curl, false); state.CopyHeaders(req); @@ -684,6 +689,13 @@ class XrdHttpTPC : public XrdHttpExtHandler { m_log.Emsg("Config", "https.dests value is invalid", val); return false; } + } else if (!strcmp("http.cadir", val)) { + if (!(val = Config.GetWord())) { + Config.Close(); + m_log.Emsg("Config", "http.cadir value not specified"); + return false; + } + m_cadir = val; } } Config.Close(); @@ -733,6 +745,7 @@ class XrdHttpTPC : public XrdHttpExtHandler { static constexpr int m_marker_period = 5; bool m_desthttps{false}; + std::string m_cadir; static std::atomic m_monid; XrdSysError &m_log; std::unique_ptr m_sfs; From 19336eed271500e5a5478188e7ed838f9dfd46aa Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 28 Dec 2017 22:38:21 -0600 Subject: [PATCH 14/29] Bump spec for 0.3.2 version. --- rpm/xrootd-tpc.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rpm/xrootd-tpc.spec b/rpm/xrootd-tpc.spec index a07ff65a033..662ed5353b2 100644 --- a/rpm/xrootd-tpc.spec +++ b/rpm/xrootd-tpc.spec @@ -1,6 +1,6 @@ Name: xrootd-tpc -Version: 0.3.1 +Version: 0.3.2 Release: 1%{?dist} Summary: HTTP Third Party Copy plugin for XRootD @@ -43,6 +43,9 @@ rm -rf $RPM_BUILD_ROOT %{_libdir}/libXrdHttpTPC-4.so %changelog +* Fri Dec 29 2017 Brian Bockelman - 0.3.2-1 +- Allow CA directory to be overridden in Xrootd. + * Fri Dec 29 2017 Brian Bockelman - 0.3.1-1 - Add support for dCache-style TransferHeader. From 1a4e67b391db60f275a45cd83b3fb9d6c50a0ea1 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 1 Jan 2018 21:59:17 -0600 Subject: [PATCH 15/29] Remove overly-chatty version of transfer markers. libdavix only parser transfer marker information once every 1024 characters (which caused us to create overly-chatty markers). This removes that workaround as a fix is available from upstream libdavix. --- rpm/xrootd-tpc.spec | 5 ++++- src/tpc.cpp | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rpm/xrootd-tpc.spec b/rpm/xrootd-tpc.spec index 662ed5353b2..459ff8cc961 100644 --- a/rpm/xrootd-tpc.spec +++ b/rpm/xrootd-tpc.spec @@ -1,6 +1,6 @@ Name: xrootd-tpc -Version: 0.3.2 +Version: 0.3.3 Release: 1%{?dist} Summary: HTTP Third Party Copy plugin for XRootD @@ -43,6 +43,9 @@ rm -rf $RPM_BUILD_ROOT %{_libdir}/libXrdHttpTPC-4.so %changelog +* Tue Jan 02 2018 Brian Bockelman - 0.3.3-1 +- Remove workaround from bad version of libdavix. + * Fri Dec 29 2017 Brian Bockelman - 0.3.2-1 - Allow CA directory to be overridden in Xrootd. diff --git a/src/tpc.cpp b/src/tpc.cpp index 4936efff4a5..f75310a73e1 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -345,7 +345,6 @@ class XrdHttpTPC : public XrdHttpExtHandler { ss << "Perf Marker" << crlf; ss << "Timestamp: " << time(NULL) << crlf; ss << "Stripe Index: 0" << crlf; - for (int i=0; i<30; i++) ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; ss << "Total Stripe Count: 1" << crlf; From 200170c75f6da65c9a76750a0f62eccbd38d7110 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Wed, 3 Jan 2018 22:59:22 -0600 Subject: [PATCH 16/29] Allow TPC code to determine remote size when pulling. Knowing the size of the remote resource is the first step in doing a multi-stream HTTP GET transfer. --- src/tpc.cpp | 85 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 7 deletions(-) diff --git a/src/tpc.cpp b/src/tpc.cpp index f75310a73e1..305032dd5a4 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -156,13 +157,17 @@ class XrdHttpTPCState { } } - /** - * Perform the curl-based transfer, responding periodically with transfer - * markers. - */ + off_t GetContentLength() const {return m_content_length;} int GetStatusCode() const {return m_status_code;} + void ResetAfterSize() { + m_status_code = -1; + m_content_length = -1; + m_recv_all_headers = false; + m_recv_status_line = false; + } + private: static size_t HeaderCB(char *buffer, size_t size, size_t nitems, void *userdata) { XrdHttpTPCState *obj = static_cast(userdata); @@ -189,8 +194,33 @@ class XrdHttpTPCState { return 0; } m_recv_status_line = true; + } else if (header.size() == 0 || header == "\n") { + m_recv_all_headers = true; + } + else if (header != "\r\n") { + // Parse the header + std::size_t found = header.find(":"); + if (found != std::string::npos) { + std::string header_name = header.substr(0, found); + std::transform(header_name.begin(), header_name.end(), header_name.begin(), ::tolower); + std::string header_value = header.substr(found+1); + if (header_name == "content-length") + { + try { + m_content_length = std::stoll(header_value); + } catch (...) { + // Header unparseable -- not a great sign, fail request. + //printf("Content-length header unparseable\n"); + return 0; + } + } + } else { + // Non-empty header that isn't the status line, but no ':' present -- + // malformed request? + //printf("Malformed header: %s\n", header.c_str()); + return 0; + } } - if (header.size() == 0 || header == "\n") {m_recv_all_headers = true;} return header.size(); } @@ -233,6 +263,7 @@ class XrdHttpTPCState { bool m_recv_all_headers{false}; off_t m_offset{0}; int m_status_code{-1}; + off_t m_content_length{-1}; std::unique_ptr m_fh; CURL *m_curl{nullptr}; struct curl_slist *m_headers{nullptr}; @@ -339,6 +370,36 @@ class XrdHttpTPC : public XrdHttpExtHandler { } #ifdef XRD_CHUNK_RESP + /** + * Determine size at remote end. + */ + int DetermineXferSize(CURL *curl, XrdHttpExtReq &req, XrdHttpTPCState &state, + bool &success) { + success = false; + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + CURLcode res; + res = curl_easy_perform(curl); + if (res == CURLE_HTTP_RETURNED_ERROR) { + m_log.Emsg("DetermineXferSize", "Remote server failed request", curl_easy_strerror(res)); + curl_easy_cleanup(curl); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); + } else if (state.GetStatusCode() >= 400) { + std::stringstream ss; + ss << "Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg("DetermineXferSize", "Remote server failed request", ss.str().c_str()); + curl_easy_cleanup(curl); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); + } else if (res) { + m_log.Emsg("DetermineXferSize", "Curl failed", curl_easy_strerror(res)); + char msg[] = "Unknown internal transfer failure"; + curl_easy_cleanup(curl); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + curl_easy_setopt(curl, CURLOPT_NOBODY, 0); + success = true; + return 0; + } + int SendPerfMarker(XrdHttpExtReq &req, XrdHttpTPCState &state) { std::stringstream ss; const std::string crlf = "\n"; @@ -599,9 +660,19 @@ class XrdHttpTPC : public XrdHttpExtHandler { state.CopyHeaders(req); #ifdef XRD_CHUNK_RESP - return RunCurlWithUpdates(curl, req, state, "ProcessPushReq"); + int result; + bool success; + if ((result = DetermineXferSize(curl, req, state, success)) || !success) { + return result; + } + std::stringstream ss; + ss << "Successfully determined remote size for pull request: " << state.GetContentLength(); + m_log.Emsg("ProcessPullReq", ss.str().c_str()); + state.ResetAfterSize(); + + return RunCurlWithUpdates(curl, req, state, "ProcessPullReq"); #else - return RunCurlBasic(curl, req, state, "ProcessPushReq"); + return RunCurlBasic(curl, req, state, "ProcessPullReq"); #endif } From 6992a6d9e028689d8d68b0d4d0e45be739668be8 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 4 Jan 2018 09:34:44 -0600 Subject: [PATCH 17/29] Refactor implementation to smaller files. Split tpc.cpp to several helper files. This is done in preparation of the addition of multi-streaming support, which will make the implementation larger and more complex. --- CMakeLists.txt | 2 +- src/configure.cpp | 182 ++++++++ src/state.cpp | 161 +++++++ src/state.hh | 62 +++ src/tpc.cpp | 1030 +++++++++++++++------------------------------ src/tpc.hh | 67 +++ 6 files changed, 811 insertions(+), 693 deletions(-) create mode 100644 src/configure.cpp create mode 100644 src/state.cpp create mode 100644 src/state.hh create mode 100644 src/tpc.hh diff --git a/CMakeLists.txt b/CMakeLists.txt index d3a8ed67434..68ccbdfbb5c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ pkg_check_modules(CURL REQUIRED libcurl) include_directories(${XROOTD_INCLUDES} ${XROOTD_PRIVATE_INCLUDES} ${CURL_INCLUDE_DIRS}) -add_library(XrdHttpTPC SHARED src/tpc.cpp) +add_library(XrdHttpTPC SHARED src/tpc.cpp src/state.cpp src/configure.cpp) if ( XRD_CHUNK_RESP ) set_target_properties(XrdHttpTPC PROPERTIES COMPILE_DEFINITIONS "XRD_CHUNK_RESP" ) endif () diff --git a/src/configure.cpp b/src/configure.cpp new file mode 100644 index 00000000000..d1976118896 --- /dev/null +++ b/src/configure.cpp @@ -0,0 +1,182 @@ + +#include "tpc.hh" + +#include +#include + +#include "XrdOuc/XrdOucStream.hh" +#include "XrdOuc/XrdOucPinPath.hh" +#include "XrdSfs/XrdSfsInterface.hh" + +extern XrdSfsFileSystem *XrdSfsGetDefaultFileSystem(XrdSfsFileSystem *native_fs, + XrdSysLogger *lp, + const char *configfn, + XrdOucEnv *EnvInfo); + + +using namespace TPC; + + +static XrdSfsFileSystem *load_sfs(void *handle, bool alt, XrdSysError &log, const std::string &libpath, const char *configfn, XrdOucEnv &myEnv, XrdSfsFileSystem *prior_sfs) { + XrdSfsFileSystem *sfs = nullptr; + if (alt) { + auto ep = (XrdSfsFileSystem *(*)(XrdSfsFileSystem *, XrdSysLogger *, const char *, XrdOucEnv *)) + (dlsym(handle, "XrdSfsGetFileSystem2")); + if (ep == nullptr) { + log.Emsg("Config", "Failed to load XrdSfsGetFileSystem2 from library ", libpath.c_str(), dlerror()); + return nullptr; + } + sfs = ep(prior_sfs, log.logger(), configfn, &myEnv); + } else { + auto ep = (XrdSfsFileSystem *(*)(XrdSfsFileSystem *, XrdSysLogger *, const char *)) + (dlsym(nullptr, "XrdSfsGetFileSystem")); + if (ep == nullptr) { + log.Emsg("Config", "Failed to load XrdSfsGetFileSystem from library ", libpath.c_str(), dlerror()); + return nullptr; + } + sfs = ep(prior_sfs, log.logger(), configfn); + } + if (!sfs) { + log.Emsg("Config", "Failed to initialize filesystem library for TPC handler from ", libpath.c_str()); + return nullptr; + } + return sfs; +} + + +bool TPCHandler::ConfigureFSLib(XrdOucStream &Config, std::string &path1, bool &path1_alt, std::string &path2, bool &path2_alt) { + char *val; + if (!(val = Config.GetWord())) { + m_log.Emsg("Config", "fslib not specified"); + return false; + } + if (!strcmp("throttle", val)) { + path2 = "libXrdThrottle.so"; + if (!(val = Config.GetWord())) { + m_log.Emsg("Config", "fslib throttle target library not specified"); + return false; + } + } + else if (!strcmp("-2", val)) { + path2_alt = true; + if (!(val = Config.GetWord())) { + m_log.Emsg("Config", "fslib library not specified"); + return false; + } + path2 = val; + } + else { + path2 = val; + } + if (!(val = Config.GetWord()) || !strcmp("default", val)) { + if (path2 == "libXrdThrottle.so") { + path1 = "default"; + } else if (!path2.empty()) { + path1 = path2; + path2 = ""; + path1_alt = path2_alt; + } + } else if (!strcmp("-2", val)) { + path1_alt = true; + if (!(val = Config.GetWord())) { + m_log.Emsg("Config", "fslib base library not specified"); + return false; + } + path1 = val; + } else { + path2 = val; + } + return true; +} + +bool TPCHandler::Configure(const char *configfn, XrdOucEnv *myEnv) +{ + XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), myEnv, "=====> "); + + std::string authLib; + std::string authLibParms; + int cfgFD = open(configfn, O_RDONLY, 0); + if (cfgFD < 0) { + m_log.Emsg("Config", errno, "open config file", configfn); + return false; + } + Config.Attach(cfgFD); + const char *val; + std::string path2, path1 = "default"; + bool path1_alt = false, path2_alt = false; + while ((val = Config.GetMyFirstWord())) { + if (!strcmp("xrootd.fslib", val)) { + if (!ConfigureFSLib(Config, path1, path1_alt, path2, path2_alt)) { + Config.Close(); + m_log.Emsg("Config", "Failed to parse the xrootd.fslib directive"); + return false; + } + m_log.Emsg("Config", "xrootd.fslib line successfully processed by TPC handler/"); + } else if (!strcmp("http.desthttps", val)) { + if (!(val = Config.GetWord())) { + Config.Close(); + m_log.Emsg("Config", "http.desthttps value not specified"); + return false; + } + if (!strcmp("1", val) || !strcasecmp("yes", val) || !strcasecmp("true", val)) { + m_desthttps = true; + } else if (!strcmp("1", val) || !strcasecmp("yes", val) || !strcasecmp("true", val)) { + m_desthttps = false; + } else { + Config.Close(); + m_log.Emsg("Config", "https.dests value is invalid", val); + return false; + } + } else if (!strcmp("http.cadir", val)) { + if (!(val = Config.GetWord())) { + Config.Close(); + m_log.Emsg("Config", "http.cadir value not specified"); + return false; + } + m_cadir = val; + } + } + Config.Close(); + + XrdSfsFileSystem *base_sfs = nullptr; + if (path1 == "default") { + m_log.Emsg("Config", "Loading the default filesystem"); + base_sfs = XrdSfsGetDefaultFileSystem(nullptr, m_log.logger(), configfn, myEnv); + m_log.Emsg("Config", "Finished loading the default filesystem"); + } else { + char resolvePath[2048]; + bool usedAltPath{true}; + if (!XrdOucPinPath(path1.c_str(), usedAltPath, resolvePath, 2048)) { + m_log.Emsg("Config", "Failed to locate appropriately versioned base filesystem library for ", path1.c_str()); + return false; + } + m_handle_base = dlopen(resolvePath, RTLD_LOCAL|RTLD_NOW); + if (m_handle_base == nullptr) { + m_log.Emsg("Config", "Failed to base plugin ", resolvePath, dlerror()); + return false; + } + base_sfs = load_sfs(m_handle_base, path1_alt, m_log, path1, configfn, *myEnv, nullptr); + } + if (!base_sfs) { + m_log.Emsg("Config", "Failed to initialize filesystem library for TPC handler from ", path1.c_str()); + return false; + } + XrdSfsFileSystem *chained_sfs = nullptr; + if (!path2.empty()) { + char resolvePath[2048]; + bool usedAltPath{true}; + if (!XrdOucPinPath(path2.c_str(), usedAltPath, resolvePath, 2048)) { + m_log.Emsg("Config", "Failed to locate appropriately versioned chained filesystem library for ", path2.c_str()); + return false; + } + m_handle_chained = dlopen(resolvePath, RTLD_LOCAL|RTLD_NOW); + if (m_handle_chained == nullptr) { + m_log.Emsg("Config", "Failed to chained plugin ", resolvePath, dlerror()); + return false; + } + chained_sfs = load_sfs(m_handle_chained, path2_alt, m_log, path2, configfn, *myEnv, base_sfs); + } + m_sfs.reset(chained_sfs ? chained_sfs : base_sfs); + m_log.Emsg("Config", "Successfully configured the filesystem object for TPC handler"); + return true; +} diff --git a/src/state.cpp b/src/state.cpp new file mode 100644 index 00000000000..d8435f3a0db --- /dev/null +++ b/src/state.cpp @@ -0,0 +1,161 @@ + +#include +#include + +#include "XrdHttp/XrdHttpExtHandler.hh" +#include "XrdSfs/XrdSfsInterface.hh" + +#include + +#include "XrdTpcVersion.hh" +#include "state.hh" + +using namespace TPC; + +State::~State() { + if (m_headers) { + curl_slist_free_all(m_headers); + m_headers = nullptr; + curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, m_headers); + } + m_fh->close(); +} + +bool State::InstallHandlers(CURL *curl) { + curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/" XRDTPC_VERSION); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &State::HeaderCB); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, this); + if (m_push) { + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, &State::ReadCB); + curl_easy_setopt(curl, CURLOPT_READDATA, this); + struct stat buf; + if (SFS_OK == m_fh->stat(&buf)) { + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, buf.st_size); + } + } else { + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &State::WriteCB); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); + } + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + return true; +} + +/** + * Handle the 'Copy-Headers' feature + */ +void State::CopyHeaders(XrdHttpExtReq &req) { + struct curl_slist *list = NULL; + for (auto &hdr : req.headers) { + if (hdr.first == "Copy-Header") { + list = curl_slist_append(list, hdr.second.c_str()); + } + // Note: len("TransferHeader") == 14 + if (!hdr.first.compare(0, 14, "TransferHeader")) { + std::stringstream ss; + ss << hdr.first.substr(14) << ": " << hdr.second; + list = curl_slist_append(list, ss.str().c_str()); + } + } + if (list != nullptr) { + curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, list); + m_headers = list; + } +} + +void State::ResetAfterSize() { + m_status_code = -1; + m_content_length = -1; + m_recv_all_headers = false; + m_recv_status_line = false; +} + +size_t State::HeaderCB(char *buffer, size_t size, size_t nitems, void *userdata) +{ + State *obj = static_cast(userdata); + std::string header(buffer, size*nitems); + return obj->Header(header); +} + +int State::Header(const std::string &header) { + //printf("Recieved remote header (%d, %d): %s", m_recv_all_headers, m_recv_status_line, header.c_str()); + if (m_recv_all_headers) { // This is the second request -- maybe processed a redirect? + m_recv_all_headers = false; + m_recv_status_line = false; + } + if (!m_recv_status_line) { + std::stringstream ss(header); + std::string item; + if (!std::getline(ss, item, ' ')) return 0; + m_resp_protocol = item; + //printf("\n\nResponse protocol: %s\n", m_resp_protocol.c_str()); + if (!std::getline(ss, item, ' ')) return 0; + try { + m_status_code = std::stol(item); + } catch (...) { + return 0; + } + m_recv_status_line = true; + } else if (header.size() == 0 || header == "\n") { + m_recv_all_headers = true; + } + else if (header != "\r\n") { + // Parse the header + std::size_t found = header.find(":"); + if (found != std::string::npos) { + std::string header_name = header.substr(0, found); + std::transform(header_name.begin(), header_name.end(), header_name.begin(), ::tolower); + std::string header_value = header.substr(found+1); + if (header_name == "content-length") + { + try { + m_content_length = std::stoll(header_value); + } catch (...) { + // Header unparseable -- not a great sign, fail request. + //printf("Content-length header unparseable\n"); + return 0; + } + } + } else { + // Non-empty header that isn't the status line, but no ':' present -- + // malformed request? + //printf("Malformed header: %s\n", header.c_str()); + return 0; + } + } + return header.size(); +} + +size_t State::WriteCB(void *buffer, size_t size, size_t nitems, void *userdata) { + State *obj = static_cast(userdata); + if (obj->GetStatusCode() < 0) {return 0;} // malformed request - got body before headers. + if (obj->GetStatusCode() >= 400) {return 0;} // Status indicates failure. + return obj->Write(static_cast(buffer), size*nitems); +} + +int State::Write(char *buffer, size_t size) { + int retval = m_fh->write(m_offset, buffer, size); + if (retval == SFS_ERROR) { + return -1; + } + m_offset += retval; + //printf("Wrote a total of %ld bytes.\n", m_offset); + return retval; +} + +size_t State::ReadCB(void *buffer, size_t size, size_t nitems, void *userdata) { + State *obj = static_cast(userdata); + if (obj->GetStatusCode() < 0) {return 0;} // malformed request - got body before headers. + if (obj->GetStatusCode() >= 400) {return 0;} // Status indicates failure. + return obj->Read(static_cast(buffer), size*nitems); +} + +int State::Read(char *buffer, size_t size) { + int retval = m_fh->read(m_offset, buffer, size); + if (retval == SFS_ERROR) { + return -1; + } + m_offset += retval; + //printf("Read a total of %ld bytes.\n", m_offset); + return retval; +} diff --git a/src/state.hh b/src/state.hh new file mode 100644 index 00000000000..3ba5b7a68cb --- /dev/null +++ b/src/state.hh @@ -0,0 +1,62 @@ +/** + * state.hh: + * + * Helper class for managing the state of a single TPC request. + */ + +#include + +// Forward dec'ls +class XrdSfsFile; +class XrdHttpExtReq; +typedef void CURL; + +namespace TPC { + +class State { +public: + State (std::unique_ptr fh, CURL *curl, bool push) : + m_push(push), + m_fh(std::move(fh)), + m_curl(curl) + { + InstallHandlers(curl); + } + + ~State(); + + void CopyHeaders(XrdHttpExtReq &req); + + off_t BytesTransferred() const {return m_offset;} + + off_t GetContentLength() const {return m_content_length;} + + int GetStatusCode() const {return m_status_code;} + + void ResetAfterSize(); + +private: + bool InstallHandlers(CURL *curl); + + // libcurl callback functions, along with the corresponding class methods. + static size_t HeaderCB(char *buffer, size_t size, size_t nitems, + void *userdata); + int Header(const std::string &header); + static size_t WriteCB(void *buffer, size_t size, size_t nitems, void *userdata); + int Write(char *buffer, size_t size); + static size_t ReadCB(void *buffer, size_t size, size_t nitems, void *userdata); + int Read(char *buffer, size_t size); + + bool m_push{true}; // whether we are transferring in "push-mode" + bool m_recv_status_line{false}; // whether we have received a status line in the response from the remote host. + bool m_recv_all_headers{false}; // true if we have seen the end of headers. + off_t m_offset{0}; // number of bytes we have received. + int m_status_code{-1}; // status code from HTTP response. + off_t m_content_length{-1}; // value of Content-Length header, if we received one. + std::unique_ptr m_fh; // file-handle corresponding to this transfer. + CURL *m_curl{nullptr}; // libcurl handle + struct curl_slist *m_headers{nullptr}; // any headers we set as part of the libcurl request. + std::string m_resp_protocol; // Response protocol in the HTTP status line. +}; + +}; diff --git a/src/tpc.cpp b/src/tpc.cpp index 305032dd5a4..30d3f4b4511 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -1,8 +1,6 @@ #include "XrdHttp/XrdHttpExtHandler.hh" #include "XrdOuc/XrdOucEnv.hh" -#include "XrdOuc/XrdOucPinPath.hh" -#include "XrdOuc/XrdOucStream.hh" #include "XrdSec/XrdSecEntity.hh" #include "XrdSfs/XrdSfsInterface.hh" #include "XrdVersion.hh" @@ -18,12 +16,14 @@ #include #include "XrdTpcVersion.hh" +#include "state.hh" +#include "tpc.hh" + +using namespace TPC; + +std::atomic TPCHandler::m_monid{0}; XrdVERSIONINFO(XrdHttpGetExtHandler, HttpTPC); -extern XrdSfsFileSystem *XrdSfsGetDefaultFileSystem(XrdSfsFileSystem *native_fs, - XrdSysLogger *lp, - const char *configfn, - XrdOucEnv *EnvInfo); static char *quote(const char *str) { @@ -67,412 +67,200 @@ static char *quote(const char *str) { } -static XrdSfsFileSystem *load_sfs(void *handle, bool alt, XrdSysError &log, const std::string &libpath, const char *configfn, XrdOucEnv &myEnv, XrdSfsFileSystem *prior_sfs) { - XrdSfsFileSystem *sfs = nullptr; - if (alt) { - auto ep = (XrdSfsFileSystem *(*)(XrdSfsFileSystem *, XrdSysLogger *, const char *, XrdOucEnv *)) - (dlsym(handle, "XrdSfsGetFileSystem2")); - if (ep == nullptr) { - log.Emsg("Config", "Failed to load XrdSfsGetFileSystem2 from library ", libpath.c_str(), dlerror()); - return nullptr; - } - sfs = ep(prior_sfs, log.logger(), configfn, &myEnv); - } else { - auto ep = (XrdSfsFileSystem *(*)(XrdSfsFileSystem *, XrdSysLogger *, const char *)) - (dlsym(nullptr, "XrdSfsGetFileSystem")); - if (ep == nullptr) { - log.Emsg("Config", "Failed to load XrdSfsGetFileSystem from library ", libpath.c_str(), dlerror()); - return nullptr; - } - sfs = ep(prior_sfs, log.logger(), configfn); - } - if (!sfs) { - log.Emsg("Config", "Failed to initialize filesystem library for XrdHttpTPC from ", libpath.c_str()); - return nullptr; - } - return sfs; +bool TPCHandler::MatchesPath(const char *verb, const char *path) { + return !strcmp(verb, "COPY") || !strcmp(verb, "OPTIONS"); } -class XrdHttpTPCState { -public: - XrdHttpTPCState (std::unique_ptr fh, CURL *curl, bool push) : - m_push(push), - m_fh(std::move(fh)), - m_curl(curl) - { - InstallHandlers(curl); - } - - ~XrdHttpTPCState() { - if (m_headers) { - curl_slist_free_all(m_headers); - m_headers = nullptr; - curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, m_headers); - } - m_fh->close(); - } - - off_t BytesTransferred() const {return m_offset;} - - bool InstallHandlers(CURL *curl) { - curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/" XRDTPC_VERSION); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &XrdHttpTPCState::HeaderCB); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, this); - if (m_push) { - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(curl, CURLOPT_READFUNCTION, &XrdHttpTPCState::ReadCB); - curl_easy_setopt(curl, CURLOPT_READDATA, this); - struct stat buf; - if (SFS_OK == m_fh->stat(&buf)) { - curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, buf.st_size); - } - } else { - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &XrdHttpTPCState::WriteCB); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); - } - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - //curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); - return true; - } - - /** - * Handle the 'Copy-Headers' feature - */ - void CopyHeaders(XrdHttpExtReq &req) { - struct curl_slist *list = NULL; - for (auto &hdr : req.headers) { - if (hdr.first == "Copy-Header") { - list = curl_slist_append(list, hdr.second.c_str()); - } - // Note: len("TransferHeader") == 14 - if (!hdr.first.compare(0, 14, "TransferHeader")) { - std::stringstream ss; - ss << hdr.first.substr(14) << ": " << hdr.second; - list = curl_slist_append(list, ss.str().c_str()); - } - } - if (list != nullptr) { - curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, list); - m_headers = list; - } - } - - off_t GetContentLength() const {return m_content_length;} - - int GetStatusCode() const {return m_status_code;} - - void ResetAfterSize() { - m_status_code = -1; - m_content_length = -1; - m_recv_all_headers = false; - m_recv_status_line = false; +int TPCHandler::ProcessReq(XrdHttpExtReq &req) { + if (req.verb == "OPTIONS") { + return ProcessOptionsReq(req); } - -private: - static size_t HeaderCB(char *buffer, size_t size, size_t nitems, void *userdata) { - XrdHttpTPCState *obj = static_cast(userdata); - std::string header(buffer, size*nitems); - return obj->Header(header); + auto header = req.headers.find("Source"); + if (header != req.headers.end()) { + return ProcessPullReq(header->second, req); } - - int Header(const std::string &header) { - //printf("Recieved remote header (%d, %d): %s", m_recv_all_headers, m_recv_status_line, header.c_str()); - if (m_recv_all_headers) { // This is the second request -- maybe processed a redirect? - m_recv_all_headers = false; - m_recv_status_line = false; - } - if (!m_recv_status_line) { - std::stringstream ss(header); - std::string item; - if (!std::getline(ss, item, ' ')) return 0; - m_resp_protocol = item; - //printf("\n\nResponse protocol: %s\n", m_resp_protocol.c_str()); - if (!std::getline(ss, item, ' ')) return 0; - try { - m_status_code = std::stol(item); - } catch (...) { - return 0; - } - m_recv_status_line = true; - } else if (header.size() == 0 || header == "\n") { - m_recv_all_headers = true; - } - else if (header != "\r\n") { - // Parse the header - std::size_t found = header.find(":"); - if (found != std::string::npos) { - std::string header_name = header.substr(0, found); - std::transform(header_name.begin(), header_name.end(), header_name.begin(), ::tolower); - std::string header_value = header.substr(found+1); - if (header_name == "content-length") - { - try { - m_content_length = std::stoll(header_value); - } catch (...) { - // Header unparseable -- not a great sign, fail request. - //printf("Content-length header unparseable\n"); - return 0; - } - } - } else { - // Non-empty header that isn't the status line, but no ':' present -- - // malformed request? - //printf("Malformed header: %s\n", header.c_str()); - return 0; - } - } - return header.size(); + header = req.headers.find("Destination"); + if (header != req.headers.end()) { + return ProcessPushReq(header->second, req); } + m_log.Emsg("ProcessReq", "COPY verb requested but no source or destination specified."); + return req.SendSimpleResp(400, NULL, NULL, "No Source or Destination specified", 0); +} - static size_t WriteCB(void *buffer, size_t size, size_t nitems, void *userdata) { - XrdHttpTPCState *obj = static_cast(userdata); - if (obj->GetStatusCode() < 0) {return 0;} // malformed request - got body before headers. - if (obj->GetStatusCode() >= 400) {return 0;} // Status indicates failure. - return obj->Write(static_cast(buffer), size*nitems); +TPCHandler::~TPCHandler() { + m_sfs = nullptr; // NOTE: must delete the SFS here as we may unload the destructor from memory below! + if (m_handle_base) { + dlclose(m_handle_base); + m_handle_base = nullptr; } - - int Write(char *buffer, size_t size) { - int retval = m_fh->write(m_offset, buffer, size); - if (retval == SFS_ERROR) { - return -1; - } - m_offset += retval; - //printf("Wrote a total of %ld bytes.\n", m_offset); - return retval; - } - - static size_t ReadCB(void *buffer, size_t size, size_t nitems, void *userdata) { - XrdHttpTPCState *obj = static_cast(userdata); - if (obj->GetStatusCode() < 0) {return 0;} // malformed request - got body before headers. - if (obj->GetStatusCode() >= 400) {return 0;} // Status indicates failure. - return obj->Read(static_cast(buffer), size*nitems); + if (m_handle_chained) { + dlclose(m_handle_chained); + m_handle_chained = nullptr; } +} - int Read(char *buffer, size_t size) { - int retval = m_fh->read(m_offset, buffer, size); - if (retval == SFS_ERROR) { - return -1; - } - m_offset += retval; - //printf("Read a total of %ld bytes.\n", m_offset); - return retval; +TPCHandler::TPCHandler(XrdSysError *log, const char *config, XrdOucEnv *myEnv) : + m_log(*log) +{ + if (!Configure(config, myEnv)) { + throw std::runtime_error("Failed to configure the HTTP third-party-copy handler."); } +} - bool m_push{true}; - bool m_recv_status_line{false}; - bool m_recv_all_headers{false}; - off_t m_offset{0}; - int m_status_code{-1}; - off_t m_content_length{-1}; - std::unique_ptr m_fh; - CURL *m_curl{nullptr}; - struct curl_slist *m_headers{nullptr}; - std::string m_resp_protocol; -}; - - -class XrdHttpTPC : public XrdHttpExtHandler { -public: - virtual bool MatchesPath(const char *verb, const char *path) { - return !strcmp(verb, "COPY") || !strcmp(verb, "OPTIONS"); - } +/** + * Handle the OPTIONS verb as we have added a new one... + */ +int TPCHandler::ProcessOptionsReq(XrdHttpExtReq &req) { + return req.SendSimpleResp(200, NULL, (char *) "DAV: 1\r\nDAV: \r\nAllow: HEAD,GET,PUT,PROPFIND,DELETE,OPTIONS,COPY", NULL, 0); +} - virtual int ProcessReq(XrdHttpExtReq &req) { - if (req.verb == "OPTIONS") { - return ProcessOptionsReq(req); - } - auto header = req.headers.find("Source"); - if (header != req.headers.end()) { - return ProcessPullReq(header->second, req); - } - header = req.headers.find("Destination"); - if (header != req.headers.end()) { - return ProcessPushReq(header->second, req); - } - m_log.Emsg("ProcessReq", "COPY verb requested but no source or destination specified."); - return req.SendSimpleResp(400, NULL, NULL, "No Source or Destination specified", 0); +std::string TPCHandler::GetAuthz(XrdHttpExtReq &req) { + std::string authz; + auto authz_header = req.headers.find("Authorization"); + if (authz_header != req.headers.end()) { + char * quoted_url = quote(authz_header->second.c_str()); + std::stringstream ss; + ss << "authz=" << quoted_url; + free(quoted_url); + authz = ss.str(); } + return authz; +} - /** - * Abstract method in the base class, but does not seem to be used. - */ - virtual int Init(const char *cfgfile) { - return 0; +int TPCHandler::RedirectTransfer(XrdHttpExtReq &req, XrdOucErrInfo &error) { + int port; + const char *host = error.getErrText(port); + if ((host == nullptr) || (*host == '\0') || (port == 0)) { + char msg[] = "Internal error: redirect without hostname"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } + std::stringstream ss; + ss << "Location: http" << (m_desthttps ? "s" : "") << "://" << host << ":" << port << "/" << req.resource; + return req.SendSimpleResp(307, nullptr, const_cast(ss.str().c_str()), nullptr, 0); +} - virtual ~XrdHttpTPC() { - m_sfs = nullptr; // NOTE: must delete the SFS here as we may unload the destructor from memory below! - if (m_handle_base) { - dlclose(m_handle_base); - m_handle_base = nullptr; - } - if (m_handle_chained) { - dlclose(m_handle_chained); - m_handle_chained = nullptr; +int TPCHandler::OpenWaitStall(XrdSfsFile &fh, const std::string &resource, + int mode, int openMode, const XrdSecEntity &sec, + const std::string &authz) +{ + int open_result; + while (1) { + open_result = fh.open(resource.c_str(), mode, openMode, &sec, + authz.empty() ? nullptr : authz.c_str()); + if ((open_result == SFS_STALL) || (open_result == SFS_STARTED)) { + int secs_to_stall = fh.error.getErrInfo(); + if (open_result == SFS_STARTED) {secs_to_stall = secs_to_stall/2 + 5;} + sleep(secs_to_stall); } + break; } + return open_result; +} - XrdHttpTPC(XrdSysError *log, const char *config, XrdOucEnv *myEnv) : - m_log(*log) - { - if (!Configure(config, myEnv)) { - throw std::runtime_error("Failed to configure the HTTP third-party-copy handler."); - } +#ifdef XRD_CHUNK_RESP +/** + * Determine size at remote end. + */ +int TPCHandler::DetermineXferSize(CURL *curl, XrdHttpExtReq &req, TPC::State &state, + bool &success) { + success = false; + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + CURLcode res; + res = curl_easy_perform(curl); + if (res == CURLE_HTTP_RETURNED_ERROR) { + m_log.Emsg("DetermineXferSize", "Remote server failed request", curl_easy_strerror(res)); + curl_easy_cleanup(curl); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); + } else if (state.GetStatusCode() >= 400) { + std::stringstream ss; + ss << "Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg("DetermineXferSize", "Remote server failed request", ss.str().c_str()); + curl_easy_cleanup(curl); + return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); + } else if (res) { + m_log.Emsg("DetermineXferSize", "Curl failed", curl_easy_strerror(res)); + char msg[] = "Unknown internal transfer failure"; + curl_easy_cleanup(curl); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } + curl_easy_setopt(curl, CURLOPT_NOBODY, 0); + success = true; + return 0; +} -private: +int TPCHandler::SendPerfMarker(XrdHttpExtReq &req, TPC::State &state) { + std::stringstream ss; + const std::string crlf = "\n"; + ss << "Perf Marker" << crlf; + ss << "Timestamp: " << time(NULL) << crlf; + ss << "Stripe Index: 0" << crlf; + ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; + ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; + ss << "Total Stripe Count: 1" << crlf; + ss << "End" << crlf; + + return req.ChunkResp(ss.str().c_str(), 0); +} - /** - * Handle the OPTIONS verb as we have added a new one... - */ - int ProcessOptionsReq(XrdHttpExtReq &req) { - return req.SendSimpleResp(200, NULL, (char *) "DAV: 1\r\nDAV: \r\nAllow: HEAD,GET,PUT,PROPFIND,DELETE,OPTIONS,COPY", NULL, 0); +int TPCHandler::RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, TPC::State &state, + const char *log_prefix) +{ + // Create the multi-handle and add in the current transfer to it. + CURLM *multi_handle = curl_multi_init(); + if (!multi_handle) { + m_log.Emsg(log_prefix, "Failed to initialize a libcurl multi-handle"); + char msg[] = "Failed to initialize internal server memory"; + curl_easy_cleanup(curl); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } - static std::string GetAuthz(XrdHttpExtReq &req) { - std::string authz; - auto authz_header = req.headers.find("Authorization"); - if (authz_header != req.headers.end()) { - char * quoted_url = quote(authz_header->second.c_str()); - std::stringstream ss; - ss << "authz=" << quoted_url; - free(quoted_url); - authz = ss.str(); - } - return authz; + CURLMcode mres; + mres = curl_multi_add_handle(multi_handle, curl); + if (mres) { + m_log.Emsg(log_prefix, "Failed to add transfer to libcurl multi-handle", + curl_multi_strerror(mres)); + char msg[] = "Failed to initialize internal server handle"; + curl_easy_cleanup(curl); + curl_multi_cleanup(multi_handle); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); } - int RedirectTransfer(XrdHttpExtReq &req, XrdOucErrInfo &error) { - int port; - const char *host = error.getErrText(port); - if ((host == nullptr) || (*host == '\0') || (port == 0)) { - char msg[] = "Internal error: redirect without hostname"; - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } - std::stringstream ss; - ss << "Location: http" << (m_desthttps ? "s" : "") << "://" << host << ":" << port << "/" << req.resource; - return req.SendSimpleResp(307, nullptr, const_cast(ss.str().c_str()), nullptr, 0); + // Start response to client prior to the first call to curl_multi_perform + int retval = req.StartChunkedResp(201, "Created", "Content-Type: text/plain"); + if (retval) { + curl_easy_cleanup(curl); + curl_multi_cleanup(multi_handle); + return retval; } - int OpenWaitStall(XrdSfsFile &fh, const std::string &resource, int mode, int openMode, const XrdSecEntity &sec, - const std::string &authz) { - int open_result; - while (1) { - open_result = fh.open(resource.c_str(), mode, openMode, &sec, authz.empty() ? nullptr : authz.c_str()); - if ((open_result == SFS_STALL) || (open_result == SFS_STARTED)) { - int secs_to_stall = fh.error.getErrInfo(); - if (open_result == SFS_STARTED) {secs_to_stall = secs_to_stall/2 + 5;} - sleep(secs_to_stall); + // Transfer loop: use curl to actually run the transfer, but periodically + // interrupt things to send back performance updates to the client. + int running_handles = 1; + time_t last_marker = 0; + CURLcode res = static_cast(-1); + do { + time_t now = time(NULL); + time_t next_marker = last_marker + m_marker_period; + if (now >= next_marker) { + if (SendPerfMarker(req, state)) { + curl_multi_remove_handle(multi_handle, curl); + curl_easy_cleanup(curl); + curl_multi_cleanup(multi_handle); + return -1; } + last_marker = now; + } + mres = curl_multi_perform(multi_handle, &running_handles); + if (mres == CURLM_CALL_MULTI_PERFORM) { + // curl_multi_perform should be called again immediately. On newer + // versions of curl, this is no longer used. + continue; + } else if (mres != CURLM_OK) { + break; + } else if (running_handles == 0) { break; } - return open_result; - } - -#ifdef XRD_CHUNK_RESP - /** - * Determine size at remote end. - */ - int DetermineXferSize(CURL *curl, XrdHttpExtReq &req, XrdHttpTPCState &state, - bool &success) { - success = false; - curl_easy_setopt(curl, CURLOPT_NOBODY, 1); - CURLcode res; - res = curl_easy_perform(curl); - if (res == CURLE_HTTP_RETURNED_ERROR) { - m_log.Emsg("DetermineXferSize", "Remote server failed request", curl_easy_strerror(res)); - curl_easy_cleanup(curl); - return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); - } else if (state.GetStatusCode() >= 400) { - std::stringstream ss; - ss << "Remote side failed with status code " << state.GetStatusCode(); - m_log.Emsg("DetermineXferSize", "Remote server failed request", ss.str().c_str()); - curl_easy_cleanup(curl); - return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); - } else if (res) { - m_log.Emsg("DetermineXferSize", "Curl failed", curl_easy_strerror(res)); - char msg[] = "Unknown internal transfer failure"; - curl_easy_cleanup(curl); - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } - curl_easy_setopt(curl, CURLOPT_NOBODY, 0); - success = true; - return 0; - } - - int SendPerfMarker(XrdHttpExtReq &req, XrdHttpTPCState &state) { - std::stringstream ss; - const std::string crlf = "\n"; - ss << "Perf Marker" << crlf; - ss << "Timestamp: " << time(NULL) << crlf; - ss << "Stripe Index: 0" << crlf; - ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; - ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; - ss << "Total Stripe Count: 1" << crlf; - ss << "End" << crlf; - - return req.ChunkResp(ss.str().c_str(), 0); - } - - int RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, XrdHttpTPCState &state, - const char *log_prefix) { - - // Create the multi-handle and add in the current transfer to it. - CURLM *multi_handle = curl_multi_init(); - if (!multi_handle) { - m_log.Emsg(log_prefix, "Failed to initialize a libcurl multi-handle"); - char msg[] = "Failed to initialize internal server memory"; - curl_easy_cleanup(curl); - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } - - CURLMcode mres; - mres = curl_multi_add_handle(multi_handle, curl); - if (mres) { - m_log.Emsg(log_prefix, "Failed to add transfer to libcurl multi-handle", - curl_multi_strerror(mres)); - char msg[] = "Failed to initialize internal server handle"; - curl_easy_cleanup(curl); - curl_multi_cleanup(multi_handle); - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } - - // Start response to client prior to the first call to curl_multi_perform - int retval = req.StartChunkedResp(201, "Created", "Content-Type: text/plain"); - if (retval) { - curl_easy_cleanup(curl); - curl_multi_cleanup(multi_handle); - return retval; - } - - // Transfer loop: use curl to actually run the transfer, but periodically - // interrupt things to send back performance updates to the client. - int running_handles = 1; - time_t last_marker = 0; - CURLcode res = static_cast(-1); - do { - time_t now = time(NULL); - time_t next_marker = last_marker + m_marker_period; - if (now >= next_marker) { - if (SendPerfMarker(req, state)) { - curl_multi_remove_handle(multi_handle, curl); - curl_easy_cleanup(curl); - curl_multi_cleanup(multi_handle); - return -1; - } - last_marker = now; - } - mres = curl_multi_perform(multi_handle, &running_handles); - if (mres == CURLM_CALL_MULTI_PERFORM) { - // curl_multi_perform should be called again immediately. On newer - // versions of curl, this is no longer used. - continue; - } else if (mres != CURLM_OK) { - break; - } else if (running_handles == 0) { - break; - } - //printf("There are %d running handles\n", running_handles); + //printf("There are %d running handles\n", running_handles); // Harvest any messages, looking for CURLMSG_DONE. CURLMsg *msg; @@ -487,343 +275,201 @@ class XrdHttpTPC : public XrdHttpExtHandler { } } while (msg); - int64_t max_sleep_time = next_marker - time(NULL); - if (max_sleep_time <= 0) { - continue; - } - int fd_count; - mres = curl_multi_wait(multi_handle, NULL, 0, max_sleep_time*1000, &fd_count); - if (mres != CURLM_OK) { - break; - } - } while (running_handles); - + int64_t max_sleep_time = next_marker - time(NULL); + if (max_sleep_time <= 0) { + continue; + } + int fd_count; + mres = curl_multi_wait(multi_handle, NULL, 0, max_sleep_time*1000, &fd_count); if (mres != CURLM_OK) { - m_log.Emsg(log_prefix, "Internal libcurl multi-handle error", - curl_multi_strerror(mres)); - char msg[] = "Internal server error due to libcurl"; - curl_multi_remove_handle(multi_handle, curl); - curl_easy_cleanup(curl); - - curl_multi_cleanup(multi_handle); - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + break; } + } while (running_handles); - // Harvest any messages, looking for CURLMSG_DONE. - CURLMsg *msg; - do { - int msgq = 0; - msg = curl_multi_info_read(multi_handle, &msgq); - if (msg && (msg->msg == CURLMSG_DONE)) { - CURL *easy_handle = msg->easy_handle; - res = msg->data.result; - curl_multi_remove_handle(multi_handle, easy_handle); - curl_easy_cleanup(easy_handle); - } - } while (msg); + if (mres != CURLM_OK) { + m_log.Emsg(log_prefix, "Internal libcurl multi-handle error", + curl_multi_strerror(mres)); + char msg[] = "Internal server error due to libcurl"; + curl_multi_remove_handle(multi_handle, curl); + curl_easy_cleanup(curl); - if (res == -1) { // No transfers returned?!? - curl_multi_remove_handle(multi_handle, curl); - curl_easy_cleanup(curl); - curl_multi_cleanup(multi_handle); - char msg[] = "Internal state error in libcurl"; - m_log.Emsg(log_prefix, msg); - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } curl_multi_cleanup(multi_handle); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } - // Generate the final response back to the client. - std::stringstream ss; - if (res != CURLE_OK) { - m_log.Emsg(log_prefix, "Remote server failed request", curl_easy_strerror(res)); - ss << "failure: " << curl_easy_strerror(res); - } else if (state.GetStatusCode() >= 400) { - ss << "failure: Remote side failed with status code " << state.GetStatusCode(); - m_log.Emsg(log_prefix, "Remote server failed request", ss.str().c_str()); - } else { - ss << "success: Created"; + // Harvest any messages, looking for CURLMSG_DONE. + CURLMsg *msg; + do { + int msgq = 0; + msg = curl_multi_info_read(multi_handle, &msgq); + if (msg && (msg->msg == CURLMSG_DONE)) { + CURL *easy_handle = msg->easy_handle; + res = msg->data.result; + curl_multi_remove_handle(multi_handle, easy_handle); + curl_easy_cleanup(easy_handle); } + } while (msg); - if ((retval = req.ChunkResp(ss.str().c_str(), 0))) { - return retval; - } - return req.ChunkResp(nullptr, 0); + if (res == -1) { // No transfers returned?!? + curl_multi_remove_handle(multi_handle, curl); + curl_easy_cleanup(curl); + curl_multi_cleanup(multi_handle); + char msg[] = "Internal state error in libcurl"; + m_log.Emsg(log_prefix, msg); + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + curl_multi_cleanup(multi_handle); + + // Generate the final response back to the client. + std::stringstream ss; + if (res != CURLE_OK) { + m_log.Emsg(log_prefix, "Remote server failed request", curl_easy_strerror(res)); + ss << "failure: " << curl_easy_strerror(res); + } else if (state.GetStatusCode() >= 400) { + ss << "failure: Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg(log_prefix, "Remote server failed request", ss.str().c_str()); + } else { + ss << "success: Created"; + } + + if ((retval = req.ChunkResp(ss.str().c_str(), 0))) { + return retval; } + return req.ChunkResp(nullptr, 0); +} #else - int RunCurlBasic(CURL *curl, XrdHttpExtReq &req, XrdHttpTPCState &state, - const char *log_prefix) { - CURLcode res; - res = curl_easy_perform(curl); - curl_easy_cleanup(curl); - if (res == CURLE_HTTP_RETURNED_ERROR) { - m_log.Emsg(log_prefix, "Remote server failed request", curl_easy_strerror(res)); - return req.SendSimpleResp(500, nullptr, nullptr, const_cast(curl_easy_strerror(res)), 0); - } else if (state.GetStatusCode() >= 400) { - std::stringstream ss; - ss << "Remote side failed with status code " << state.GetStatusCode(); - m_log.Emsg(log_prefix, "Remote server failed request", ss.str().c_str()); - return req.SendSimpleResp(500, nullptr, nullptr, const_cast(ss.str().c_str()), 0); - } else if (res) { - m_log.Emsg(log_prefix, "Curl failed", curl_easy_strerror(res)); - char msg[] = "Unknown internal transfer failure"; - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } else { - char msg[] = "Created"; - return req.SendSimpleResp(201, nullptr, nullptr, msg, 0); - } +int TPCHandler::RunCurlBasic(CURL *curl, XrdHttpExtReq &req, TPC::State &state, + const char *log_prefix) { + CURLcode res; + res = curl_easy_perform(curl); + curl_easy_cleanup(curl); + if (res == CURLE_HTTP_RETURNED_ERROR) { + m_log.Emsg(log_prefix, "Remote server failed request", curl_easy_strerror(res)); + return req.SendSimpleResp(500, nullptr, nullptr, + const_cast(curl_easy_strerror(res)), 0); + } else if (state.GetStatusCode() >= 400) { + std::stringstream ss; + ss << "Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg(log_prefix, "Remote server failed request", ss.str().c_str()); + return req.SendSimpleResp(500, nullptr, nullptr, + const_cast(ss.str().c_str()), 0); + } else if (res) { + m_log.Emsg(log_prefix, "Curl failed", curl_easy_strerror(res)); + char msg[] = "Unknown internal transfer failure"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } else { + char msg[] = "Created"; + return req.SendSimpleResp(201, nullptr, nullptr, msg, 0); } +} #endif - int ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) { - m_log.Emsg("ProcessPushReq", "Starting a push request for resource", resource.c_str()); - CURL *curl = curl_easy_init(); - if (!curl) { - char msg[] = "Failed to initialize internal transfer resources"; - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } - char *name = req.GetSecEntity().name; - std::unique_ptr fh(m_sfs->newFile(name, m_monid++)); - if (!fh.get()) { - char msg[] = "Failed to initialize internal transfer file handle"; - return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } - std::string authz = GetAuthz(req); - - int open_results = OpenWaitStall(*fh, req.resource, SFS_O_RDONLY, 0644, req.GetSecEntity(), authz); - if (SFS_REDIRECT == open_results) { - return RedirectTransfer(req, fh->error); - } else if (SFS_OK != open_results) { - int code; - char msg_generic[] = "Failed to open local resource"; - const char *msg = fh->error.getErrText(code); - if (msg == nullptr) msg = msg_generic; - int status_code = 400; - if (code == EACCES) status_code = 401; - int resp_result = req.SendSimpleResp(status_code, nullptr, nullptr, const_cast(msg), 0); - fh->close(); - return resp_result; - } - if (!m_cadir.empty()) { +int TPCHandler::ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) { + m_log.Emsg("ProcessPushReq", "Starting a push request for resource", resource.c_str()); + CURL *curl = curl_easy_init(); + if (!curl) { + char msg[] = "Failed to initialize internal transfer resources"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + char *name = req.GetSecEntity().name; + std::unique_ptr fh(m_sfs->newFile(name, m_monid++)); + if (!fh.get()) { + char msg[] = "Failed to initialize internal transfer file handle"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + std::string authz = GetAuthz(req); + + int open_results = OpenWaitStall(*fh, req.resource, SFS_O_RDONLY, 0644, + req.GetSecEntity(), authz); + if (SFS_REDIRECT == open_results) { + return RedirectTransfer(req, fh->error); + } else if (SFS_OK != open_results) { + int code; + char msg_generic[] = "Failed to open local resource"; + const char *msg = fh->error.getErrText(code); + if (msg == nullptr) msg = msg_generic; + int status_code = 400; + if (code == EACCES) status_code = 401; + int resp_result = req.SendSimpleResp(status_code, nullptr, nullptr, + const_cast(msg), 0); + fh->close(); + return resp_result; + } + if (!m_cadir.empty()) { curl_easy_setopt(curl, CURLOPT_CAPATH, m_cadir.c_str()); - } - curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); + } + curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - XrdHttpTPCState state(std::move(fh), curl, true); - state.CopyHeaders(req); + TPC::State state(std::move(fh), curl, true); + state.CopyHeaders(req); #ifdef XRD_CHUNK_RESP - return RunCurlWithUpdates(curl, req, state, "ProcessPushReq"); + return RunCurlWithUpdates(curl, req, state, "ProcessPushReq"); #else - return RunCurlBasic(curl, req, state, "ProcessPushReq"); + return RunCurlBasic(curl, req, state, "ProcessPushReq"); #endif - } +} - int ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) { - CURL *curl = curl_easy_init(); - if (!curl) { +int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) { + CURL *curl = curl_easy_init(); + if (!curl) { char msg[] = "Failed to initialize internal transfer resources"; return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } - char *name = req.GetSecEntity().name; - std::unique_ptr fh(m_sfs->newFile(name, m_monid++)); - if (!fh.get()) { + } + char *name = req.GetSecEntity().name; + std::unique_ptr fh(m_sfs->newFile(name, m_monid++)); + if (!fh.get()) { char msg[] = "Failed to initialize internal transfer file handle"; return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); - } - std::string authz = GetAuthz(req); - XrdSfsFileOpenMode mode = SFS_O_CREAT; - /*auto overwrite_header = req.headers.find("Overwrite"); - if ((overwrite_header == req.headers.end()) || (overwrite_header->second == "T")) { - mode = SFS_O_TRUNC|SFS_O_POSC; - }*/ - - int open_result = OpenWaitStall(*fh, req.resource, mode|SFS_O_WRONLY, 0644, req.GetSecEntity(), authz); - if (SFS_REDIRECT == open_result) { - return RedirectTransfer(req, fh->error); - } else if (SFS_OK != open_result) { - int code; - char msg_generic[] = "Failed to open local resource"; - const char *msg = fh->error.getErrText(code); - if ((msg == nullptr) || (*msg == '\0')) msg = msg_generic; - int status_code = 400; - if (code == EACCES) status_code = 401; - if (code == EEXIST) status_code = 412; - int resp_result = req.SendSimpleResp(status_code, nullptr, nullptr, const_cast(msg), 0); - fh->close(); - return resp_result; - } - if (!m_cadir.empty()) { - curl_easy_setopt(curl, CURLOPT_CAPATH, m_cadir.c_str()); - } - curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - XrdHttpTPCState state(std::move(fh), curl, false); - state.CopyHeaders(req); + } + std::string authz = GetAuthz(req); + XrdSfsFileOpenMode mode = SFS_O_CREAT; + /*auto overwrite_header = req.headers.find("Overwrite"); + if ((overwrite_header == req.headers.end()) || (overwrite_header->second == "T")) { + mode = SFS_O_TRUNC|SFS_O_POSC; + }*/ + + int open_result = OpenWaitStall(*fh, req.resource, mode|SFS_O_WRONLY, 0644, + req.GetSecEntity(), authz); + if (SFS_REDIRECT == open_result) { + return RedirectTransfer(req, fh->error); + } else if (SFS_OK != open_result) { + int code; + char msg_generic[] = "Failed to open local resource"; + const char *msg = fh->error.getErrText(code); + if ((msg == nullptr) || (*msg == '\0')) msg = msg_generic; + int status_code = 400; + if (code == EACCES) status_code = 401; + if (code == EEXIST) status_code = 412; + int resp_result = req.SendSimpleResp(status_code, nullptr, nullptr, + const_cast(msg), 0); + fh->close(); + return resp_result; + } + if (!m_cadir.empty()) { + curl_easy_setopt(curl, CURLOPT_CAPATH, m_cadir.c_str()); + } + curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); + TPC::State state(std::move(fh), curl, false); + state.CopyHeaders(req); #ifdef XRD_CHUNK_RESP - int result; - bool success; - if ((result = DetermineXferSize(curl, req, state, success)) || !success) { - return result; - } - std::stringstream ss; - ss << "Successfully determined remote size for pull request: " << state.GetContentLength(); - m_log.Emsg("ProcessPullReq", ss.str().c_str()); - state.ResetAfterSize(); + int result; + bool success; + if ((result = DetermineXferSize(curl, req, state, success)) || !success) { + return result; + } + std::stringstream ss; + ss << "Successfully determined remote size for pull request: " << state.GetContentLength(); + m_log.Emsg("ProcessPullReq", ss.str().c_str()); + state.ResetAfterSize(); - return RunCurlWithUpdates(curl, req, state, "ProcessPullReq"); + return RunCurlWithUpdates(curl, req, state, "ProcessPullReq"); #else - return RunCurlBasic(curl, req, state, "ProcessPullReq"); + return RunCurlBasic(curl, req, state, "ProcessPullReq"); #endif - } - - bool ConfigureFSLib(XrdOucStream &Config, std::string &path1, bool &path1_alt, std::string &path2, bool &path2_alt) { - char *val; - if (!(val = Config.GetWord())) { - m_log.Emsg("Config", "fslib not specified"); - return false; - } - if (!strcmp("throttle", val)) { - path2 = "libXrdThrottle.so"; - if (!(val = Config.GetWord())) { - m_log.Emsg("Config", "fslib throttle target library not specified"); - return false; - } - } - else if (!strcmp("-2", val)) { - path2_alt = true; - if (!(val = Config.GetWord())) { - m_log.Emsg("Config", "fslib library not specified"); - return false; - } - path2 = val; - } - else { - path2 = val; - } - if (!(val = Config.GetWord()) || !strcmp("default", val)) { - if (path2 == "libXrdThrottle.so") { - path1 = "default"; - } else if (!path2.empty()) { - path1 = path2; - path2 = ""; - path1_alt = path2_alt; - } - } else if (!strcmp("-2", val)) { - path1_alt = true; - if (!(val = Config.GetWord())) { - m_log.Emsg("Config", "fslib base library not specified"); - return false; - } - path1 = val; - } else { - path2 = val; - } - return true; - } - - bool Configure(const char *configfn, XrdOucEnv *myEnv) { - - XrdOucStream Config(&m_log, getenv("XRDINSTANCE"), myEnv, "=====> "); - - std::string authLib; - std::string authLibParms; - int cfgFD = open(configfn, O_RDONLY, 0); - if (cfgFD < 0) { - m_log.Emsg("Config", errno, "open config file", configfn); - return false; - } - Config.Attach(cfgFD); - const char *val; - std::string path2, path1 = "default"; - bool path1_alt = false, path2_alt = false; - while ((val = Config.GetMyFirstWord())) { - if (!strcmp("xrootd.fslib", val)) { - if (!ConfigureFSLib(Config, path1, path1_alt, path2, path2_alt)) { - Config.Close(); - m_log.Emsg("Config", "Failed to parse the xrootd.fslib directive"); - return false; - } - m_log.Emsg("Config", "xrootd.fslib line successfully processed by XrdHttpTPC"); - } else if (!strcmp("http.desthttps", val)) { - if (!(val = Config.GetWord())) { - Config.Close(); - m_log.Emsg("Config", "http.desthttps value not specified"); - return false; - } - if (!strcmp("1", val) || !strcasecmp("yes", val) || !strcasecmp("true", val)) { - m_desthttps = true; - } else if (!strcmp("1", val) || !strcasecmp("yes", val) || !strcasecmp("true", val)) { - m_desthttps = false; - } else { - Config.Close(); - m_log.Emsg("Config", "https.dests value is invalid", val); - return false; - } - } else if (!strcmp("http.cadir", val)) { - if (!(val = Config.GetWord())) { - Config.Close(); - m_log.Emsg("Config", "http.cadir value not specified"); - return false; - } - m_cadir = val; - } - } - Config.Close(); - - XrdSfsFileSystem *base_sfs = nullptr; - if (path1 == "default") { - m_log.Emsg("Config", "Loading the default filesystem"); - base_sfs = XrdSfsGetDefaultFileSystem(nullptr, m_log.logger(), configfn, myEnv); - m_log.Emsg("Config", "Finished loading the default filesystem"); - } else { - char resolvePath[2048]; - bool usedAltPath{true}; - if (!XrdOucPinPath(path1.c_str(), usedAltPath, resolvePath, 2048)) { - m_log.Emsg("Config", "Failed to locate appropriately versioned base filesystem library for ", path1.c_str()); - return false; - } - m_handle_base = dlopen(resolvePath, RTLD_LOCAL|RTLD_NOW); - if (m_handle_base == nullptr) { - m_log.Emsg("Config", "Failed to base plugin ", resolvePath, dlerror()); - return false; - } - base_sfs = load_sfs(m_handle_base, path1_alt, m_log, path1, configfn, *myEnv, nullptr); - } - if (!base_sfs) { - m_log.Emsg("Config", "Failed to initialize filesystem library for XrdHttpTPC from ", path1.c_str()); - return false; - } - XrdSfsFileSystem *chained_sfs = nullptr; - if (!path2.empty()) { - char resolvePath[2048]; - bool usedAltPath{true}; - if (!XrdOucPinPath(path2.c_str(), usedAltPath, resolvePath, 2048)) { - m_log.Emsg("Config", "Failed to locate appropriately versioned chained filesystem library for ", path2.c_str()); - return false; - } - m_handle_chained = dlopen(resolvePath, RTLD_LOCAL|RTLD_NOW); - if (m_handle_chained == nullptr) { - m_log.Emsg("Config", "Failed to chained plugin ", resolvePath, dlerror()); - return false; - } - chained_sfs = load_sfs(m_handle_chained, path2_alt, m_log, path2, configfn, *myEnv, base_sfs); - } - m_sfs.reset(chained_sfs ? chained_sfs : base_sfs); - m_log.Emsg("Config", "Successfully configured the filesystem object for XrdHttpTPC"); - return true; - } - - static constexpr int m_marker_period = 5; - bool m_desthttps{false}; - std::string m_cadir; - static std::atomic m_monid; - XrdSysError &m_log; - std::unique_ptr m_sfs; - void *m_handle_base{nullptr}; - void *m_handle_chained{nullptr}; -}; +} -std::atomic XrdHttpTPC::m_monid{0}; extern "C" { @@ -833,14 +479,14 @@ XrdHttpExtHandler *XrdHttpGetExtHandler(XrdSysError *log, const char * config, c return nullptr; } - XrdHttpTPC *retval{nullptr}; + TPCHandler *retval{nullptr}; if (!config) { - log->Emsg("Initialize", "XrdHttpTPC requires a config filename in order to load"); + log->Emsg("Initialize", "TPC handler requires a config filename in order to load"); return nullptr; } try { - log->Emsg("Initialize", "Will load configuration for XrdHttpTPC from", config); - retval = new XrdHttpTPC(log, config, myEnv); + log->Emsg("Initialize", "Will load configuration for the TPC handler from", config); + retval = new TPCHandler(log, config, myEnv); } catch (std::runtime_error &re) { log->Emsg("Initialize", "Encountered a runtime failure when loading ", re.what()); //printf("Provided env vars: %p, XrdInet*: %p\n", myEnv, myEnv->GetPtr("XrdInet*")); diff --git a/src/tpc.hh b/src/tpc.hh new file mode 100644 index 00000000000..163ba77c956 --- /dev/null +++ b/src/tpc.hh @@ -0,0 +1,67 @@ + +#include +#include +#include + +#include "XrdHttp/XrdHttpExtHandler.hh" + +class XrdOucErrInfo; +class XrdOucStream; +class XrdSfsFile; +class XrdSfsFileSystem; +typedef void CURL; + +namespace TPC { +class State; + +class TPCHandler : public XrdHttpExtHandler { +public: + TPCHandler(XrdSysError *log, const char *config, XrdOucEnv *myEnv); + virtual ~TPCHandler(); + + virtual bool MatchesPath(const char *verb, const char *path) override; + virtual int ProcessReq(XrdHttpExtReq &req) override; + // Abstract method in the base class, but does not seem to be used + virtual int Init(const char *cfgfile) override {return 0;} + +private: + int ProcessOptionsReq(XrdHttpExtReq &req); + + static std::string GetAuthz(XrdHttpExtReq &req); + + int RedirectTransfer(XrdHttpExtReq &req, XrdOucErrInfo &error); + + int OpenWaitStall(XrdSfsFile &fh, const std::string &resource, int mode, + int openMode, const XrdSecEntity &sec, + const std::string &authz); + +#ifdef XRD_CHUNK_RESP + int DetermineXferSize(CURL *curl, XrdHttpExtReq &req, TPC::State &state, + bool &success); + + int SendPerfMarker(XrdHttpExtReq &req, TPC::State &state); + + int RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, TPC::State &state, + const char *log_prefix); +#else + int RunCurlBasic(CURL *curl, XrdHttpExtReq &req, TPC::State &state, + const char *log_prefix); +#endif + + int ProcessPushReq(const std::string & resource, XrdHttpExtReq &req); + int ProcessPullReq(const std::string &resource, XrdHttpExtReq &req); + + bool ConfigureFSLib(XrdOucStream &Config, std::string &path1, bool &path1_alt, + std::string &path2, bool &path2_alt); + bool Configure(const char *configfn, XrdOucEnv *myEnv); + + static constexpr int m_marker_period = 5; + bool m_desthttps{false}; + std::string m_cadir; + static std::atomic m_monid; + XrdSysError &m_log; + std::unique_ptr m_sfs; + void *m_handle_base{nullptr}; + void *m_handle_chained{nullptr}; +}; +} From 0ce71badc22b1c650f62682e45b0d4ff375ced38 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 4 Jan 2018 10:19:36 -0600 Subject: [PATCH 18/29] Refactor file interaction out of the state object. Currently, there is a one-to-one correspondence between a HTTP request (a state object) and a stream going out to a Xrootd file handle. In preparation for multiple streams, we want to split up the concept of a request and a file stream. Multiple streams will be implemented by doing multiple large requests going to the same file. --- CMakeLists.txt | 2 +- src/state.cpp | 8 ++++---- src/state.hh | 9 ++++++--- src/stream.cpp | 30 ++++++++++++++++++++++++++++++ src/stream.hh | 34 ++++++++++++++++++++++++++++++++++ src/tpc.cpp | 15 +++++++++------ 6 files changed, 84 insertions(+), 14 deletions(-) create mode 100644 src/stream.cpp create mode 100644 src/stream.hh diff --git a/CMakeLists.txt b/CMakeLists.txt index 68ccbdfbb5c..f249a3bb4dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ pkg_check_modules(CURL REQUIRED libcurl) include_directories(${XROOTD_INCLUDES} ${XROOTD_PRIVATE_INCLUDES} ${CURL_INCLUDE_DIRS}) -add_library(XrdHttpTPC SHARED src/tpc.cpp src/state.cpp src/configure.cpp) +add_library(XrdHttpTPC SHARED src/tpc.cpp src/state.cpp src/configure.cpp src/stream.cpp) if ( XRD_CHUNK_RESP ) set_target_properties(XrdHttpTPC PROPERTIES COMPILE_DEFINITIONS "XRD_CHUNK_RESP" ) endif () diff --git a/src/state.cpp b/src/state.cpp index d8435f3a0db..faa6abb67c1 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -9,6 +9,7 @@ #include "XrdTpcVersion.hh" #include "state.hh" +#include "stream.hh" using namespace TPC; @@ -18,7 +19,6 @@ State::~State() { m_headers = nullptr; curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, m_headers); } - m_fh->close(); } bool State::InstallHandlers(CURL *curl) { @@ -30,7 +30,7 @@ bool State::InstallHandlers(CURL *curl) { curl_easy_setopt(curl, CURLOPT_READFUNCTION, &State::ReadCB); curl_easy_setopt(curl, CURLOPT_READDATA, this); struct stat buf; - if (SFS_OK == m_fh->stat(&buf)) { + if (SFS_OK == m_stream.Stat(&buf)) { curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, buf.st_size); } } else { @@ -134,7 +134,7 @@ size_t State::WriteCB(void *buffer, size_t size, size_t nitems, void *userdata) } int State::Write(char *buffer, size_t size) { - int retval = m_fh->write(m_offset, buffer, size); + int retval = m_stream.Write(m_start_offset + m_offset, buffer, size); if (retval == SFS_ERROR) { return -1; } @@ -151,7 +151,7 @@ size_t State::ReadCB(void *buffer, size_t size, size_t nitems, void *userdata) { } int State::Read(char *buffer, size_t size) { - int retval = m_fh->read(m_offset, buffer, size); + int retval = m_stream.Read(m_start_offset + m_offset, buffer, size); if (retval == SFS_ERROR) { return -1; } diff --git a/src/state.hh b/src/state.hh index 3ba5b7a68cb..01f46b88168 100644 --- a/src/state.hh +++ b/src/state.hh @@ -12,12 +12,14 @@ class XrdHttpExtReq; typedef void CURL; namespace TPC { +class Stream; class State { public: - State (std::unique_ptr fh, CURL *curl, bool push) : + State (off_t start_offset, Stream &stream, CURL *curl, bool push) : m_push(push), - m_fh(std::move(fh)), + m_start_offset(start_offset), + m_stream(stream), m_curl(curl) { InstallHandlers(curl); @@ -51,9 +53,10 @@ private: bool m_recv_status_line{false}; // whether we have received a status line in the response from the remote host. bool m_recv_all_headers{false}; // true if we have seen the end of headers. off_t m_offset{0}; // number of bytes we have received. + const off_t m_start_offset{0}; // offset where we started in the file. int m_status_code{-1}; // status code from HTTP response. off_t m_content_length{-1}; // value of Content-Length header, if we received one. - std::unique_ptr m_fh; // file-handle corresponding to this transfer. + Stream &m_stream; // stream corresponding to this transfer. CURL *m_curl{nullptr}; // libcurl handle struct curl_slist *m_headers{nullptr}; // any headers we set as part of the libcurl request. std::string m_resp_protocol; // Response protocol in the HTTP status line. diff --git a/src/stream.cpp b/src/stream.cpp new file mode 100644 index 00000000000..4d55ad61fd6 --- /dev/null +++ b/src/stream.cpp @@ -0,0 +1,30 @@ + +#include "stream.hh" + +#include "XrdSfs/XrdSfsInterface.hh" + +using namespace TPC; + +Stream::~Stream() +{ + m_fh->close(); +} + + +int +Stream::Stat(struct stat* buf) +{ + return m_fh->stat(buf); +} + +int +Stream::Write(off_t offset, const char *buf, size_t size) +{ + return m_fh->write(offset, buf, size); +} + +int +Stream::Read(off_t offset, char *buf, size_t size) +{ + return m_fh->read(offset, buf, size); +} diff --git a/src/stream.hh b/src/stream.hh new file mode 100644 index 00000000000..9c52ff0c5e5 --- /dev/null +++ b/src/stream.hh @@ -0,0 +1,34 @@ + +/** + * The "stream" interface is a simple abstraction of a file handle. + * + * The abstraction layer is necessary to do the necessary buffering + * of multi-stream writes where the underlying filesystem only + * supports single-stream writes. + */ + +#include + +struct stat; + +class XrdSfsFile; + +namespace TPC { +class Stream { +public: + Stream(std::unique_ptr fh) + : m_fh(std::move(fh)) + {} + + ~Stream(); + + int Stat(struct stat *); + + int Read(off_t offset, char *buffer, size_t size); + + int Write(off_t offset, const char *buffer, size_t size); + +private: + std::unique_ptr m_fh; +}; +} diff --git a/src/tpc.cpp b/src/tpc.cpp index 30d3f4b4511..59142035e44 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -17,6 +17,7 @@ #include "XrdTpcVersion.hh" #include "state.hh" +#include "stream.hh" #include "tpc.hh" using namespace TPC; @@ -161,7 +162,7 @@ int TPCHandler::OpenWaitStall(XrdSfsFile &fh, const std::string &resource, /** * Determine size at remote end. */ -int TPCHandler::DetermineXferSize(CURL *curl, XrdHttpExtReq &req, TPC::State &state, +int TPCHandler::DetermineXferSize(CURL *curl, XrdHttpExtReq &req, State &state, bool &success) { success = false; curl_easy_setopt(curl, CURLOPT_NOBODY, 1); @@ -188,7 +189,7 @@ int TPCHandler::DetermineXferSize(CURL *curl, XrdHttpExtReq &req, TPC::State &st return 0; } -int TPCHandler::SendPerfMarker(XrdHttpExtReq &req, TPC::State &state) { +int TPCHandler::SendPerfMarker(XrdHttpExtReq &req, State &state) { std::stringstream ss; const std::string crlf = "\n"; ss << "Perf Marker" << crlf; @@ -202,7 +203,7 @@ int TPCHandler::SendPerfMarker(XrdHttpExtReq &req, TPC::State &state) { return req.ChunkResp(ss.str().c_str(), 0); } -int TPCHandler::RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, TPC::State &state, +int TPCHandler::RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, State &state, const char *log_prefix) { // Create the multi-handle and add in the current transfer to it. @@ -338,7 +339,7 @@ int TPCHandler::RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, TPC::State &s return req.ChunkResp(nullptr, 0); } #else -int TPCHandler::RunCurlBasic(CURL *curl, XrdHttpExtReq &req, TPC::State &state, +int TPCHandler::RunCurlBasic(CURL *curl, XrdHttpExtReq &req, State &state, const char *log_prefix) { CURLcode res; res = curl_easy_perform(curl); @@ -400,7 +401,8 @@ int TPCHandler::ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) } curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - TPC::State state(std::move(fh), curl, true); + Stream stream(std::move(fh)); + State state(0, stream, curl, true); state.CopyHeaders(req); #ifdef XRD_CHUNK_RESP @@ -450,7 +452,8 @@ int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) curl_easy_setopt(curl, CURLOPT_CAPATH, m_cadir.c_str()); } curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - TPC::State state(std::move(fh), curl, false); + Stream stream(std::move(fh)); + State state(0, stream, curl, false); state.CopyHeaders(req); #ifdef XRD_CHUNK_RESP From 1e8d9907f85d910a969de945c8cf6313575577de Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 4 Jan 2018 14:37:06 -0600 Subject: [PATCH 19/29] Add buffering / re-ordering infrastructure. Add skeleton of the data re-ordering infrastructure. As there's only a single-stream (no re-order necessary), none of this will get invoked for now. --- src/stream.cpp | 57 ++++++++++++++++++++++++++++++++++++- src/stream.hh | 77 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/tpc.cpp | 4 +-- 3 files changed, 132 insertions(+), 6 deletions(-) diff --git a/src/stream.cpp b/src/stream.cpp index 4d55ad61fd6..6deef39685e 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -20,7 +20,62 @@ Stream::Stat(struct stat* buf) int Stream::Write(off_t offset, const char *buf, size_t size) { - return m_fh->write(offset, buf, size); + bool buffer_accepted = false; + int retval = size; + if (offset == m_offset) { + retval = m_fh->write(offset, buf, size); + buffer_accepted = true; + if (retval != SFS_ERROR) { + m_offset += retval; + } + // If there are no in-use buffers, then we don't need to + // do any accounting. + if (m_avail_count == m_buffers.size()) { + return retval; + } + } + //printf("Performing stream buffer accounting. Available buffers: %lu, total buffers %lu.\n", m_avail_count, m_buffers.size()); + // Even if we already accepted the current data, always + // iterate through available buffers and try to write as + // much out to disk as possible. + Entry *avail_entry = nullptr; + bool buffer_was_written; + do { + m_avail_count = 0; + buffer_was_written = false; + for (Entry &entry : m_buffers) { + // Always try to dump from memory. + if (entry.Write(*this) > 0) { + buffer_was_written = true; + } + if (entry.Available()) { // Empty buffer + if (!avail_entry) {avail_entry = &entry;} + m_avail_count ++; + } + else if (!buffer_accepted && entry.Accept(offset, buf, size)) { + buffer_accepted = true; + } + } + } while ((m_avail_count != m_buffers.size()) && buffer_was_written); + + if (!buffer_accepted) { // No place for this data in allocated buffers + if (!avail_entry) { // No available buffers to allocate. + return SFS_ERROR; + } + if (!avail_entry->Accept(offset, buf, size)) { // Empty buffer cannot accept?!? + return SFS_ERROR; + } + m_avail_count --; + } + + // If we have low buffer occupancy, then release memory. + if ((m_buffers.size() > 2) && (m_avail_count * 2 > m_buffers.size())) { + for (Entry &entry : m_buffers) { + entry.ShrinkIfUnused(); + } + } + + return retval; } int diff --git a/src/stream.hh b/src/stream.hh index 9c52ff0c5e5..78d6bef028f 100644 --- a/src/stream.hh +++ b/src/stream.hh @@ -8,6 +8,9 @@ */ #include +#include + +#include struct stat; @@ -16,9 +19,15 @@ class XrdSfsFile; namespace TPC { class Stream { public: - Stream(std::unique_ptr fh) - : m_fh(std::move(fh)) - {} + Stream(std::unique_ptr fh, size_t max_blocks, size_t buffer_size) + : m_avail_count(max_blocks), + m_fh(std::move(fh)) + { + m_buffers.reserve(max_blocks); + for (size_t idx=0; idx < max_blocks; idx++) { + m_buffers.emplace_back(buffer_size); + } + } ~Stream(); @@ -29,6 +38,68 @@ public: int Write(off_t offset, const char *buffer, size_t size); private: + + class Entry { + public: + Entry(size_t capacity) : + m_capacity(capacity) + {} + + bool Available() const {return m_offset == -1;} + + int Write(Stream &stream) { + if (Available() || !CanWrite(stream)) {return 0;} + // Currently, only full writes are accepted. + int size_desired = m_size; + int retval = stream.Write(m_offset, &m_buffer[0], size_desired); + m_size = 0; + m_offset = -1; + if (retval != size_desired) { + return -1; + } + return retval; + } + + bool Accept(off_t offset, const char *buf, size_t size) { + // Validate acceptance criteria. + if ((m_offset != -1) && (offset != m_offset + static_cast(m_size))) { + return false; + } + if (size > m_capacity - m_size) { + return false; + } + + // Inflate the underlying buffer if needed. + ssize_t new_bytes_needed = (m_size + size) - m_buffer.capacity(); + if (new_bytes_needed > 0) { + m_buffer.reserve(m_capacity); + } + + // Finally, do the copy. + memcpy(&m_buffer[0] + m_size, buf, size); + m_size += size; + return true; + } + + void ShrinkIfUnused() { + if (!Available()) {return;} + m_buffer.shrink_to_fit(); + } + + private: + bool CanWrite(Stream &stream) const { + return (m_size > 0) && (m_offset == stream.m_offset); + } + + off_t m_offset{-1}; // Offset within file that m_buffer[0] represents. + const size_t m_capacity; + size_t m_size{0}; // Number of bytes held in buffer. + std::vector m_buffer; + }; + + size_t m_avail_count; std::unique_ptr m_fh; + off_t m_offset{0}; + std::vector m_buffers; }; } diff --git a/src/tpc.cpp b/src/tpc.cpp index 59142035e44..3c2704cc849 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -401,7 +401,7 @@ int TPCHandler::ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) } curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - Stream stream(std::move(fh)); + Stream stream(std::move(fh), 0, 0); State state(0, stream, curl, true); state.CopyHeaders(req); @@ -452,7 +452,7 @@ int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) curl_easy_setopt(curl, CURLOPT_CAPATH, m_cadir.c_str()); } curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - Stream stream(std::move(fh)); + Stream stream(std::move(fh), 0, 0); State state(0, stream, curl, false); state.CopyHeaders(req); From 0a9adbc941264e7a492cc943c20e0bcc9abf82d4 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Fri, 5 Jan 2018 13:36:06 -0600 Subject: [PATCH 20/29] Add support for multi-streamed HTTP transfers. --- CMakeLists.txt | 2 +- src/multistream.cpp | 333 ++++++++++++++++++++++++++++++++++++++++++ src/state.cpp | 75 +++++++++- src/state.hh | 27 +++- src/stream.cpp | 15 +- src/stream.hh | 10 +- src/tpc.cpp | 43 +++--- src/tpc.hh | 8 +- tools/xrootd-test-tpc | 8 + 9 files changed, 489 insertions(+), 32 deletions(-) create mode 100644 src/multistream.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f249a3bb4dd..5c60269dba7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ pkg_check_modules(CURL REQUIRED libcurl) include_directories(${XROOTD_INCLUDES} ${XROOTD_PRIVATE_INCLUDES} ${CURL_INCLUDE_DIRS}) -add_library(XrdHttpTPC SHARED src/tpc.cpp src/state.cpp src/configure.cpp src/stream.cpp) +add_library(XrdHttpTPC SHARED src/tpc.cpp src/state.cpp src/configure.cpp src/stream.cpp src/multistream.cpp) if ( XRD_CHUNK_RESP ) set_target_properties(XrdHttpTPC PROPERTIES COMPILE_DEFINITIONS "XRD_CHUNK_RESP" ) endif () diff --git a/src/multistream.cpp b/src/multistream.cpp new file mode 100644 index 00000000000..2779d143f19 --- /dev/null +++ b/src/multistream.cpp @@ -0,0 +1,333 @@ +/** + * Implementation of multi-stream HTTP transfers for the TPCHandler + */ + +#ifdef XRD_CHUNK_RESP + +#include "tpc.hh" +#include "state.hh" + +#include "XrdSys/XrdSysError.hh" + +#include + +#include +#include + +using namespace TPC; + +class CurlHandlerSetupError : public std::runtime_error { +public: + CurlHandlerSetupError(const std::string &msg) : + std::runtime_error(msg) + {} + virtual ~CurlHandlerSetupError() {} +}; + +namespace { +class MultiCurlHandler { +public: + MultiCurlHandler(std::vector &states) : + m_handle(curl_multi_init()), + m_states(states) + { + if (m_handle == nullptr) { + throw CurlHandlerSetupError("Failed to initialize a libcurl multi-handle"); + } + m_avail_handles.reserve(states.size()); + m_active_handles.reserve(states.size()); + for (State &state : states) { + m_avail_handles.push_back(state.GetHandle()); + } + } + + ~MultiCurlHandler() + { + if (!m_handle) {return;} + for (CURL * easy_handle : m_active_handles) { + curl_multi_remove_handle(m_handle, easy_handle); + curl_easy_cleanup(easy_handle); + } + for (auto & easy_handle : m_avail_handles) { + curl_easy_cleanup(easy_handle); + } + curl_multi_cleanup(m_handle); + } + + MultiCurlHandler(const MultiCurlHandler &) = delete; + + CURLM *Get() const {return m_handle;} + + void FinishCurlXfer(CURL *curl) { + CURLMcode mres = curl_multi_remove_handle(m_handle, curl); + if (mres) { + std::stringstream ss; + ss << "Failed to remove transfer from set: " + << curl_multi_strerror(mres); + throw std::runtime_error(ss.str()); + } + for (auto &state : m_states) { + if (curl == state.GetHandle()) { + state.ResetAfterRequest(); + break; + } + } + for (auto iter = m_active_handles.begin(); + iter != m_active_handles.end(); + ++iter) + { + if (*iter == curl) { + m_active_handles.erase(iter); + break; + } + } + m_avail_handles.push_back(curl); + } + + off_t StartTransfers(off_t current_offset, off_t content_length, size_t block_size, + int &running_handles) { + bool started_new_xfer = false; + do { + size_t xfer_size = std::min(content_length - current_offset, static_cast(block_size)); + if (xfer_size == 0) {return current_offset;} + if (!(started_new_xfer = StartTransfer(current_offset, xfer_size))) { + break; + } else { + running_handles += 1; + } + current_offset += xfer_size; + } while (true); + return current_offset; + } + +private: + + bool StartTransfer(off_t offset, size_t size) { + if (!CanStartTransfer()) {return false;} + for (auto &handle : m_avail_handles) { + for (auto &state : m_states) { + if (state.GetHandle() == handle) { // This state object represents an idle handle. + state.SetTransferParameters(offset, size); + ActivateHandle(state); + return true; + } + } + } + return false; + } + + void ActivateHandle(State &state) { + CURL *curl = state.GetHandle(); + m_active_handles.push_back(curl); + CURLMcode mres; + mres = curl_multi_add_handle(m_handle, curl); + if (mres) { + std::stringstream ss; + ss << "Failed to add transfer to libcurl multi-handle" + << curl_multi_strerror(mres); + throw std::runtime_error(ss.str()); + } + for (auto iter = m_avail_handles.begin(); + iter != m_avail_handles.end(); + ++iter) + { + if (*iter == curl) { + m_avail_handles.erase(iter); + break; + } + } + } + + bool CanStartTransfer() const { + size_t idle_handles = m_avail_handles.size(); + size_t transfer_in_progress = 0; + for (auto &state : m_states) { + for (const auto &handle : m_active_handles) { + if (handle == state.GetHandle()) { + transfer_in_progress += state.BodyTransferInProgress(); + break; + } + } + } + if (!idle_handles) { + return false; + } + ssize_t available_buffers = m_states[0].AvailableBuffers(); + // To be conservative, set aside buffers for any transfers that have been activated + // but don't have their first responses back yet. + available_buffers -= (m_active_handles.size() - transfer_in_progress); + return available_buffers > 0; + } + + CURLM *m_handle; + std::vector m_avail_handles; + std::vector m_active_handles; + std::vector &m_states; +}; +} + + +int TPCHandler::RunCurlWithStreams(XrdHttpExtReq &req, State &state, + const char *log_prefix, size_t streams) +try +{ + int result; + bool success; + CURL *curl = state.GetHandle(); + if ((result = DetermineXferSize(curl, req, state, success)) || !success) { + return result; + } + off_t content_size = state.GetContentLength(); + off_t current_offset = 0; + + { + std::stringstream ss; + ss << "Successfully determined remote size for pull request: " << content_size; + m_log.Emsg("ProcessPullReq", ss.str().c_str()); + } + state.ResetAfterRequest(); + + std::vector handles; + handles.reserve(streams); + handles.emplace_back(std::move(state)); + for (size_t idx = 1; idx < streams; idx++) { + handles.emplace_back(handles[0].Duplicate()); // Makes a duplicate of the original state + } + + // Create the multi-handle and add in the current transfer to it. + MultiCurlHandler mch(handles); + CURLM *multi_handle = mch.Get(); + + // Start response to client prior to the first call to curl_multi_perform + int retval = req.StartChunkedResp(201, "Created", "Content-Type: text/plain"); + if (retval) { + return retval; + } + + // Start assigning transfers + int running_handles = 0; + current_offset = mch.StartTransfers(current_offset, content_size, m_block_size, running_handles); + + // Transfer loop: use curl to actually run the transfer, but periodically + // interrupt things to send back performance updates to the client. + time_t last_marker = 0; + CURLcode res = static_cast(-1); + CURLMcode mres; + do { + time_t now = time(NULL); + time_t next_marker = last_marker + m_marker_period; + if (now >= next_marker) { + if (SendPerfMarker(req, current_offset)) { + return -1; + } + last_marker = now; + } + + mres = curl_multi_perform(multi_handle, &running_handles); + if (mres == CURLM_CALL_MULTI_PERFORM) { + // curl_multi_perform should be called again immediately. On newer + // versions of curl, this is no longer used. + continue; + } else if (mres != CURLM_OK) { + break; + } + + // Harvest any messages, looking for CURLMSG_DONE. + CURLMsg *msg; + do { + int msgq = 0; + msg = curl_multi_info_read(multi_handle, &msgq); + if (msg && (msg->msg == CURLMSG_DONE)) { + CURL *easy_handle = msg->easy_handle; + mch.FinishCurlXfer(easy_handle); + res = msg->data.result; + // If any requests fail, cut off the entire transfer. + if (res != CURLE_OK) { + break; + } + } + } while (msg); + if (res != -1 && res != CURLE_OK) { + break; + } + + if (running_handles < static_cast(streams)) { + // Issue new transfers if there is still pending work to do. + // Otherwise, continue running until there are no handles left. + if (current_offset != content_size) { + current_offset = mch.StartTransfers(current_offset, content_size, + m_block_size, running_handles); + } else if (running_handles == 0) { + break; + } + } + + int64_t max_sleep_time = next_marker - time(NULL); + if (max_sleep_time <= 0) { + continue; + } + int fd_count; + mres = curl_multi_wait(multi_handle, NULL, 0, max_sleep_time*1000, &fd_count); + if (mres != CURLM_OK) { + break; + } + } while (running_handles); + + if (mres != CURLM_OK) { + std::stringstream ss; + ss << "Internal libcurl multi-handle error: " + << curl_multi_strerror(mres); + throw std::runtime_error(ss.str()); + } + + // Harvest any messages, looking for CURLMSG_DONE. + CURLMsg *msg; + do { + int msgq = 0; + msg = curl_multi_info_read(multi_handle, &msgq); + if (msg && (msg->msg == CURLMSG_DONE)) { + CURL *easy_handle = msg->easy_handle; + mch.FinishCurlXfer(easy_handle); + res = msg->data.result; // Transfer result will be examined below. + } + } while (msg); + + if (res == -1) { // No transfers returned?!? + throw std::runtime_error("Internal state error in libcurl"); + } + + // Generate the final response back to the client. + std::stringstream ss; + if (res != CURLE_OK) { + m_log.Emsg(log_prefix, "request failed when processing", curl_easy_strerror(res)); + ss << "failure: " << curl_easy_strerror(res); + } else if (current_offset != content_size) { + ss << "failure: Internal logic error led to early abort"; + m_log.Emsg(log_prefix, "Internal logic error led to early abort"); + } else if (state.GetStatusCode() >= 400) { + ss << "failure: Remote side failed with status code " << state.GetStatusCode(); + m_log.Emsg(log_prefix, "Remote server failed request", ss.str().c_str()); + } else { + ss << "success: Created"; + } + + if ((retval = req.ChunkResp(ss.str().c_str(), 0))) { + return retval; + } + return req.ChunkResp(nullptr, 0); +} +catch (CurlHandlerSetupError e) { + m_log.Emsg(log_prefix, e.what()); + return req.SendSimpleResp(500, nullptr, nullptr, e.what(), 0); +} catch (std::runtime_error e) { + m_log.Emsg(log_prefix, e.what()); + std::stringstream ss; + ss << "failure: " << e.what(); + int retval; + if ((retval = req.ChunkResp(ss.str().c_str(), 0))) { + return retval; + } + return req.ChunkResp(nullptr, 0); +} + +#endif // XRD_CHUNK_RESP diff --git a/src/state.cpp b/src/state.cpp index faa6abb67c1..ace8a9eaa12 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -1,6 +1,7 @@ #include #include +#include #include "XrdHttp/XrdHttpExtHandler.hh" #include "XrdSfs/XrdSfsInterface.hh" @@ -17,10 +18,35 @@ State::~State() { if (m_headers) { curl_slist_free_all(m_headers); m_headers = nullptr; - curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, m_headers); + if (m_curl) {curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, m_headers);} } } +State::State(State && other) noexcept : + m_push(other.m_push), + m_recv_status_line(other.m_recv_status_line), + m_recv_all_headers(other.m_recv_all_headers), + m_offset(other.m_offset), + m_start_offset(other.m_start_offset), + m_status_code(other.m_status_code), + m_content_length(other.m_content_length), + m_stream(other.m_stream), + m_curl(other.m_curl), + m_headers(other.m_headers), + m_headers_copy(std::move(other.m_headers_copy)), + m_resp_protocol(std::move(m_resp_protocol)) +{ + curl_easy_setopt(m_curl, CURLOPT_HEADERDATA, this); + if (m_push) { + curl_easy_setopt(m_curl, CURLOPT_READDATA, this); + } else { + curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, this); + } + other.m_headers_copy.clear(); + other.m_curl = nullptr; + other.m_headers = nullptr; +} + bool State::InstallHandlers(CURL *curl) { curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/" XRDTPC_VERSION); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &State::HeaderCB); @@ -45,16 +71,18 @@ bool State::InstallHandlers(CURL *curl) { * Handle the 'Copy-Headers' feature */ void State::CopyHeaders(XrdHttpExtReq &req) { - struct curl_slist *list = NULL; + struct curl_slist *list = nullptr; for (auto &hdr : req.headers) { if (hdr.first == "Copy-Header") { list = curl_slist_append(list, hdr.second.c_str()); + m_headers_copy.emplace_back(hdr.second); } // Note: len("TransferHeader") == 14 if (!hdr.first.compare(0, 14, "TransferHeader")) { std::stringstream ss; ss << hdr.first.substr(14) << ": " << hdr.second; list = curl_slist_append(list, ss.str().c_str()); + m_headers_copy.emplace_back(ss.str()); } } if (list != nullptr) { @@ -63,7 +91,8 @@ void State::CopyHeaders(XrdHttpExtReq &req) { } } -void State::ResetAfterSize() { +void State::ResetAfterRequest() { + m_offset = 0; m_status_code = -1; m_content_length = -1; m_recv_all_headers = false; @@ -128,7 +157,9 @@ int State::Header(const std::string &header) { size_t State::WriteCB(void *buffer, size_t size, size_t nitems, void *userdata) { State *obj = static_cast(userdata); - if (obj->GetStatusCode() < 0) {return 0;} // malformed request - got body before headers. + if (obj->GetStatusCode() < 0) { + return 0; + } // malformed request - got body before headers. if (obj->GetStatusCode() >= 400) {return 0;} // Status indicates failure. return obj->Write(static_cast(buffer), size*nitems); } @@ -139,7 +170,6 @@ int State::Write(char *buffer, size_t size) { return -1; } m_offset += retval; - //printf("Wrote a total of %ld bytes.\n", m_offset); return retval; } @@ -159,3 +189,38 @@ int State::Read(char *buffer, size_t size) { //printf("Read a total of %ld bytes.\n", m_offset); return retval; } + +State State::Duplicate() { + CURL *curl = curl_easy_duphandle(m_curl); + if (!curl) { + throw std::runtime_error("Failed to duplicate existing curl handle."); + } + + State state(0, m_stream, curl, m_push); + + if (m_headers) { + state.m_headers_copy.reserve(m_headers_copy.size()); + for (auto &header : m_headers_copy) { + state.m_headers = curl_slist_append(state.m_headers, header.c_str()); + state.m_headers_copy.push_back(header); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, nullptr); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, state.m_headers); + } + + return std::move(state); +} + +void State::SetTransferParameters(off_t offset, size_t size) { + m_start_offset = offset; + m_offset = 0; + m_content_length = size; + std::stringstream ss; + ss << offset << "-" << (offset+size-1); + curl_easy_setopt(m_curl, CURLOPT_RANGE, ss.str().c_str()); +} + +int State::AvailableBuffers() const +{ + return m_stream.AvailableBuffers(); +} diff --git a/src/state.hh b/src/state.hh index 01f46b88168..5d52ec39cdd 100644 --- a/src/state.hh +++ b/src/state.hh @@ -5,6 +5,7 @@ */ #include +#include // Forward dec'ls class XrdSfsFile; @@ -16,6 +17,10 @@ class Stream; class State { public: + + // Note that we are "borrowing" a reference to the curl handle; + // it is not owned / freed by the State object. However, we use it + // as if there's only one handle per State. State (off_t start_offset, Stream &stream, CURL *curl, bool push) : m_push(push), m_start_offset(start_offset), @@ -27,6 +32,8 @@ public: ~State(); + void SetTransferParameters(off_t offset, size_t size); + void CopyHeaders(XrdHttpExtReq &req); off_t BytesTransferred() const {return m_offset;} @@ -35,7 +42,22 @@ public: int GetStatusCode() const {return m_status_code;} - void ResetAfterSize(); + void ResetAfterRequest(); + + CURL *GetHandle() const {return m_curl;} + + int AvailableBuffers() const; + + // Returns true if at least one byte of the response has been received, + // but not the entire contents of the response. + bool BodyTransferInProgress() const {return m_offset && (m_offset != m_content_length);} + + // Duplicate the current state; all settings are copied over, but those + // related to the transient state are reset as if from a constructor. + State Duplicate(); + + State(const State&) = delete; + State(State &&) noexcept; private: bool InstallHandlers(CURL *curl); @@ -53,12 +75,13 @@ private: bool m_recv_status_line{false}; // whether we have received a status line in the response from the remote host. bool m_recv_all_headers{false}; // true if we have seen the end of headers. off_t m_offset{0}; // number of bytes we have received. - const off_t m_start_offset{0}; // offset where we started in the file. + off_t m_start_offset{0}; // offset where we started in the file. int m_status_code{-1}; // status code from HTTP response. off_t m_content_length{-1}; // value of Content-Length header, if we received one. Stream &m_stream; // stream corresponding to this transfer. CURL *m_curl{nullptr}; // libcurl handle struct curl_slist *m_headers{nullptr}; // any headers we set as part of the libcurl request. + std::vector m_headers_copy; // Copies of custom headers. std::string m_resp_protocol; // Response protocol in the HTTP status line. }; diff --git a/src/stream.cpp b/src/stream.cpp index 6deef39685e..b2e55304330 100644 --- a/src/stream.cpp +++ b/src/stream.cpp @@ -22,6 +22,9 @@ Stream::Write(off_t offset, const char *buf, size_t size) { bool buffer_accepted = false; int retval = size; + if (offset < m_offset) { + return SFS_ERROR; + } if (offset == m_offset) { retval = m_fh->write(offset, buf, size); buffer_accepted = true; @@ -34,14 +37,15 @@ Stream::Write(off_t offset, const char *buf, size_t size) return retval; } } - //printf("Performing stream buffer accounting. Available buffers: %lu, total buffers %lu.\n", m_avail_count, m_buffers.size()); // Even if we already accepted the current data, always // iterate through available buffers and try to write as // much out to disk as possible. - Entry *avail_entry = nullptr; + Entry *avail_entry; bool buffer_was_written; + size_t avail_count = 0; do { - m_avail_count = 0; + avail_count = 0; + avail_entry = nullptr; buffer_was_written = false; for (Entry &entry : m_buffers) { // Always try to dump from memory. @@ -50,13 +54,14 @@ Stream::Write(off_t offset, const char *buf, size_t size) } if (entry.Available()) { // Empty buffer if (!avail_entry) {avail_entry = &entry;} - m_avail_count ++; + avail_count ++; } else if (!buffer_accepted && entry.Accept(offset, buf, size)) { buffer_accepted = true; } } - } while ((m_avail_count != m_buffers.size()) && buffer_was_written); + } while ((avail_count != m_buffers.size()) && buffer_was_written); + m_avail_count = avail_count; if (!buffer_accepted) { // No place for this data in allocated buffers if (!avail_entry) { // No available buffers to allocate. diff --git a/src/stream.hh b/src/stream.hh index 78d6bef028f..f123ec42301 100644 --- a/src/stream.hh +++ b/src/stream.hh @@ -23,7 +23,7 @@ public: : m_avail_count(max_blocks), m_fh(std::move(fh)) { - m_buffers.reserve(max_blocks); + //m_buffers.reserve(max_blocks); for (size_t idx=0; idx < max_blocks; idx++) { m_buffers.emplace_back(buffer_size); } @@ -37,6 +37,8 @@ public: int Write(off_t offset, const char *buffer, size_t size); + size_t AvailableBuffers() const {return m_avail_count;} + private: class Entry { @@ -45,6 +47,9 @@ private: m_capacity(capacity) {} + Entry(const Entry&) = delete; + Entry(Entry&&) = default; + bool Available() const {return m_offset == -1;} int Write(Stream &stream) { @@ -78,6 +83,9 @@ private: // Finally, do the copy. memcpy(&m_buffer[0] + m_size, buf, size); m_size += size; + if (m_offset == -1) { + m_offset = offset; + } return true; } diff --git a/src/tpc.cpp b/src/tpc.cpp index 3c2704cc849..e2b4a92478b 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -189,14 +189,13 @@ int TPCHandler::DetermineXferSize(CURL *curl, XrdHttpExtReq &req, State &state, return 0; } -int TPCHandler::SendPerfMarker(XrdHttpExtReq &req, State &state) { +int TPCHandler::SendPerfMarker(XrdHttpExtReq &req, off_t bytes_transferred) { std::stringstream ss; const std::string crlf = "\n"; ss << "Perf Marker" << crlf; ss << "Timestamp: " << time(NULL) << crlf; ss << "Stripe Index: 0" << crlf; - ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; - ss << "Stripe Bytes Transferred: " << state.BytesTransferred() << crlf; + ss << "Stripe Bytes Transferred: " << bytes_transferred << crlf; ss << "Total Stripe Count: 1" << crlf; ss << "End" << crlf; @@ -243,7 +242,7 @@ int TPCHandler::RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, State &state, time_t now = time(NULL); time_t next_marker = last_marker + m_marker_period; if (now >= next_marker) { - if (SendPerfMarker(req, state)) { + if (SendPerfMarker(req, state.BytesTransferred())) { curl_multi_remove_handle(multi_handle, curl); curl_easy_cleanup(curl); curl_multi_cleanup(multi_handle); @@ -426,10 +425,26 @@ int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) } std::string authz = GetAuthz(req); XrdSfsFileOpenMode mode = SFS_O_CREAT; - /*auto overwrite_header = req.headers.find("Overwrite"); + auto overwrite_header = req.headers.find("Overwrite"); if ((overwrite_header == req.headers.end()) || (overwrite_header->second == "T")) { mode = SFS_O_TRUNC|SFS_O_POSC; - }*/ + } + int streams = 1; + { + auto streams_header = req.headers.find("X-Number-Of-Streams"); + if (streams_header != req.headers.end()) { + int stream_req = -1; + try { + stream_req = std::stol(streams_header->second); + } catch (...) { // Handled below + } + if (stream_req < 1 || stream_req > 100) { + char msg[] = "Invalid request for number of streams"; + return req.SendSimpleResp(500, nullptr, nullptr, msg, 0); + } + streams = stream_req; + } + } int open_result = OpenWaitStall(*fh, req.resource, mode|SFS_O_WRONLY, 0644, req.GetSecEntity(), authz); @@ -452,22 +467,16 @@ int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) curl_easy_setopt(curl, CURLOPT_CAPATH, m_cadir.c_str()); } curl_easy_setopt(curl, CURLOPT_URL, resource.c_str()); - Stream stream(std::move(fh), 0, 0); + Stream stream(std::move(fh), streams, m_block_size); State state(0, stream, curl, false); state.CopyHeaders(req); #ifdef XRD_CHUNK_RESP - int result; - bool success; - if ((result = DetermineXferSize(curl, req, state, success)) || !success) { - return result; + if (streams > 1) { + return RunCurlWithStreams(req, state, "ProcessPullReq", streams); + } else { + return RunCurlWithUpdates(curl, req, state, "ProcessPullReq"); } - std::stringstream ss; - ss << "Successfully determined remote size for pull request: " << state.GetContentLength(); - m_log.Emsg("ProcessPullReq", ss.str().c_str()); - state.ResetAfterSize(); - - return RunCurlWithUpdates(curl, req, state, "ProcessPullReq"); #else return RunCurlBasic(curl, req, state, "ProcessPullReq"); #endif diff --git a/src/tpc.hh b/src/tpc.hh index 163ba77c956..cfdd9896301 100644 --- a/src/tpc.hh +++ b/src/tpc.hh @@ -39,10 +39,15 @@ private: int DetermineXferSize(CURL *curl, XrdHttpExtReq &req, TPC::State &state, bool &success); - int SendPerfMarker(XrdHttpExtReq &req, TPC::State &state); + int SendPerfMarker(XrdHttpExtReq &req, off_t bytes_transferred); + // Perform the libcurl transfer, periodically sending back chunked updates. int RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, TPC::State &state, const char *log_prefix); + + // Experimental multi-stream version of RunCurlWithUpdates + int RunCurlWithStreams(XrdHttpExtReq &req, TPC::State &state, + const char *log_prefix, size_t streams); #else int RunCurlBasic(CURL *curl, XrdHttpExtReq &req, TPC::State &state, const char *log_prefix); @@ -56,6 +61,7 @@ private: bool Configure(const char *configfn, XrdOucEnv *myEnv); static constexpr int m_marker_period = 5; + static constexpr size_t m_block_size = 16*1024*1024; bool m_desthttps{false}; std::string m_cadir; static std::atomic m_monid; diff --git a/tools/xrootd-test-tpc b/tools/xrootd-test-tpc index 851618ab390..2508444a786 100755 --- a/tools/xrootd-test-tpc +++ b/tools/xrootd-test-tpc @@ -14,11 +14,17 @@ def parse_args(): parser.add_argument("--push", dest="mode", action="store_const", const="push", help="Use push-mode for transfer (source manages transfer)", default="auto") parser.add_argument("--pull", dest="mode", action="store_const", const="pull", help="Use pull-mode for transfer (destination manages transfer)") parser.add_argument("--no-overwrite", dest="overwrite", action="store_false", default=True, help="Disable overwrite of existing files.") + parser.add_argument("--streams", dest="streams", help="Allow multiple streams", default=1, + type=int) parser.add_argument("src") parser.add_argument("dest") args = parser.parse_args() + if args.streams < 1: + print >> sys.stderr, "Invalid number of streams specified: %d" % args.streams + sys.exit(1) + if not args.token and (not args.src_token or not args.dest_token): if 'SCITOKEN' in os.environ and os.path.exists(os.environ['SCITOKEN']): args.token = os.environ['SCITOKEN'] @@ -63,6 +69,8 @@ def main(): headers['Overwrite'] = 'T' else: headers['Overwrite'] = 'F' + if args.streams > 1: + headers['X-Number-Of-Streams'] = str(args.streams) mode = args.mode if mode == "auto": mode = determine_mode(args) From c3c1913d01a15c0270284975173c6ec6e3f7825f Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Fri, 5 Jan 2018 21:59:25 -0600 Subject: [PATCH 21/29] Convert davs:// URLs to HTTPS. --- src/tpc.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/tpc.cpp b/src/tpc.cpp index e2b4a92478b..7d40ed4ae14 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -72,13 +72,20 @@ bool TPCHandler::MatchesPath(const char *verb, const char *path) { return !strcmp(verb, "COPY") || !strcmp(verb, "OPTIONS"); } +static std::string PrepareURL(const std::string &input) { + if (!strncmp(input.c_str(), "davs://", 7)) { + return "https://" + input.substr(7); + } + return input; +} + int TPCHandler::ProcessReq(XrdHttpExtReq &req) { if (req.verb == "OPTIONS") { return ProcessOptionsReq(req); } auto header = req.headers.find("Source"); if (header != req.headers.end()) { - return ProcessPullReq(header->second, req); + return ProcessPullReq(PrepareURL(header->second), req); } header = req.headers.find("Destination"); if (header != req.headers.end()) { From fb536d920daae119b48b8d5c7979efd55f108513 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 8 Feb 2018 08:55:41 -0600 Subject: [PATCH 22/29] Fix copy/paste error in option parsing. --- src/configure.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/configure.cpp b/src/configure.cpp index d1976118896..cf9915cdac6 100644 --- a/src/configure.cpp +++ b/src/configure.cpp @@ -120,11 +120,11 @@ bool TPCHandler::Configure(const char *configfn, XrdOucEnv *myEnv) } if (!strcmp("1", val) || !strcasecmp("yes", val) || !strcasecmp("true", val)) { m_desthttps = true; - } else if (!strcmp("1", val) || !strcasecmp("yes", val) || !strcasecmp("true", val)) { + } else if (!strcmp("0", val) || !strcasecmp("no", val) || !strcasecmp("false", val)) { m_desthttps = false; } else { Config.Close(); - m_log.Emsg("Config", "https.dests value is invalid", val); + m_log.Emsg("Config", "https.desthttps value is invalid", val); return false; } } else if (!strcmp("http.cadir", val)) { From 85916d7418d25afb90a99e38017758f30d5e1773 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 19 Mar 2018 16:45:46 -0500 Subject: [PATCH 23/29] Debugging from transfers with UCSD. - Do not use POSC as this requires some fchmod support we don't have. - Log the URL utilized for pull requests. --- src/tpc.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tpc.cpp b/src/tpc.cpp index 7d40ed4ae14..8c210158df2 100644 --- a/src/tpc.cpp +++ b/src/tpc.cpp @@ -85,7 +85,9 @@ int TPCHandler::ProcessReq(XrdHttpExtReq &req) { } auto header = req.headers.find("Source"); if (header != req.headers.end()) { - return ProcessPullReq(PrepareURL(header->second), req); + std::string src = PrepareURL(header->second); + m_log.Emsg("ProcessReq", "Pull request from", src.c_str()); + return ProcessPullReq(src, req); } header = req.headers.find("Destination"); if (header != req.headers.end()) { @@ -434,7 +436,7 @@ int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) XrdSfsFileOpenMode mode = SFS_O_CREAT; auto overwrite_header = req.headers.find("Overwrite"); if ((overwrite_header == req.headers.end()) || (overwrite_header->second == "T")) { - mode = SFS_O_TRUNC|SFS_O_POSC; + mode = SFS_O_TRUNC; } int streams = 1; { From 40d211c94b4c73a9c4f903475ba41666cfb8d57c Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Mon, 19 Mar 2018 16:48:30 -0500 Subject: [PATCH 24/29] Bump TPC version for 0.3.4 release. --- rpm/xrootd-tpc.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rpm/xrootd-tpc.spec b/rpm/xrootd-tpc.spec index 459ff8cc961..ac763066940 100644 --- a/rpm/xrootd-tpc.spec +++ b/rpm/xrootd-tpc.spec @@ -1,6 +1,6 @@ Name: xrootd-tpc -Version: 0.3.3 +Version: 0.3.4 Release: 1%{?dist} Summary: HTTP Third Party Copy plugin for XRootD @@ -43,6 +43,9 @@ rm -rf $RPM_BUILD_ROOT %{_libdir}/libXrdHttpTPC-4.so %changelog +* Mon Mar 19 2018 Brian Bockelman - 0.3.4-1 +- Fix overwrites. + * Tue Jan 02 2018 Brian Bockelman - 0.3.3-1 - Remove workaround from bad version of libdavix. From 6e8729a2c1de94e032a87f1ba2626b666d186a11 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Tue, 27 Mar 2018 10:52:04 -0500 Subject: [PATCH 25/29] Timeout slow transfers. --- src/state.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/state.cpp b/src/state.cpp index ace8a9eaa12..7069aa2f0fe 100644 --- a/src/state.cpp +++ b/src/state.cpp @@ -64,6 +64,11 @@ bool State::InstallHandlers(CURL *curl) { curl_easy_setopt(curl, CURLOPT_WRITEDATA, this); } curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + + // Require a minimum speed from the transfer: must move at least 1MB every 2 minutes + // (roughly 8KB/s). + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 2*60); + curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1024*1024); return true; } From 07ee38e09164d14404865035cd1617dc96d2d682 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Tue, 27 Mar 2018 10:52:58 -0500 Subject: [PATCH 26/29] Bump release version for slow transfer timeouts. --- rpm/xrootd-tpc.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rpm/xrootd-tpc.spec b/rpm/xrootd-tpc.spec index ac763066940..1ba42d37cdb 100644 --- a/rpm/xrootd-tpc.spec +++ b/rpm/xrootd-tpc.spec @@ -1,6 +1,6 @@ Name: xrootd-tpc -Version: 0.3.4 +Version: 0.4.0 Release: 1%{?dist} Summary: HTTP Third Party Copy plugin for XRootD @@ -43,6 +43,9 @@ rm -rf $RPM_BUILD_ROOT %{_libdir}/libXrdHttpTPC-4.so %changelog +* Tue Mar 27 2018 Brian Bockelman - 0.4.0-1 +- Add timeout for slow transfers. + * Mon Mar 19 2018 Brian Bockelman - 0.3.4-1 - Fix overwrites. From a78764c8340a7d4cb756f20dbed4d10adc5fe880 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 29 Mar 2018 03:01:25 -0500 Subject: [PATCH 27/29] Re-arrange directory structure to fit inside XRootD source code. --- LICENSE | 201 -------------------- cmake/FindXrootd.cmake | 51 ----- rpm/xrootd-tpc.spec | 63 ------ CMakeLists.txt => src/XrdTpc/CMakeLists.txt | 0 README.md => src/XrdTpc/README.md | 0 src/{ => XrdTpc}/configure.cpp | 0 {configs => src/XrdTpc}/export-lib-symbols | 0 src/{ => XrdTpc}/multistream.cpp | 0 src/{ => XrdTpc}/state.cpp | 0 src/{ => XrdTpc}/state.hh | 0 src/{ => XrdTpc}/stream.cpp | 0 src/{ => XrdTpc}/stream.hh | 0 src/{ => XrdTpc}/tpc.cpp | 0 src/{ => XrdTpc}/tpc.hh | 0 {tools => src/XrdTpc}/xrootd-test-tpc | 0 src/XrdTpcVersion.hh | 3 - 16 files changed, 318 deletions(-) delete mode 100644 LICENSE delete mode 100644 cmake/FindXrootd.cmake delete mode 100644 rpm/xrootd-tpc.spec rename CMakeLists.txt => src/XrdTpc/CMakeLists.txt (100%) rename README.md => src/XrdTpc/README.md (100%) rename src/{ => XrdTpc}/configure.cpp (100%) rename {configs => src/XrdTpc}/export-lib-symbols (100%) rename src/{ => XrdTpc}/multistream.cpp (100%) rename src/{ => XrdTpc}/state.cpp (100%) rename src/{ => XrdTpc}/state.hh (100%) rename src/{ => XrdTpc}/stream.cpp (100%) rename src/{ => XrdTpc}/stream.hh (100%) rename src/{ => XrdTpc}/tpc.cpp (100%) rename src/{ => XrdTpc}/tpc.hh (100%) rename {tools => src/XrdTpc}/xrootd-test-tpc (100%) delete mode 100644 src/XrdTpcVersion.hh diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 8dada3edaf5..00000000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/cmake/FindXrootd.cmake b/cmake/FindXrootd.cmake deleted file mode 100644 index ad951ba90c2..00000000000 --- a/cmake/FindXrootd.cmake +++ /dev/null @@ -1,51 +0,0 @@ - -FIND_PATH(XROOTD_INCLUDES XrdVersion.hh - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd/ - PATH_SUFFIXES include/xrootd - PATHS /opt/xrootd -) - -FIND_PATH(XROOTD_PRIVATE_INCLUDES XrdHttp/XrdHttpExtHandler.hh - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd/ - PATH_SUFFIXES include/xrootd/private - PATHS /opt/xrootd -) - -FIND_LIBRARY(XROOTD_UTILS_LIB XrdUtils - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd/ - PATH_SUFFIXES lib -) - -FIND_LIBRARY(XROOTD_SERVER_LIB XrdServer - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd/ - PATH_SUFFIXES lib -) - -FIND_LIBRARY(XROOTD_HTTP_LIB XrdHttp-4 - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd/ - PATH_SUFFIXES lib -) - -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(Xrootd DEFAULT_MSG XROOTD_UTILS_LIB XROOTD_HTTP_LIB XROOTD_INCLUDES XROOTD_PRIVATE_INCLUDES) - diff --git a/rpm/xrootd-tpc.spec b/rpm/xrootd-tpc.spec deleted file mode 100644 index 1ba42d37cdb..00000000000 --- a/rpm/xrootd-tpc.spec +++ /dev/null @@ -1,63 +0,0 @@ - -Name: xrootd-tpc -Version: 0.4.0 -Release: 1%{?dist} -Summary: HTTP Third Party Copy plugin for XRootD - -Group: System Environment/Daemons -License: BSD -URL: https://github.com/bbockelm/xrootd-tpc -# Generated from: -# git archive v%{version} --prefix=%{name}-%{version}/ | gzip -7 > ~/rpmbuild/SOURCES/%{name}-%{version}.tar.gz -Source0: %{name}-%{version}.tar.gz -BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) -BuildRequires: xrootd-devel -BuildRequires: xrootd-server-libs -BuildRequires: xrootd-server-devel -BuildRequires: xrootd-private-devel -BuildRequires: cmake -BuildRequires: gcc-c++ -BuildRequires: libcurl-devel - -%description -%{summary} - -%prep -%setup -q - -sed -i 's|".*"|"%{version}"|' src/XrdTpcVersion.hh - -%build -%cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo . -make VERBOSE=1 %{?_smp_mflags} - -%install -rm -rf $RPM_BUILD_ROOT -make install DESTDIR=$RPM_BUILD_ROOT - -%clean -rm -rf $RPM_BUILD_ROOT - -%files -%defattr(-,root,root,-) -%{_libdir}/libXrdHttpTPC-4.so - -%changelog -* Tue Mar 27 2018 Brian Bockelman - 0.4.0-1 -- Add timeout for slow transfers. - -* Mon Mar 19 2018 Brian Bockelman - 0.3.4-1 -- Fix overwrites. - -* Tue Jan 02 2018 Brian Bockelman - 0.3.3-1 -- Remove workaround from bad version of libdavix. - -* Fri Dec 29 2017 Brian Bockelman - 0.3.2-1 -- Allow CA directory to be overridden in Xrootd. - -* Fri Dec 29 2017 Brian Bockelman - 0.3.1-1 -- Add support for dCache-style TransferHeader. - -* Mon Nov 13 2017 Brian Bockelman - 0.3-1 -- Add support for redirections in COPY requests. -- Add RPM packaging. diff --git a/CMakeLists.txt b/src/XrdTpc/CMakeLists.txt similarity index 100% rename from CMakeLists.txt rename to src/XrdTpc/CMakeLists.txt diff --git a/README.md b/src/XrdTpc/README.md similarity index 100% rename from README.md rename to src/XrdTpc/README.md diff --git a/src/configure.cpp b/src/XrdTpc/configure.cpp similarity index 100% rename from src/configure.cpp rename to src/XrdTpc/configure.cpp diff --git a/configs/export-lib-symbols b/src/XrdTpc/export-lib-symbols similarity index 100% rename from configs/export-lib-symbols rename to src/XrdTpc/export-lib-symbols diff --git a/src/multistream.cpp b/src/XrdTpc/multistream.cpp similarity index 100% rename from src/multistream.cpp rename to src/XrdTpc/multistream.cpp diff --git a/src/state.cpp b/src/XrdTpc/state.cpp similarity index 100% rename from src/state.cpp rename to src/XrdTpc/state.cpp diff --git a/src/state.hh b/src/XrdTpc/state.hh similarity index 100% rename from src/state.hh rename to src/XrdTpc/state.hh diff --git a/src/stream.cpp b/src/XrdTpc/stream.cpp similarity index 100% rename from src/stream.cpp rename to src/XrdTpc/stream.cpp diff --git a/src/stream.hh b/src/XrdTpc/stream.hh similarity index 100% rename from src/stream.hh rename to src/XrdTpc/stream.hh diff --git a/src/tpc.cpp b/src/XrdTpc/tpc.cpp similarity index 100% rename from src/tpc.cpp rename to src/XrdTpc/tpc.cpp diff --git a/src/tpc.hh b/src/XrdTpc/tpc.hh similarity index 100% rename from src/tpc.hh rename to src/XrdTpc/tpc.hh diff --git a/tools/xrootd-test-tpc b/src/XrdTpc/xrootd-test-tpc similarity index 100% rename from tools/xrootd-test-tpc rename to src/XrdTpc/xrootd-test-tpc diff --git a/src/XrdTpcVersion.hh b/src/XrdTpcVersion.hh deleted file mode 100644 index 8f792c5138c..00000000000 --- a/src/XrdTpcVersion.hh +++ /dev/null @@ -1,3 +0,0 @@ - -#define XRDTPC_VERSION "devel" - From 9c587d041310201ba7c8628ee3d9c1f8e5e77a82 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 29 Mar 2018 04:12:34 -0500 Subject: [PATCH 28/29] Integrate xrootd-tpc build with xrootd itself. --- cmake/XRootDFindLibs.cmake | 9 +++++++++ src/CMakeLists.txt | 1 + src/XrdHttp.cmake | 5 ++++- src/XrdTpc/state.cpp | 4 ++-- src/XrdTpc/tpc.cpp | 1 - 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/cmake/XRootDFindLibs.cmake b/cmake/XRootDFindLibs.cmake index b1e23550446..d07063aad52 100644 --- a/cmake/XRootDFindLibs.cmake +++ b/cmake/XRootDFindLibs.cmake @@ -27,6 +27,9 @@ if( SYSTEMD_FOUND ) add_definitions( -DHAVE_SYSTEMD ) endif() +include (FindPkgConfig) +pkg_check_modules(CURL libcurl) + if( ENABLE_CRYPTO ) find_package( OpenSSL ) if( OPENSSL_FOUND ) @@ -70,8 +73,14 @@ endif() if( ENABLE_HTTP ) if( OPENSSL_FOUND AND BUILD_CRYPTO ) set( BUILD_HTTP TRUE ) + if( CURL_FOUND ) + set( BUILD_TPC TRUE ) + else() + set( BUILD_TPC FALSE ) + endif() else() set( BUILD_HTTP FALSE ) + set( BUILD_TPC FALSE ) endif() endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 60cb227d377..eafa85da0bc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -38,6 +38,7 @@ include( XrdFileCache ) if( BUILD_HTTP ) include( XrdHttp ) + include( XrdTpc ) endif() if( BUILD_CEPH ) diff --git a/src/XrdHttp.cmake b/src/XrdHttp.cmake index e9f2befb220..843f1c0bc8c 100644 --- a/src/XrdHttp.cmake +++ b/src/XrdHttp.cmake @@ -15,9 +15,11 @@ if( BUILD_HTTP ) #----------------------------------------------------------------------------- include_directories( ${OPENSSL_INCLUDE_DIR} ) + # Note this is marked as a shared library as XrdHttp plugins are expected to + # link against this for the XrdHttpExt class implementations. add_library( ${LIB_XRD_HTTP} - MODULE + SHARED XrdHttp/XrdHttpProtocol.cc XrdHttp/XrdHttpProtocol.hh XrdHttp/XrdHttpReq.cc XrdHttp/XrdHttpReq.hh XrdHttp/XrdHttpSecXtractor.hh @@ -40,6 +42,7 @@ if( BUILD_HTTP ) ${LIB_XRD_HTTP} PROPERTIES INTERFACE_LINK_LIBRARIES "" + SUFFIX ".so" LINK_INTERFACE_LIBRARIES "" ) #----------------------------------------------------------------------------- diff --git a/src/XrdTpc/state.cpp b/src/XrdTpc/state.cpp index 7069aa2f0fe..c2c234ebf55 100644 --- a/src/XrdTpc/state.cpp +++ b/src/XrdTpc/state.cpp @@ -3,12 +3,12 @@ #include #include +#include "XrdVersion.hh" #include "XrdHttp/XrdHttpExtHandler.hh" #include "XrdSfs/XrdSfsInterface.hh" #include -#include "XrdTpcVersion.hh" #include "state.hh" #include "stream.hh" @@ -48,7 +48,7 @@ State::State(State && other) noexcept : } bool State::InstallHandlers(CURL *curl) { - curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/" XRDTPC_VERSION); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "xrootd-tpc/" XrdVERSION); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &State::HeaderCB); curl_easy_setopt(curl, CURLOPT_HEADERDATA, this); if (m_push) { diff --git a/src/XrdTpc/tpc.cpp b/src/XrdTpc/tpc.cpp index 8c210158df2..d83fa8b1967 100644 --- a/src/XrdTpc/tpc.cpp +++ b/src/XrdTpc/tpc.cpp @@ -15,7 +15,6 @@ #include #include -#include "XrdTpcVersion.hh" #include "state.hh" #include "stream.hh" #include "tpc.hh" From 507f665058df90e169307ba4502de775b7f92e34 Mon Sep 17 00:00:00 2001 From: Brian Bockelman Date: Thu, 29 Mar 2018 04:46:27 -0500 Subject: [PATCH 29/29] Add forgotten cmake file. --- src/XrdTpc.cmake | 51 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/XrdTpc.cmake diff --git a/src/XrdTpc.cmake b/src/XrdTpc.cmake new file mode 100644 index 00000000000..0b83958a0a2 --- /dev/null +++ b/src/XrdTpc.cmake @@ -0,0 +1,51 @@ +include( XRootDCommon ) + +#------------------------------------------------------------------------------- +# Modules +#------------------------------------------------------------------------------- +set( LIB_XRD_TPC XrdHttpTPC-${PLUGIN_VERSION} ) + +#------------------------------------------------------------------------------- +# Shared library version +#------------------------------------------------------------------------------- + +if( BUILD_TPC ) + #----------------------------------------------------------------------------- + # The XrdHttp library + #----------------------------------------------------------------------------- + include_directories( ${CURL_INCLUDE_DIRS} ) + + add_library( + ${LIB_XRD_TPC} + MODULE + XrdTpc/configure.cpp + XrdTpc/multistream.cpp + XrdTpc/state.cpp XrdTpc/state.hh + XrdTpc/stream.cpp XrdTpc/stream.hh + XrdTpc/tpc.cpp XrdTpc/tpc.hh) + + target_link_libraries( + ${LIB_XRD_TPC} + XrdServer + XrdUtils + XrdHttp-${PLUGIN_VERSION} + dl + pthread + ${CURL_LIBRARIES} ) + + set_target_properties( + ${LIB_XRD_TPC} + PROPERTIES + INTERFACE_LINK_LIBRARIES "" + LINK_INTERFACE_LIBRARIES "" + LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/XrdTpc/export-lib-symbols" + COMPILE_DEFINITIONS "XRD_CHUNK_RESP") + + #----------------------------------------------------------------------------- + # Install + #----------------------------------------------------------------------------- + install( + TARGETS ${LIB_XRD_TPC} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) + +endif()