Third-party-copy plugin for XrdHttp

cmake/XRootDFindLibs.cmake

@@ -27,6 +27,9 @@ if( SYSTEMD_FOUND )
   add_definitions( -DHAVE_SYSTEMD )
 endif()
 
+include (FindPkgConfig)
+pkg_check_modules(CURL libcurl)
+
 if( ENABLE_CRYPTO )
   find_package( OpenSSL )
   if( OPENSSL_FOUND )
@@ -70,8 +73,14 @@ endif()
 if( ENABLE_HTTP )
   if( OPENSSL_FOUND AND BUILD_CRYPTO )
     set( BUILD_HTTP TRUE )
+    if( CURL_FOUND )
+      set( BUILD_TPC TRUE )
+    else()
+      set( BUILD_TPC FALSE )
+    endif()
   else()
     set( BUILD_HTTP FALSE )
+    set( BUILD_TPC FALSE )
   endif()
 endif()
 

src/CMakeLists.txt

@@ -38,6 +38,7 @@ include( XrdFileCache )
 
 if( BUILD_HTTP )
   include( XrdHttp )
+  include( XrdTpc )
 endif()
 
 if( BUILD_CEPH )

src/XrdHttp.cmake

@@ -15,9 +15,11 @@ if( BUILD_HTTP )
   #-----------------------------------------------------------------------------
   include_directories( ${OPENSSL_INCLUDE_DIR} )
 
+  # Note this is marked as a shared library as XrdHttp plugins are expected to
+  # link against this for the XrdHttpExt class implementations.
   add_library(
     ${LIB_XRD_HTTP}
-    MODULE
+    SHARED
     XrdHttp/XrdHttpProtocol.cc    XrdHttp/XrdHttpProtocol.hh
     XrdHttp/XrdHttpReq.cc         XrdHttp/XrdHttpReq.hh
                                   XrdHttp/XrdHttpSecXtractor.hh
@@ -40,6 +42,7 @@ if( BUILD_HTTP )
     ${LIB_XRD_HTTP}
     PROPERTIES
     INTERFACE_LINK_LIBRARIES ""
+    SUFFIX ".so"
     LINK_INTERFACE_LIBRARIES "" )
 
   #-----------------------------------------------------------------------------

src/XrdTpc.cmake

@@ -0,0 +1,51 @@
+include( XRootDCommon )
+
+#-------------------------------------------------------------------------------
+# Modules
+#-------------------------------------------------------------------------------
+set( LIB_XRD_TPC       XrdHttpTPC-${PLUGIN_VERSION} )
+
+#-------------------------------------------------------------------------------
+# Shared library version
+#-------------------------------------------------------------------------------
+
+if( BUILD_TPC )
+  #-----------------------------------------------------------------------------
+  # The XrdHttp library
+  #-----------------------------------------------------------------------------
+  include_directories( ${CURL_INCLUDE_DIRS} )
+
+  add_library(
+    ${LIB_XRD_TPC}
+    MODULE
+    XrdTpc/configure.cpp
+    XrdTpc/multistream.cpp
+    XrdTpc/state.cpp         XrdTpc/state.hh
+    XrdTpc/stream.cpp        XrdTpc/stream.hh
+    XrdTpc/tpc.cpp           XrdTpc/tpc.hh)
+
+  target_link_libraries(
+    ${LIB_XRD_TPC}
+    XrdServer
+    XrdUtils
+    XrdHttp-${PLUGIN_VERSION}
+    dl
+    pthread
+    ${CURL_LIBRARIES} )
+
+  set_target_properties(
+    ${LIB_XRD_TPC}
+    PROPERTIES
+    INTERFACE_LINK_LIBRARIES ""
+    LINK_INTERFACE_LIBRARIES ""
+    LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/XrdTpc/export-lib-symbols"
+    COMPILE_DEFINITIONS "XRD_CHUNK_RESP")
+
+  #-----------------------------------------------------------------------------
+  # Install
+  #-----------------------------------------------------------------------------
+  install(
+    TARGETS ${LIB_XRD_TPC}
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} )
+
+endif()

src/XrdTpc/CMakeLists.txt

@@ -0,0 +1,59 @@
+
+cmake_minimum_required( VERSION 2.8 )
+project( xrootd-tpc )
+
+set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake )
+
+find_package( Xrootd REQUIRED )
+
+macro(use_cxx11)
+  if (CMAKE_VERSION VERSION_LESS "3.1")
+    if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+      set (CMAKE_CXX_FLAGS "-std=gnu++11 ${CMAKE_CXX_FLAGS}")
+    endif ()
+  else ()
+    set (CMAKE_CXX_STANDARD 11)
+  endif ()
+endmacro(use_cxx11)
+use_cxx11()
+
+# Chunked responses were introduced after Xrootd 4.8.0.  Check to see if the symbol exists.
+#include (CheckCXXSymbolExists)
+SET( CMAKE_REQUIRED_INCLUDES "${XROOTD_PRIVATE_INCLUDES}" )
+SET( CMAKE_REQUIRED_LIBRARIES "${XROOTD_HTTP_LIB}" )
+SET( CMAKE_REQUIRED_FLAGS "" )
+include (CheckCXXSourceCompiles)
+CHECK_CXX_SOURCE_COMPILES("#include <XrdHttp/XrdHttpExtHandler.hh>
+
+int main(int argc, char** argv)
+{
+  (void)argv;
+  return ((int*)reinterpret_cast<void*>(&XrdHttpExtReq::ChunkResp))[argc];
+}
+" XRD_CHUNK_RESP)
+
+if( CMAKE_COMPILER_IS_GNUCXX )
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror" )
+endif()
+SET( CMAKE_SHARED_LINKER_FLAGS "-Wl,--no-undefined")
+SET( CMAKE_MODULE_LINKER_FLAGS "-Wl,--no-undefined")
+
+include (FindPkgConfig)
+pkg_check_modules(CURL REQUIRED libcurl)
+
+include_directories(${XROOTD_INCLUDES} ${XROOTD_PRIVATE_INCLUDES} ${CURL_INCLUDE_DIRS})
+
+add_library(XrdHttpTPC SHARED src/tpc.cpp src/state.cpp src/configure.cpp src/stream.cpp src/multistream.cpp)
+if ( XRD_CHUNK_RESP )
+  set_target_properties(XrdHttpTPC PROPERTIES COMPILE_DEFINITIONS "XRD_CHUNK_RESP" )
+endif ()
+
+target_link_libraries(XrdHttpTPC -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${XROOTD_HTTP_LIB} ${CURL_LIBRARIES})
+set_target_properties(XrdHttpTPC PROPERTIES OUTPUT_NAME "XrdHttpTPC-4" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols")
+
+SET(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib" CACHE PATH "Install path for libraries")
+
+install(
+  TARGETS XrdHttpTPC
+  LIBRARY DESTINATION ${LIB_INSTALL_DIR}
+)

src/XrdTpc/README.md

@@ -0,0 +1,121 @@
+# HTTPS Third Party Copy for XRootD
+
+The `xrootd-tpc` module provides an implementation of HTTPS third-party-copy
+for the HTTPS implementation inside XRootD.
+
+To enable, set the following in the configuration file:
+
+```
+http.exthandler xrdtpc libXrdHttpTPC.so
+```
+
+
+## HTTPS TPC technical details.
+
+Third-party-copy is done using the standard WebDAV copy verb.  The actors involved are:
+
+1.  The client orchestrating the transfer.
+2.  The source server where the resource is read from.
+3.  The destination server where the resource is written to.
+
+The client initiates the third party copy by issuing a COPY request to _either_ the source or destination
+endpoint.  Typically, when this is done, the initial endpoint redirects the client to a second service
+(the disk server) that will performs the actual transfer.
+
+When the client contacts the source server, it should set the `Destination` header so the source knows
+where to put the data.  Analogously, if the client first contacts the destination server, it should set
+the `Source` header.
+
+For the former case, the interaction looks like this:
+
+```
+-> COPY /sfn/of/source/replica HTTP/1.1
+   Destination: https://<dest-endpoint>/pfn/of/dest/replica
+<- HTTP/1.1 302 Found
+   Location: https://<source-disk-server>/pfn/of/source/replica
+```
+
+where `<source-disk-server>` is the service that will actually perform the transfer.
+
+The client would follow the redirection and retry the `COPY`:
+
+```
+-> COPY /sfn/of/source/replica HTTP/1.1
+   Destination: https://<dest-endpoint>/pfn/of/dest/replica
+<- HTTP/1.1 201 Created
+   Transfer-Encoding: chunked
+```
+
+The source server *should* respond with an appropriate status code (such as 201) in a timely manner.
+As a TPC can take a significant amount of time, the source server SHOULD NOT wait until the transfer is
+finished before responding with a success.  In the case when a transfer can be started, it is recommended
+that the response be started as soon as possible.
+
+In the case of a transfer being started (or queued) by the source server, it SHOULD use chunked encoding
+and send a multipart response.
+
+Next `<source-disk-server>` will connect to the destination endpoint and actually perform the `PUT`:
+
+```
+-> PUT /pfn/of/destination/replica HTTP/1.1
+<- HTTP/1.1 201 Created
+```
+
+As the PUT is ongoing, the source disk server should send back a periodic transfer chunk of the following
+form:
+
+```
+Perf Marker
+	Timestamp: 1360578938
+	Stripe Index: 0
+	Stripe Bytes Transferred: 49397760
+	Total Stripe Count: 1
+End
+```
+
+Timestamps should be seconds from Unix epoch.  It is recommended that the time period between chunks be
+less than 30 seconds.
+
+If the transfer ultimately succeeds, then the last chunk should be of the following form:
+
+```
+success: Created
+```
+
+Here, `success` indicates the transfer status: subsequent text is informational for the user.
+
+Failure of the transfer can be indicated with the prefix `failed` or `failure`:
+
+```
+failure: Network connection unexpectedly closed.
+```
+
+Finally, if the source disk server decides to cancel or abort the transfer, use the `aborted` prefix:
+
+```
+aborted: Transfer took too long.
+```
+
+There is an analogous case when the client contacts the destination server to perform the `COPY` and
+sets the `Source` header.  In such a case, the response to the client looks identical but the destination
+server will perform a `GET` instead of a `PUT`.
+
+In some cases, the user may want the server performing the transfer to append additional headers
+(such as an HTTP `Authorization` header) to the transfer it initiates.  In such case, the client should
+utilize the `TransferHeader` mechanism.  Add a header of the form:
+
+```
+TransferHeader<header> : <value>
+```
+
+For example, if the client sends the following to the source server as part of its `COPY` request:
+
+```
+TransferHeaderAuthorization: Basic cGF1bDpwYXNzd29yZA==
+```
+
+then the source server should set the following header as part of its `PUT` request to the destination:
+
+```
+Authorization: Basic cGF1bDpwYXNzd29yZA==
+```