Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Revert "Revert "Merge pull request #679 from bbockelm/xrootd-tpc-rena…
…me"" This reverts commit d46c516. Reverting-the-revert to start working on cleanup of this branch for C++03 support and eventual re-merge.
- Loading branch information
Showing
16 changed files
with
1,980 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
include( XRootDCommon ) | ||
|
||
#------------------------------------------------------------------------------- | ||
# Modules | ||
#------------------------------------------------------------------------------- | ||
set( LIB_XRD_TPC XrdHttpTPC-${PLUGIN_VERSION} ) | ||
|
||
#------------------------------------------------------------------------------- | ||
# Shared library version | ||
#------------------------------------------------------------------------------- | ||
|
||
if( BUILD_TPC ) | ||
#----------------------------------------------------------------------------- | ||
# The XrdHttp library | ||
#----------------------------------------------------------------------------- | ||
include_directories( ${CURL_INCLUDE_DIRS} ) | ||
|
||
add_library( | ||
${LIB_XRD_TPC} | ||
MODULE | ||
XrdTpc/configure.cpp | ||
XrdTpc/multistream.cpp | ||
XrdTpc/state.cpp XrdTpc/state.hh | ||
XrdTpc/stream.cpp XrdTpc/stream.hh | ||
XrdTpc/tpc.cpp XrdTpc/tpc.hh) | ||
|
||
target_link_libraries( | ||
${LIB_XRD_TPC} | ||
XrdServer | ||
XrdUtils | ||
XrdHttp-${PLUGIN_VERSION} | ||
dl | ||
pthread | ||
${CURL_LIBRARIES} ) | ||
|
||
set_target_properties( | ||
${LIB_XRD_TPC} | ||
PROPERTIES | ||
INTERFACE_LINK_LIBRARIES "" | ||
LINK_INTERFACE_LIBRARIES "" | ||
LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/src/XrdTpc/export-lib-symbols" | ||
COMPILE_DEFINITIONS "XRD_CHUNK_RESP") | ||
|
||
#----------------------------------------------------------------------------- | ||
# Install | ||
#----------------------------------------------------------------------------- | ||
install( | ||
TARGETS ${LIB_XRD_TPC} | ||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) | ||
|
||
endif() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
|
||
cmake_minimum_required( VERSION 2.8 ) | ||
project( xrootd-tpc ) | ||
|
||
set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ) | ||
|
||
find_package( Xrootd REQUIRED ) | ||
|
||
macro(use_cxx11) | ||
if (CMAKE_VERSION VERSION_LESS "3.1") | ||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") | ||
set (CMAKE_CXX_FLAGS "-std=gnu++11 ${CMAKE_CXX_FLAGS}") | ||
endif () | ||
else () | ||
set (CMAKE_CXX_STANDARD 11) | ||
endif () | ||
endmacro(use_cxx11) | ||
use_cxx11() | ||
|
||
# Chunked responses were introduced after Xrootd 4.8.0. Check to see if the symbol exists. | ||
#include (CheckCXXSymbolExists) | ||
SET( CMAKE_REQUIRED_INCLUDES "${XROOTD_PRIVATE_INCLUDES}" ) | ||
SET( CMAKE_REQUIRED_LIBRARIES "${XROOTD_HTTP_LIB}" ) | ||
SET( CMAKE_REQUIRED_FLAGS "" ) | ||
include (CheckCXXSourceCompiles) | ||
CHECK_CXX_SOURCE_COMPILES("#include <XrdHttp/XrdHttpExtHandler.hh> | ||
int main(int argc, char** argv) | ||
{ | ||
(void)argv; | ||
return ((int*)reinterpret_cast<void*>(&XrdHttpExtReq::ChunkResp))[argc]; | ||
} | ||
" XRD_CHUNK_RESP) | ||
|
||
if( CMAKE_COMPILER_IS_GNUCXX ) | ||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror" ) | ||
endif() | ||
SET( CMAKE_SHARED_LINKER_FLAGS "-Wl,--no-undefined") | ||
SET( CMAKE_MODULE_LINKER_FLAGS "-Wl,--no-undefined") | ||
|
||
include (FindPkgConfig) | ||
pkg_check_modules(CURL REQUIRED libcurl) | ||
|
||
include_directories(${XROOTD_INCLUDES} ${XROOTD_PRIVATE_INCLUDES} ${CURL_INCLUDE_DIRS}) | ||
|
||
add_library(XrdHttpTPC SHARED src/tpc.cpp src/state.cpp src/configure.cpp src/stream.cpp src/multistream.cpp) | ||
if ( XRD_CHUNK_RESP ) | ||
set_target_properties(XrdHttpTPC PROPERTIES COMPILE_DEFINITIONS "XRD_CHUNK_RESP" ) | ||
endif () | ||
|
||
target_link_libraries(XrdHttpTPC -ldl ${XROOTD_UTILS_LIB} ${XROOTD_SERVER_LIB} ${XROOTD_HTTP_LIB} ${CURL_LIBRARIES}) | ||
set_target_properties(XrdHttpTPC PROPERTIES OUTPUT_NAME "XrdHttpTPC-4" SUFFIX ".so" LINK_FLAGS "-Wl,--version-script=${CMAKE_SOURCE_DIR}/configs/export-lib-symbols") | ||
|
||
SET(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib" CACHE PATH "Install path for libraries") | ||
|
||
install( | ||
TARGETS XrdHttpTPC | ||
LIBRARY DESTINATION ${LIB_INSTALL_DIR} | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,121 @@ | ||
# HTTPS Third Party Copy for XRootD | ||
|
||
The `xrootd-tpc` module provides an implementation of HTTPS third-party-copy | ||
for the HTTPS implementation inside XRootD. | ||
|
||
To enable, set the following in the configuration file: | ||
|
||
``` | ||
http.exthandler xrdtpc libXrdHttpTPC.so | ||
``` | ||
|
||
|
||
## HTTPS TPC technical details. | ||
|
||
Third-party-copy is done using the standard WebDAV copy verb. The actors involved are: | ||
|
||
1. The client orchestrating the transfer. | ||
2. The source server where the resource is read from. | ||
3. The destination server where the resource is written to. | ||
|
||
The client initiates the third party copy by issuing a COPY request to _either_ the source or destination | ||
endpoint. Typically, when this is done, the initial endpoint redirects the client to a second service | ||
(the disk server) that will performs the actual transfer. | ||
|
||
When the client contacts the source server, it should set the `Destination` header so the source knows | ||
where to put the data. Analogously, if the client first contacts the destination server, it should set | ||
the `Source` header. | ||
|
||
For the former case, the interaction looks like this: | ||
|
||
``` | ||
-> COPY /sfn/of/source/replica HTTP/1.1 | ||
Destination: https://<dest-endpoint>/pfn/of/dest/replica | ||
<- HTTP/1.1 302 Found | ||
Location: https://<source-disk-server>/pfn/of/source/replica | ||
``` | ||
|
||
where `<source-disk-server>` is the service that will actually perform the transfer. | ||
|
||
The client would follow the redirection and retry the `COPY`: | ||
|
||
``` | ||
-> COPY /sfn/of/source/replica HTTP/1.1 | ||
Destination: https://<dest-endpoint>/pfn/of/dest/replica | ||
<- HTTP/1.1 201 Created | ||
Transfer-Encoding: chunked | ||
``` | ||
|
||
The source server *should* respond with an appropriate status code (such as 201) in a timely manner. | ||
As a TPC can take a significant amount of time, the source server SHOULD NOT wait until the transfer is | ||
finished before responding with a success. In the case when a transfer can be started, it is recommended | ||
that the response be started as soon as possible. | ||
|
||
In the case of a transfer being started (or queued) by the source server, it SHOULD use chunked encoding | ||
and send a multipart response. | ||
|
||
Next `<source-disk-server>` will connect to the destination endpoint and actually perform the `PUT`: | ||
|
||
``` | ||
-> PUT /pfn/of/destination/replica HTTP/1.1 | ||
<- HTTP/1.1 201 Created | ||
``` | ||
|
||
As the PUT is ongoing, the source disk server should send back a periodic transfer chunk of the following | ||
form: | ||
|
||
``` | ||
Perf Marker | ||
Timestamp: 1360578938 | ||
Stripe Index: 0 | ||
Stripe Bytes Transferred: 49397760 | ||
Total Stripe Count: 1 | ||
End | ||
``` | ||
|
||
Timestamps should be seconds from Unix epoch. It is recommended that the time period between chunks be | ||
less than 30 seconds. | ||
|
||
If the transfer ultimately succeeds, then the last chunk should be of the following form: | ||
|
||
``` | ||
success: Created | ||
``` | ||
|
||
Here, `success` indicates the transfer status: subsequent text is informational for the user. | ||
|
||
Failure of the transfer can be indicated with the prefix `failed` or `failure`: | ||
|
||
``` | ||
failure: Network connection unexpectedly closed. | ||
``` | ||
|
||
Finally, if the source disk server decides to cancel or abort the transfer, use the `aborted` prefix: | ||
|
||
``` | ||
aborted: Transfer took too long. | ||
``` | ||
|
||
There is an analogous case when the client contacts the destination server to perform the `COPY` and | ||
sets the `Source` header. In such a case, the response to the client looks identical but the destination | ||
server will perform a `GET` instead of a `PUT`. | ||
|
||
In some cases, the user may want the server performing the transfer to append additional headers | ||
(such as an HTTP `Authorization` header) to the transfer it initiates. In such case, the client should | ||
utilize the `TransferHeader` mechanism. Add a header of the form: | ||
|
||
``` | ||
TransferHeader<header> : <value> | ||
``` | ||
|
||
For example, if the client sends the following to the source server as part of its `COPY` request: | ||
|
||
``` | ||
TransferHeaderAuthorization: Basic cGF1bDpwYXNzd29yZA== | ||
``` | ||
|
||
then the source server should set the following header as part of its `PUT` request to the destination: | ||
|
||
``` | ||
Authorization: Basic cGF1bDpwYXNzd29yZA== | ||
``` |
Oops, something went wrong.