From 0477d4fdd67240474720dafa4bc271a5b45cf791 Mon Sep 17 00:00:00 2001 From: Michal Simon Date: Wed, 15 Jun 2016 14:27:11 +0200 Subject: [PATCH] [XrdCl] Support local metalink files --- docs/man/xrdcp.1 | 12 +++++- src/XrdCl/XrdClConstants.hh | 1 + src/XrdCl/XrdClDefaultEnv.cc | 1 + src/XrdCl/XrdClMetalinkRedirector.cc | 59 ++++++++++++++++++++++++++-- src/XrdCl/XrdClMetalinkRedirector.hh | 16 +++++++- src/XrdCl/XrdClRedirectorRegistry.cc | 8 ++-- src/XrdCl/XrdClRedirectorRegistry.hh | 4 +- 7 files changed, 90 insertions(+), 11 deletions(-) diff --git a/docs/man/xrdcp.1 b/docs/man/xrdcp.1 index 1e0d4a86dd7..d804392b2c6 100644 --- a/docs/man/xrdcp.1 +++ b/docs/man/xrdcp.1 @@ -186,7 +186,7 @@ equivalent to "\fB--sources 12\fR". .SH OPERANDS \fIsource\fR .RS 5 -a local file, a local directory name suffixed by /, or +a dash (i.e. \fB-\fR) indicating stanard in, a local file, a local directory name suffixed by /, or an xrootd URL in the form of .ce 1 \fBxroot://[\fIuser\fB@\fR]\fIhost[\fB:\fIport\fR]\fB/\fIabsolutepath\fR @@ -426,6 +426,16 @@ Number of unacknowledged probes before considering the connection dead (Linux only) .RE +XRD_METALINKPROCESSING +.RS 5 +Enable/Disable Metalink processing (enabled by default) +.RE + +XRD_LOCALMETALINKFILE +.RS 5 +Enable/Disable local Metalink file processing (by convention the following URL schema has to be used: root://localfile//path/filename.meta4) +.RE + .SH NOTES Documentation for all components associated with \fBxrdcp\fR can be found at http://xrootd.org/docs.html diff --git a/src/XrdCl/XrdClConstants.hh b/src/XrdCl/XrdClConstants.hh index c30239ab6b5..c91e75bc1ad 100644 --- a/src/XrdCl/XrdClConstants.hh +++ b/src/XrdCl/XrdClConstants.hh @@ -65,6 +65,7 @@ namespace XrdCl const int DefaultMultiProtocol = 0; const int DefaultParallelEvtLoop = 1; const int DefaultMetalinkProcessing = 1; + const int DefaultLocalMetalinkFile = 1; const char * const DefaultPollerPreference = "built-in"; const char * const DefaultNetworkStack = "IPAuto"; diff --git a/src/XrdCl/XrdClDefaultEnv.cc b/src/XrdCl/XrdClDefaultEnv.cc index 0128a53f801..cc840011397 100644 --- a/src/XrdCl/XrdClDefaultEnv.cc +++ b/src/XrdCl/XrdClDefaultEnv.cc @@ -202,6 +202,7 @@ namespace XrdCl REGISTER_VAR_INT( varsInt, "MultiProtocol", DefaultMultiProtocol ); REGISTER_VAR_INT( varsInt, "ParallelEvtLoop", DefaultParallelEvtLoop ); REGISTER_VAR_INT( varsInt, "MetalinkProcessing", DefaultMetalinkProcessing ); + REGISTER_VAR_INT( varsInt, "LocalMetalinkFile", DefaultLocalMetalinkFile ); REGISTER_VAR_STR( varsStr, "PollerPreference", DefaultPollerPreference ); REGISTER_VAR_STR( varsStr, "ClientMonitor", DefaultClientMonitor ); diff --git a/src/XrdCl/XrdClMetalinkRedirector.cc b/src/XrdCl/XrdClMetalinkRedirector.cc index 193d96a50d2..cd609446e3d 100644 --- a/src/XrdCl/XrdClMetalinkRedirector.cc +++ b/src/XrdCl/XrdClMetalinkRedirector.cc @@ -22,6 +22,7 @@ #include "XProtocol/XProtocol.hh" #include +#include namespace XrdCl { @@ -189,6 +190,8 @@ class MetalinkOpenHandler : public ResponseHandler ResponseHandler *pUserHandler; }; +const std::string MetalinkRedirector::LocalFile = "localfile"; + //---------------------------------------------------------------------------- // Constructor //---------------------------------------------------------------------------- @@ -208,18 +211,66 @@ MetalinkRedirector::~MetalinkRedirector() //---------------------------------------------------------------------------- XRootDStatus MetalinkRedirector::Load( ResponseHandler *userHandler ) { - MetalinkOpenHandler * handler = new MetalinkOpenHandler( this, userHandler ); + if( IsLocalFile( pUrl ) ) + return LoadLocalFile( userHandler ); + MetalinkOpenHandler *handler = new MetalinkOpenHandler( this, userHandler ); XRootDStatus st = pFile->Open( pUrl, OpenFlags::Read, Access::None, handler, 0, false ); - if( !st.IsOK() ) - { delete handler; - } return st; } +//---------------------------------------------------------------------------- +// Loads a local Metalink file +//---------------------------------------------------------------------------- +XRootDStatus MetalinkRedirector::LoadLocalFile( ResponseHandler *userHandler ) +{ + URL u( pUrl ); + int fd = open( u.GetPath().c_str(), O_RDONLY ); + if( fd == -1 ) + { + if( userHandler ) userHandler->HandleResponseWithHosts( new XRootDStatus( stError, errOSError, errno ), 0, 0 ); + return XRootDStatus( stError, errOSError, errno ); + } + const uint32_t toRead = DefaultCPChunkSize; + char *buffer = new char[toRead]; + std::string content; + int64_t bytesRead = 0; + while( ( bytesRead = read( fd, buffer, toRead ) ) ) + { + if( bytesRead < 0 ) + { + close( fd ); + delete buffer; + if( userHandler ) userHandler->HandleResponseWithHosts( new XRootDStatus( stError, errOSError, errno ), 0, 0 ); + return XRootDStatus( stError, errOSError, errno ); + } + content += std::string( buffer, bytesRead ); + } + close( fd ); + delete buffer; + XRootDStatus st = Parse( content ); + FinalizeInitialization( st ); + if( userHandler ) userHandler->HandleResponseWithHosts( new XRootDStatus( st ), 0, 0 ); + return st; +} + +//---------------------------------------------------------------------------- +// Checks if the given URL points to a local file +// (by convention we assume that a file is local +// if the host name equals to 'localfile') +//---------------------------------------------------------------------------- +bool MetalinkRedirector::IsLocalFile( const std::string &url ) +{ + Env *env = DefaultEnv::GetEnv(); + int mlLocalFile = DefaultLocalMetalinkFile; + env->GetInt( "LocalMetalinkFile", mlLocalFile ); + URL u( url ); + return ( u.GetHostName() == LocalFile ) && mlLocalFile && u.IsMetalink(); +} + //---------------------------------------------------------------------------- // Parses the metalink file //---------------------------------------------------------------------------- diff --git a/src/XrdCl/XrdClMetalinkRedirector.hh b/src/XrdCl/XrdClMetalinkRedirector.hh index 383e8f19c07..da51c906915 100644 --- a/src/XrdCl/XrdClMetalinkRedirector.hh +++ b/src/XrdCl/XrdClMetalinkRedirector.hh @@ -125,10 +125,22 @@ class MetalinkRedirector : public VirtualRedirector XRootDStatus GetReplica( const Message *msg, std::string &replica ) const; //---------------------------------------------------------------------------- - //! Extracts an element from url cgi + //! Extracts an element from URL cgi //---------------------------------------------------------------------------- XRootDStatus GetCgiInfo( const Message *msg, const std::string &key, std::string &out ) const; + //---------------------------------------------------------------------------- + //! Loads a local Metalink file + //---------------------------------------------------------------------------- + XRootDStatus LoadLocalFile( ResponseHandler *userHandler ); + + //---------------------------------------------------------------------------- + //! Checks if the given URL points to a local file + //! (by convention we assume that a file is local + //! if the host name equals to 'localfile') + //---------------------------------------------------------------------------- + static bool IsLocalFile( const std::string &url ); + typedef std::list< std::pair > RedirectList; typedef std::map CksumMap; @@ -146,6 +158,8 @@ class MetalinkRedirector : public VirtualRedirector XrdSysMutex pMutex; + static const std::string LocalFile; + }; } /* namespace XrdCl */ diff --git a/src/XrdCl/XrdClRedirectorRegistry.cc b/src/XrdCl/XrdClRedirectorRegistry.cc index 8f1d1a697f2..ad6e5e5ab46 100644 --- a/src/XrdCl/XrdClRedirectorRegistry.cc +++ b/src/XrdCl/XrdClRedirectorRegistry.cc @@ -22,7 +22,7 @@ RedirectorRegistry& RedirectorRegistry::Instance() RedirectorRegistry::~RedirectorRegistry() { - std::map< std::string, std::pair >::iterator itr; + RedirectorMap::iterator itr; for( itr = pRegistry.begin(); itr != pRegistry.end(); ++itr ) delete itr->second.first; } @@ -35,7 +35,7 @@ XRootDStatus RedirectorRegistry::RegisterImpl( const URL &url, ResponseHandler * XrdSysMutexHelper scopedLock( pMutex ); // get the key and check if it is already in the registry const std::string key = url.GetLocation(); - std::map< std::string, std::pair >::iterator itr = pRegistry.find( key ); + RedirectorMap::iterator itr = pRegistry.find( key ); if( itr != pRegistry.end() ) { // increment user counter @@ -78,7 +78,7 @@ VirtualRedirector* RedirectorRegistry::Get( const URL &url ) const // get the key and return the value if it is in the registry // offset 24 is where the path has been stored const std::string key = url.GetLocation(); - std::map< std::string, std::pair< VirtualRedirector*, size_t> >::const_iterator itr = pRegistry.find( key ); + RedirectorMap::const_iterator itr = pRegistry.find( key ); if( itr != pRegistry.end() ) return itr->second.first; // otherwise return null @@ -94,7 +94,7 @@ void RedirectorRegistry::Release( const URL &url ) // get the key and return the value if it is in the registry // offset 24 is where the path has been stored const std::string key = url.GetLocation(); - std::map< std::string, std::pair< VirtualRedirector*, size_t> >::iterator itr = pRegistry.find( key ); + RedirectorMap::iterator itr = pRegistry.find( key ); if( itr == pRegistry.end() ) return; // decrement user counter --itr->second.second; diff --git a/src/XrdCl/XrdClRedirectorRegistry.hh b/src/XrdCl/XrdClRedirectorRegistry.hh index 2a5ccb24421..417521cdef6 100644 --- a/src/XrdCl/XrdClRedirectorRegistry.hh +++ b/src/XrdCl/XrdClRedirectorRegistry.hh @@ -102,6 +102,8 @@ class RedirectorRegistry private: + typedef std::map< std::string, std::pair > RedirectorMap; + //---------------------------------------------------------------------------- //! Register implementation. //---------------------------------------------------------------------------- @@ -122,7 +124,7 @@ class RedirectorRegistry //---------------------------------------------------------------------------- RedirectorRegistry& operator=( const RedirectorRegistry & ); - std::map< std::string, std::pair > pRegistry; + RedirectorMap pRegistry; mutable XrdSysMutex pMutex; };