diff --git a/docs/man/xrdcp.1 b/docs/man/xrdcp.1 index 9f354bde41a..5a6acc9b090 100644 --- a/docs/man/xrdcp.1 +++ b/docs/man/xrdcp.1 @@ -12,7 +12,7 @@ xrdcp - copy files [\fB--recursive\fR] [\fB--retry\fR \fItime\fR] [\fB--server\fR] [\fB--silent\fR] [\fB--sources\fR \fInum\fR] [\fB--streams\fR \fInum\fR] [\fB--tpc\fR \fIfirst\fR|\fIonly\fR] [\fB--verbose\fR] [\fB--version\fR] -[\fB--xrate\fR \fIrate\fR] +[\fB--xrate\fR \fIrate\fR] [\fB--metalink\fR] \fIlegacy options\fR: [\fB-adler\fR] [\fB-DS\fR\fIparm string\fR] [\fB-DI\fR\fIparm number\fR] [\fB-md5\fR] [\fB-np\fR] [\fB-OD\fR\fIcgi\fR] [\fB-OS\fR\fIcgi\fR] [\fB-x\fR] @@ -133,6 +133,11 @@ displays version information and immediately exits. .RS 5 [NOT YET IMPLEMENTED] +.RE +\fB-M\fR | \fB--metalink\fR +.RS 5 +treats source as a metalink. + limits the copy speed to the specified \fIrate\fB. The rate may be qualified with the letter \fBk\fR, \fBm\fR, or \fBg\fR to indicate kilo, mega, or giga bytes, respectively. The option only applies when the source or destination is @@ -248,6 +253,15 @@ XRD_PARALLELEVTLOOP The number of event loops. .RE +XRD_READRECOVERY +.RS 5 +Determines if read recovery should be enabled or disabled (enabled by default). +.RE + +XRD_WRITERECOVERY +.RS 5 +Determines if write recovery should be enabled or disabled (enabled by default). +.RE XRD_CONNECTIONWINDOW (-DIConnectionWindow) .RS 5 diff --git a/src/XrdApps/XrdCpConfig.cc b/src/XrdApps/XrdCpConfig.cc index 4aa6fa69fe3..b02700a13d2 100644 --- a/src/XrdApps/XrdCpConfig.cc +++ b/src/XrdApps/XrdCpConfig.cc @@ -81,7 +81,7 @@ static XrdSysError eDest(&Logger, ""); XrdSysError *XrdCpConfig::Log = &XrdCpConfiguration::eDest; -const char *XrdCpConfig::opLetters = ":C:d:D:fFhHI:NpPrRsS:t:T:vVX:y:Z"; +const char *XrdCpConfig::opLetters = ":C:d:D:fFhHI:NpPrRsS:t:T:vVX:y:Z:M"; struct option XrdCpConfig::opVec[] = // For getopt_long() { @@ -108,6 +108,7 @@ struct option XrdCpConfig::opVec[] = // For getopt_long() {OPT_TYPE "version", 0, 0, XrdCpConfig::OpVersion}, {OPT_TYPE "xrate", 1, 0, XrdCpConfig::OpXrate}, {OPT_TYPE "parallel", 1, 0, XrdCpConfig::OpParallel}, + {OPT_TYPE "metalink", 0, 0, XrdCpConfig::OpMetalink}, {0, 0, 0, 0} }; @@ -221,6 +222,8 @@ do{while(optind < Argc && Legacy(optind)) {} break; case OpForce: OpSpec |= DoForce; break; + case OpMetalink: OpSpec |= DoMetalink; + break; case OpHelp: Usage(0); break; case OpIfile: if (inFile) free(inFile); @@ -288,7 +291,7 @@ do{while(optind < Argc && Legacy(optind)) {} // if (inFile) {if (!parmCnt ) UMSG("Destination not specified.");} else { if (!parmCnt ) UMSG("No files specified."); - if ( parmCnt == 1) UMSG("Destination not specified."); + if ( parmCnt == 1 && !( OpSpec & DoMetalink ) ) UMSG("Destination not specified."); } // Check for conflicts wit third party copy @@ -307,26 +310,36 @@ do{while(optind < Argc && Legacy(optind)) {} // if (getenv("XRD_MAKEPATH")) OpSpec |= DoPath; + if( parmCnt > 1 ) + { // Process the destination first as it is special // - dstFile = new XrdCpFile(parmVal[--parmCnt], rc); - if (rc) FMSG("Invalid url, '" <Path <<"'.", 22); + dstFile = new XrdCpFile(parmVal[--parmCnt], rc); + if (rc) FMSG("Invalid url, '" <Path <<"'.", 22); // Do a protocol check // - if (dstFile->Protocol != XrdCpFile::isFile - && dstFile->Protocol != XrdCpFile::isStdIO - && dstFile->Protocol != XrdCpFile::isXroot) - {FMSG(dstFile->ProtName <<"file protocol is not supported.", 22)} + if (dstFile->Protocol != XrdCpFile::isFile + && dstFile->Protocol != XrdCpFile::isStdIO + && dstFile->Protocol != XrdCpFile::isXroot) + {FMSG(dstFile->ProtName <<"file protocol is not supported.", 22)} // Resolve this file if it is a local file // - isLcl = (dstFile->Protocol == XrdCpFile::isFile) - | (dstFile->Protocol == XrdCpFile::isStdIO); - if (isLcl && (rc = dstFile->Resolve())) - {if (rc != ENOENT || (Argc - optind - 1) > 1 || OpSpec & DoRecurse) - FMSG(strerror(rc) <<" processing " <Path, 2); - } + isLcl = (dstFile->Protocol == XrdCpFile::isFile) + | (dstFile->Protocol == XrdCpFile::isStdIO); + if (isLcl && (rc = dstFile->Resolve())) + {if (rc != ENOENT || (Argc - optind - 1) > 1 || OpSpec & DoRecurse) + FMSG(strerror(rc) <<" processing " <Path, 2); + } + } + else + { +// Create an empty destination file +// + dstFile = new XrdCpFile(); + dstFile->Path = strdup( "" ); + } // Now pick up all the source files from the command line // @@ -353,7 +366,7 @@ do{while(optind < Argc && Legacy(optind)) {} // Check if we have an appropriate destination // - if (dstFile->Protocol == XrdCpFile::isFile && (numFiles > 1 + if (dstFile->Protocol == XrdCpFile::isFile && (numFiles > 1 || (OpSpec & DoRecurse && srcFile->Protocol != XrdCpFile::isFile))) FMSG("Destination is neither remote nor a directory.", 2); @@ -846,7 +859,7 @@ void XrdCpConfig::Usage(int rc) " [--path] [--posc] [--proxy :] [--recursive]\n" " [--retry ] [--server] [--silent] [--sources ] [--streams ]\n" " [--tpc {first|only}] [--verbose] [--version] [--xrate ]\n" - " [--parallel ]"; + " [--parallel ] [--metalink]"; static const char *Syntax2= "\n" ": [[x]root://[:]/] | -"; @@ -890,7 +903,8 @@ void XrdCpConfig::Usage(int rc) "-V | --version prints the version number\n" "-X | --xrate limits the transfer to the specified rate. You can\n" " suffix the value with 'k', 'm', or 'g'\n" - " --parallel number of copy jobs to be run simultaneously\n\n" + " --parallel number of copy jobs to be run simultaneously\n" + "-M | --metalink treats source as a metalink\n\n" "Legacy options: [-adler] [-DI ] [-DS ] [-np]\n" " [-md5] [-OD] [-OS] [-version] [-x]"; diff --git a/src/XrdApps/XrdCpConfig.hh b/src/XrdApps/XrdCpConfig.hh index 045c30bd94b..1d70890b5bc 100644 --- a/src/XrdApps/XrdCpConfig.hh +++ b/src/XrdApps/XrdCpConfig.hh @@ -155,6 +155,9 @@ static const int DoParallel = 0x00200000; // --parallel static const int OpDynaSrc = 'Z'; static const int DoDynaSrc = 0x00400000; // --dynamic-src +static const int OpMetalink = 'M'; +static const int DoMetalink = 0x01000000; // -M | --metalink + // Call Config with the parameters passed to main() to fill out this object. If // the method returns then no errors have been found. Otherwise, it exits. // The following options may be passed (largely to support legacy stuff): diff --git a/src/XrdCl/CMakeLists.txt b/src/XrdCl/CMakeLists.txt index cf2a7391a36..cf09bf40c25 100644 --- a/src/XrdCl/CMakeLists.txt +++ b/src/XrdCl/CMakeLists.txt @@ -62,10 +62,12 @@ add_library( XrdClCopyJob.hh XrdClFileSystemUtils.cc XrdClFileSystemUtils.hh XrdClTPFallBackCopyJob.cc XrdClTPFallBackCopyJob.hh + XrdClMetalinkCopyJob.cc XrdClMetalinkCopyJob.hh ) target_link_libraries( XrdCl + XrdXml XrdUtils pthread dl) diff --git a/src/XrdCl/XrdClCopy.cc b/src/XrdCl/XrdClCopy.cc index e6299c436b7..f606c0253e6 100644 --- a/src/XrdCl/XrdClCopy.cc +++ b/src/XrdCl/XrdClCopy.cc @@ -547,6 +547,7 @@ int main( int argc, char **argv ) bool coerce = false; bool makedir = false; bool dynSrc = false; + bool metalink = false; std::string thirdParty = "none"; if( config.Want( XrdCpConfig::DoPosc ) ) posc = true; @@ -557,6 +558,7 @@ int main( int argc, char **argv ) if( config.Want( XrdCpConfig::DoRecurse ) ) makedir = true; if( config.Want( XrdCpConfig::DoPath ) ) makedir = true; if( config.Want( XrdCpConfig::DoDynaSrc ) ) dynSrc = true; + if( config.Want( XrdCpConfig::DoMetalink ) ) metalink = true; //---------------------------------------------------------------------------- // Checksums @@ -650,17 +652,24 @@ int main( int argc, char **argv ) else if( config.dstFile->Protocol == XrdCpFile::isXroot ) { URL target( dest ); - FileSystem fs( target ); - StatInfo *statInfo = 0; - XRootDStatus st = fs.Stat( target.GetPath(), statInfo ); - if( st.IsOK() ) - {if (statInfo->TestFlags( StatInfo::IsDir ) ) targetIsDir = true;} - else if (st.errNo == kXR_NotFound && config.Want( XrdCpConfig::DoPath )) - {int n = strlen(config.dstFile->Path); - if (config.dstFile->Path[n-1] == '/') targetIsDir = true; - } - - delete statInfo; + //---------------------------------------------------------------------------- + // In case of metalink we accept an empty path, otherwise we do a stat + // (the remaining part of the URL might be given in the metalink file) + //---------------------------------------------------------------------------- + if( !( target.GetPath().empty() && config.Want( XrdCpConfig::DoMetalink ) ) ) + { + FileSystem fs( target ); + StatInfo *statInfo = 0; + XRootDStatus st = fs.Stat( target.GetPath(), statInfo ); + if( st.IsOK() ) + {if (statInfo->TestFlags( StatInfo::IsDir ) ) targetIsDir = true;} + else if (st.errNo == kXR_NotFound && config.Want( XrdCpConfig::DoPath )) + {int n = strlen(config.dstFile->Path); + if (config.dstFile->Path[n-1] == '/') targetIsDir = true; + } + + delete statInfo; + } } //---------------------------------------------------------------------------- @@ -754,6 +763,7 @@ int main( int argc, char **argv ) properties.Set( "checkSumPreset", checkSumPreset ); properties.Set( "chunkSize", chunkSize ); properties.Set( "parallelChunks", parallelChunks ); + properties.Set( "metalink", metalink ); XRootDStatus st = process.AddJob( properties, results ); if( !st.IsOK() ) diff --git a/src/XrdCl/XrdClCopyProcess.cc b/src/XrdCl/XrdClCopyProcess.cc index ab7830cbd32..1cc77b5d78b 100644 --- a/src/XrdCl/XrdClCopyProcess.cc +++ b/src/XrdCl/XrdClCopyProcess.cc @@ -31,6 +31,7 @@ #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClMonitor.hh" #include "XrdCl/XrdClCopyJob.hh" +#include "XrdClMetalinkCopyJob.hh" #include "XrdCl/XrdClUtils.hh" #include "XrdCl/XrdClJobManager.hh" #include "XrdCl/XrdClUglyHacks.hh" @@ -257,6 +258,25 @@ namespace XrdCl if( !source.IsValid() ) return XRootDStatus( stError, errInvalidArgs, 0, "invalid source" ); + bool metalink = false; + props.Get( "metalink", metalink ); + + if( metalink && !source.IsMetalink()) + { + log->Debug( UtilityMsg, "CopyProcess (job #%d): metalink transfer requested, but no metalink found.", + i ); + CleanUpJobs(); + XRootDStatus st = XRootDStatus( stError, errInvalidArgs ); + res->Set( "status", st ); + return st; + } + + if( metalink && source.IsMetalink() ) + { + pJobs.push_back( new MetalinkCopyJob( i+1, &props, res ) ); + continue; + } + props.Get( "target", tmp ); URL target = tmp; if( !target.IsValid() ) diff --git a/src/XrdCl/XrdClMetalinkCopyJob.cc b/src/XrdCl/XrdClMetalinkCopyJob.cc new file mode 100644 index 00000000000..dbb72459d06 --- /dev/null +++ b/src/XrdCl/XrdClMetalinkCopyJob.cc @@ -0,0 +1,288 @@ +/* + * XrdClMetalinkCopy.cpp + * + * Created on: Sep 2, 2015 + * Author: simonm + */ + +#include "XrdCl/XrdClMetalinkCopyJob.hh" +#include "XrdCl/XrdClLog.hh" +#include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdCl/XrdClConstants.hh" +#include "XrdCl/XrdClClassicCopyJob.hh" +#include "XrdCl/XrdClTPFallBackCopyJob.hh" +#include "XrdCl/XrdClUglyHacks.hh" +#include "XrdCl/XrdClFileSystem.hh" + +#include "XrdXml/XrdXmlMetaLink.hh" + +#include +#include + +#include +#include +#include +#include + +namespace XrdCl +{ + + //---------------------------------------------------------------------------- + // Constructor + //---------------------------------------------------------------------------- + MetalinkCopyJob::MetalinkCopyJob( uint16_t jobId, + PropertyList *jobProperties, + PropertyList *jobResults ): + CopyJob( jobId, jobProperties, jobResults ), pJobId(jobId), pFileInfos(0), size(0), pMetalinkFile("/tmp/XXXXXX"), pLocalFile( false ) + { + Log *log = DefaultEnv::GetLog(); + log->Debug( UtilityMsg, "Creating a metalink copy job, from %s to %s", + GetSource().GetURL().c_str(), GetTarget().GetURL().c_str() ); + } + + MetalinkCopyJob::~MetalinkCopyJob() + { + if( pFileInfos ) + XrdXmlMetaLink::DeleteAll(pFileInfos, size); + + if( !pMetalinkFile.empty() && !pLocalFile) + remove(pMetalinkFile.c_str()); + } + + //---------------------------------------------------------------------------- + // Run the copy job + //---------------------------------------------------------------------------- + XRootDStatus MetalinkCopyJob::Run( CopyProgressHandler *progress ) + { + XRootDStatus ret; + // download the metalink file + ret = DownloadMetalink( progress ); + if( !ret.IsOK() ) return ret; + // parse the metalink file + ret = ParseMetalink(); + if( !ret.IsOK() ) return ret; + // copy files retrieved from metalink + return CopyFiles( progress ); + } + + XRootDStatus MetalinkCopyJob::DownloadMetalink( CopyProgressHandler *progress ) + { + Log *log = DefaultEnv::GetLog(); + // check if it is a local file + if (pSource.GetProtocol() == "file") + { + // if yes use that one + pMetalinkFile = pSource.GetPath(); + pLocalFile = true; + return XRootDStatus(); + } + // otherwise download the file with a tmp name + log->Info( UtilityMsg, "Downloading the metalink file." ); + // generate the tmp name + mktemp( &*pMetalinkFile.begin() ); + if( pMetalinkFile.empty() ) + return XRootDStatus(stError, errRetry, 0); + + // set copy job parameters + PropertyList res; + PropertyList props; + props.Set( "source", pSource.GetURL() ); + props.Set( "target", pMetalinkFile ); + props.Set( "force", false ); + props.Set( "posc", false ); + props.Set( "coerce", false ); + props.Set( "makeDir", false ); + props.Set( "dynamicSource", false ); + props.Set( "thirdParty", "none" ); + props.Set( "checkSumMode", "none" ); + props.Set( "checkSumType", "" ); + props.Set( "checkSumPreset", "" ); + props.Set( "chunkSize", DefaultCPChunkSize ); + props.Set( "parallelChunks", DefaultCPParallelChunks ); + + // do a classic copy job + ClassicCopyJob mlJob( pJobId, &props, &res ); + return mlJob.Run( progress ); + } + + XRootDStatus MetalinkCopyJob::RemoveFile(const URL & url) + { + // local file + if( url.GetProtocol() == "file" ) + { + int rc = remove( url.GetPath().c_str() ); + if( rc != 0 ) + return XRootDStatus( stError, errOSError, errno, "MetalinkCopyJob: failed to cleanup a replica with a wrong checksum." ); + return XRootDStatus(); + } + // standard output + else if( url.GetProtocol() == "stdio" ) + return XRootDStatus(); // nothing to do ... + // remote file + else + { + FileSystem fs( url ); + return fs.Rm( url.GetPath() ); + } + } + + XRootDStatus MetalinkCopyJob::ParseMetalink() + { + XrdXmlMetaLink metalink; + pFileInfos = metalink.ConvertAll( pMetalinkFile.c_str(), size, 0 ); + + if( !pFileInfos ) + { + int ecode; + const char * etxt = metalink.GetStatus( ecode ); + Log *log = DefaultEnv::GetLog(); + log->Error( UtilityMsg, "Failed to parse the metalink file: %s (error code: %d)", etxt, ecode ); + return XRootDStatus( stError, errDataError ); + } + + if( size != 1 ) + { + Log *log = DefaultEnv::GetLog(); + log->Error( UtilityMsg, "Expected only one file per metalink." ); + return XRootDStatus( stError, errDataError ); + } + + return XRootDStatus(); + } + + XRootDStatus MetalinkCopyJob::CopyFiles( CopyProgressHandler *progress ) + { + Log *log = DefaultEnv::GetLog(); + // get the metalink filename + std::string metalink = GetSource().GetURL(); + size_t pos = metalink.rfind( '/' ); + metalink = metalink.substr( pos + 1 ); + // the target file name specified in metalink + std::string sufix = pFileInfos[0]->GetTargetName(); + // get the target + std::string trg = GetTarget().GetURL(), target, tmp; + // if trg is empty the destination was not specified and an absolute path is expected in the metalink file + if( GetTarget().GetPath().empty() ) + { + if( sufix[0] != '/' ) + { + log->Error( UtilityMsg, "MetalinkCopyJob: metalink target is not an absolute path and no destination has been specified." ); + XRootDStatus st = XRootDStatus( stError, errInvalidArgs ); + return st; + } + target = trg + sufix; + } + else + { + // check if a directory is the destination (make sure the metalink file name is not used as destination file name) + pos = trg.rfind( '/' ); + tmp = trg.substr( pos + 1 ); + // check if that's a path to directory or a file + // (if it's a directory we will find the metalink name appended to the end) + // (this holds for remote files) + bool isDir = ( tmp == metalink ); + // if the metalink name was appended remove it + target = isDir ? trg.substr( 0, pos + 1 ) : trg; + // set target filename + if( isDir ) + { + if( sufix[0] == '/' ) + { + log->Error( UtilityMsg, "MetalinkCopyJob: metalink target is an absolute path and a destination has been specified." ); + XRootDStatus st = XRootDStatus( stError, errInvalidArgs ); + return st; + } + target += "/" + sufix; + } + } + URL trgUrl = target; + if( !trgUrl.IsValid() ) + return XRootDStatus( stError, errInvalidArgs, 0, "invalid target" ); + + pProperties->Set("target", target); + // check if it is a third party copy + bool tpc = false; + pProperties->Get("thirdParty", tmp); + tpc = tmp != "none"; + // transfer status + XRootDStatus ret; + // take care of checksumming + // check what the user specified + std::string usrmode, usrtype, usrval; + pProperties->Get( "checkSumMode", usrmode ); + pProperties->Get( "checkSumType", usrtype ); + if( usrtype == "adler32" ) usrtype = "a32"; + pProperties->Get( "checkSumPreset", usrval ); + // if the value was defined by user we leave it as it is + if( usrval.empty() ) + { + // check if the value for the checksum type was given in the metalink + const char *chvalue = 0, *chtype = 0; + do + { + chtype = pFileInfos[0]->GetDigest( chvalue ); + } + while( !(chtype == 0 || usrtype == chtype) ); + // if the checksum value has been defined in metalink use it + if( chtype ) + { + log->Info( UtilityMsg, "Using checksum value specified in metalink: %s:%s", chtype, chvalue ); + pProperties->Set( "checkSumPreset", chvalue ); + } + } + // try the different replicas + bool cleanup = false; + const char * url = 0; + char cntry[3]; + int prty; + while( (url = pFileInfos[0]->GetUrl(cntry, &prty)) ) + { + log->Info( UtilityMsg, "Try URL: %s", url ); + + if( cleanup ) + { + ret = RemoveFile( target ); + if( !ret.IsOK() ) + { + log->Error( UtilityMsg, "MetalinkCopyJob: could not cleanup a replica with a wrong checksum." ); + return ret; + } + cleanup = false; + } + + URL srcUrl = std::string( url ); + if( !srcUrl.IsValid() ) + { + log->Error( UtilityMsg, "MetalinkCopyJob: invalid source URL." ); + ret = XRootDStatus( stError, errInvalidArgs, 0, "invalid source" ); + continue; + } + + // create the copy job + pProperties->Set("source", url); + XRDCL_SMART_PTR_T job; + if( tpc ) job.reset( new TPFallBackCopyJob( pJobId, pProperties, pResults ) ); + else job.reset( new ClassicCopyJob( pJobId, pProperties, pResults ) ); + ret = job->Run( progress ); + // if the transfer was successful we are done + if( ret.IsOK() ) break; + else + { + // if this is a OS error (e.g. file already exists) + // trying another replica doesn't make sense + if( ret.code == errOSError ) return ret; + // print the error msg for the given replica + log->Error( UtilityMsg, "%s", ret.ToStr().c_str() ); + // in case there are some left overs after previous transfer due to + // checksum failure set the force flag so the file can be overwritten + if( ret.code == errCheckSumError ) + { + cleanup = true; + } + } + } + + return ret; + } + +} diff --git a/src/XrdCl/XrdClMetalinkCopyJob.hh b/src/XrdCl/XrdClMetalinkCopyJob.hh new file mode 100644 index 00000000000..3ba95e25826 --- /dev/null +++ b/src/XrdCl/XrdClMetalinkCopyJob.hh @@ -0,0 +1,59 @@ +/* + * XrdClMetalinkCopy.h + * + * Created on: Sep 2, 2015 + * Author: simonm + */ + +#ifndef SRC_XRDCL_XRDCLMETALINKCOPYJOB_HH_ +#define SRC_XRDCL_XRDCLMETALINKCOPYJOB_HH_ + +#include "XrdCl/XrdClCopyProcess.hh" +#include "XrdCl/XrdClCopyJob.hh" +#include "XrdCl/XrdClLog.hh" + +#include "XrdOuc/XrdOucFileInfo.hh" + +namespace XrdCl +{ + + class MetalinkCopyJob : public CopyJob + { + public: + //------------------------------------------------------------------------ + // Constructor + //------------------------------------------------------------------------ + MetalinkCopyJob( uint16_t jobId, + PropertyList *jobProperties, + PropertyList *jobResults ); + + //------------------------------------------------------------------------ + //! Run the copy job + //! + //! @param progress the handler to be notified about the copy progress + //! @return status of the copy operation + //------------------------------------------------------------------------ + virtual XRootDStatus Run( CopyProgressHandler *progress = 0 ); + + //------------------------------------------------------------------------ + // Destructor + //------------------------------------------------------------------------ + virtual ~MetalinkCopyJob(); + + private: + + XRootDStatus CopyFiles( CopyProgressHandler *progress ); + XRootDStatus DownloadMetalink( CopyProgressHandler *progress ); + XRootDStatus ParseMetalink(); + XRootDStatus RemoveFile(const URL & url); + + uint16_t pJobId; + XrdOucFileInfo ** pFileInfos; + int size; + std::string pMetalinkFile; + bool pLocalFile; + }; + +} + +#endif /* SRC_XRDCL_XRDCLMETALINKCOPYJOB_HH_ */ diff --git a/src/XrdCl/XrdClURL.cc b/src/XrdCl/XrdClURL.cc index 58cafb6732f..a7c4fc9ffda 100644 --- a/src/XrdCl/XrdClURL.cc +++ b/src/XrdCl/XrdClURL.cc @@ -25,6 +25,7 @@ #include #include #include +#include namespace XrdCl { @@ -365,6 +366,18 @@ namespace XrdCl return true; } + bool URL::IsMetalink() const + { + return PathEndsWith(".meta4") || PathEndsWith(".metalink"); + } + + bool URL::PathEndsWith(const std::string & sufix) const + { + if (sufix.size() > pPath.size()) return false; + std::string::const_iterator begin = pPath.end() - sufix.size(); + return std::equal(sufix.begin(), sufix.end(), begin); + } + //---------------------------------------------------------------------------- // Recompute the host id //---------------------------------------------------------------------------- diff --git a/src/XrdCl/XrdClURL.hh b/src/XrdCl/XrdClURL.hh index 87f9ba99873..72ad4652c1e 100644 --- a/src/XrdCl/XrdClURL.hh +++ b/src/XrdCl/XrdClURL.hh @@ -51,6 +51,11 @@ namespace XrdCl //------------------------------------------------------------------------ bool IsValid() const; + //------------------------------------------------------------------------ + //! Is it a URL to a metalink + //------------------------------------------------------------------------ + bool IsMetalink() const; + //------------------------------------------------------------------------ //! Get the URL //------------------------------------------------------------------------ @@ -235,6 +240,7 @@ namespace XrdCl bool ParsePath( const std::string &path ); void ComputeHostId(); void ComputeURL(); + bool PathEndsWith( const std::string & sufix ) const; std::string pHostId; std::string pProtocol; std::string pUserName; diff --git a/src/XrdCl/XrdClXRootDResponses.hh b/src/XrdCl/XrdClXRootDResponses.hh index 4543be5a9f0..57b3c042d5a 100644 --- a/src/XrdCl/XrdClXRootDResponses.hh +++ b/src/XrdCl/XrdClXRootDResponses.hh @@ -669,7 +669,8 @@ namespace XrdCl //------------------------------------------------------------------------ void SetParentName( const std::string &parent ) { - pParent = parent; + size_t pos = parent.find( '?' ); + pParent = pos == std::string::npos ? parent : parent.substr( 0, pos ); if( !pParent.empty() && pParent[pParent.length()-1] != '/' ) pParent += "/"; } diff --git a/src/XrdOuc/XrdOucFileInfo.cc b/src/XrdOuc/XrdOucFileInfo.cc index 09662984f14..b8603446935 100644 --- a/src/XrdOuc/XrdOucFileInfo.cc +++ b/src/XrdOuc/XrdOucFileInfo.cc @@ -121,7 +121,7 @@ void XrdOucFileInfo::AddDigest(const char *hname, const char *hval) // Now make sure the hash type is lower case // - n = strlen(hval); + n = strlen(hname); for (int i = 0; i < n; i++) fHash->hName[i] = tolower(fHash->hName[i]); } diff --git a/tests/XrdClTests/CMakeLists.txt b/tests/XrdClTests/CMakeLists.txt index 6a36f0ca816..8be7475c9e9 100644 --- a/tests/XrdClTests/CMakeLists.txt +++ b/tests/XrdClTests/CMakeLists.txt @@ -15,6 +15,7 @@ add_library( FileCopyTest.cc ThreadingTest.cc IdentityPlugIn.cc + MetalinkTest.cc ) target_link_libraries( diff --git a/tests/XrdClTests/MetalinkTest.cc b/tests/XrdClTests/MetalinkTest.cc new file mode 100644 index 00000000000..b656d86cff8 --- /dev/null +++ b/tests/XrdClTests/MetalinkTest.cc @@ -0,0 +1,289 @@ +/* + * MetalinkTest.cc + * + * Created on: Feb 23, 2016 + * Author: simonm + */ + + + +#include +#include "TestEnv.hh" +#include "CppUnitXrdHelpers.hh" + +#include "XrdCl/XrdClCopyProcess.hh" +#include "XrdCl/XrdClFileSystem.hh" + +using namespace XrdClTests; + +//------------------------------------------------------------------------------ +// Declaration +//------------------------------------------------------------------------------ +class MetalinkTest: public CppUnit::TestCase +{ + public: + CPPUNIT_TEST_SUITE( MetalinkTest ); + CPPUNIT_TEST( CopySimpleTest ); + CPPUNIT_TEST( Copy2SourcesTest ); + CPPUNIT_TEST( CopyCkSumTest ); + CPPUNIT_TEST( CopyCkSum2SourcesTest ); + CPPUNIT_TEST( CopyMultipleCkSumTest ); + CPPUNIT_TEST_SUITE_END(); + + void CopySimpleTest(); + void Copy2SourcesTest(); + void CopyCkSumTest(); + void CopyCkSum2SourcesTest(); + void CopyMultipleCkSumTest(); + void DoTest( XrdCl::PropertyList & properties, const std::string & dataPath, const std::string & address ); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( MetalinkTest ); + + + +void MetalinkTest::DoTest( XrdCl::PropertyList & properties, const std::string & dataPath, const std::string & address ) +{ + using namespace XrdCl; + //---------------------------------------------------------------------------- + // Initialize and run the copy + //---------------------------------------------------------------------------- + CopyProcess process; + PropertyList results; + properties.Set( "metalink", true ); + + CPPUNIT_ASSERT_XRDST( process.AddJob( properties, &results ) ); + CPPUNIT_ASSERT_XRDST( process.Prepare() ); + CPPUNIT_ASSERT_XRDST( process.Run(0) ); + + //---------------------------------------------------------------------------- + // Cleanup + //---------------------------------------------------------------------------- + FileSystem fs( address ); + CPPUNIT_ASSERT_XRDST( fs.Rm( dataPath + "/output.dat" ) ); +} + +//------------------------------------------------------------------------------ +// Simple copy test +//------------------------------------------------------------------------------ +void MetalinkTest::CopySimpleTest() +{ + using namespace XrdCl; + + //---------------------------------------------------------------------------- + // Initialize + //---------------------------------------------------------------------------- + Env *testEnv = TestEnv::GetEnv(); + + std::string address; + std::string dataPath; + + CPPUNIT_ASSERT( testEnv->GetString( "MainServerURL", address ) ); + CPPUNIT_ASSERT( testEnv->GetString( "DataPath", dataPath ) ); + + URL url( address ); + CPPUNIT_ASSERT( url.IsValid() ); + + std::string metalinkURL = address + "/" + dataPath + "/metalink/input1.metalink"; //< single source, no checksum + std::string meta4URL = address + "/" + dataPath + "/metalink/input1.meta4"; //< single source, no checksum + std::string targetURL = address + "/" + dataPath + "/input1.metalink"; //< use metalink file name (will be automatically replaced) + std::string target4URL = address + "/" + dataPath + "/input1.meta4"; //< use metalink file name (will be automatically replaced) + + //---------------------------------------------------------------------------- + // Run the test + //---------------------------------------------------------------------------- + + // metalink 3.0 + PropertyList properties1; + properties1.Set( "source", metalinkURL ); + properties1.Set( "target", targetURL ); + DoTest( properties1, dataPath, address ); + + //metalink 4.0 + PropertyList properties2; + properties2.Set( "source", meta4URL ); + properties2.Set( "target", target4URL ); + DoTest( properties2, dataPath, address ); +} + +//------------------------------------------------------------------------------ +// Copy test - 2 sources +//------------------------------------------------------------------------------ +void MetalinkTest::Copy2SourcesTest() +{ + using namespace XrdCl; + + //---------------------------------------------------------------------------- + // Initialize + //---------------------------------------------------------------------------- + Env *testEnv = TestEnv::GetEnv(); + + std::string address; + std::string dataPath; + + CPPUNIT_ASSERT( testEnv->GetString( "MainServerURL", address ) ); + CPPUNIT_ASSERT( testEnv->GetString( "DataPath", dataPath ) ); + + URL url( address ); + CPPUNIT_ASSERT( url.IsValid() ); + + std::string metalinkURL = address + "/" + dataPath + "/metalink/input2.metalink"; //< 2 source files (1st one does not exists), no checksum + std::string meta4URL = address + "/" + dataPath + "/metalink/input2.meta4"; //< 2 source files (1st one does not exists), no checksum + std::string targetURL = address + "/" + dataPath + "/output.dat"; + + //---------------------------------------------------------------------------- + // Run the test + //---------------------------------------------------------------------------- + + // metalink 3.0 + PropertyList properties1; + properties1.Set( "source", metalinkURL ); + properties1.Set( "target", targetURL ); + DoTest( properties1, dataPath, address ); + + // metalink 4.0 + PropertyList properties2; + properties2.Set( "source", meta4URL ); + properties2.Set( "target", targetURL ); + DoTest( properties2, dataPath, address ); +} + +//------------------------------------------------------------------------------ +// Copy test with checksum +//------------------------------------------------------------------------------ +void MetalinkTest::CopyCkSumTest() +{ + using namespace XrdCl; + + //---------------------------------------------------------------------------- + // Initialize + //---------------------------------------------------------------------------- + Env *testEnv = TestEnv::GetEnv(); + + std::string address; + std::string dataPath; + + CPPUNIT_ASSERT( testEnv->GetString( "MainServerURL", address ) ); + CPPUNIT_ASSERT( testEnv->GetString( "DataPath", dataPath ) ); + + URL url( address ); + CPPUNIT_ASSERT( url.IsValid() ); + + std::string metalinkURL = address + "/" + dataPath + "/metalink/input3.metalink"; //< single source, checksum + std::string meta4URL = address + "/" + dataPath + "/metalink/input3.meta4"; //< single source, checksum + std::string targetURL = address + "/" + dataPath + "/input3.metalink"; //< use metalink file name (will be automatically replaced) + std::string target4URL = address + "/" + dataPath + "/input3.meta4"; //< use metalink file name (will be automatically replaced) + + //---------------------------------------------------------------------------- + // Run the test + //---------------------------------------------------------------------------- + + // metalink 3.0 + PropertyList properties1; + properties1.Set( "source", metalinkURL ); + properties1.Set( "target", targetURL ); + properties1.Set( "checkSumMode", "end2end" ); + properties1.Set( "checkSumType", "zcrc32" ); + DoTest( properties1, dataPath, address ); + + // metalink 4.0 + PropertyList properties2; + properties2.Set( "source", meta4URL ); + properties2.Set( "target", target4URL ); + properties2.Set( "checkSumMode", "end2end" ); + properties2.Set( "checkSumType", "zcrc32" ); + DoTest( properties2, dataPath, address ); +} + +//------------------------------------------------------------------------------ +// Copy test with checksum and 2 sources +//------------------------------------------------------------------------------ +void MetalinkTest::CopyCkSum2SourcesTest() +{ + using namespace XrdCl; + + //---------------------------------------------------------------------------- + // Initialize + //---------------------------------------------------------------------------- + Env *testEnv = TestEnv::GetEnv(); + + std::string address; + std::string dataPath; + + CPPUNIT_ASSERT( testEnv->GetString( "MainServerURL", address ) ); + CPPUNIT_ASSERT( testEnv->GetString( "DataPath", dataPath ) ); + + URL url( address ); + CPPUNIT_ASSERT( url.IsValid() ); + + std::string metalinkURL = address + "/" + dataPath + "/metalink/input4.metalink"; //< 2 source files, 1st one doesn't much the checksum + std::string meta4URL = address + "/" + dataPath + "/metalink/input4.meta4"; //< 2 source files, 1st one doesn't much the checksum + std::string targetURL = address; + + //---------------------------------------------------------------------------- + // Run the test + //---------------------------------------------------------------------------- + + // metalink 3.0 + PropertyList properties1; + properties1.Set( "source", metalinkURL ); + properties1.Set( "target", targetURL ); + properties1.Set( "checkSumMode", "end2end" ); + properties1.Set( "checkSumType", "zcrc32" ); + DoTest( properties1, dataPath, address ); + + // metalink 3.0 + PropertyList properties2; + properties2.Set( "source", meta4URL ); + properties2.Set( "target", targetURL ); + properties2.Set( "checkSumMode", "end2end" ); + properties2.Set( "checkSumType", "zcrc32" ); + DoTest( properties2, dataPath, address ); +} + +//------------------------------------------------------------------------------ +// Copy test with multiple checksums +//------------------------------------------------------------------------------ +void MetalinkTest::CopyMultipleCkSumTest() +{ + using namespace XrdCl; + + //---------------------------------------------------------------------------- + // Initialize + //---------------------------------------------------------------------------- + Env *testEnv = TestEnv::GetEnv(); + + std::string address; + std::string dataPath; + + CPPUNIT_ASSERT( testEnv->GetString( "MainServerURL", address ) ); + CPPUNIT_ASSERT( testEnv->GetString( "DataPath", dataPath ) ); + + URL url( address ); + CPPUNIT_ASSERT( url.IsValid() ); + + std::string metalinkURL = address + "/" + dataPath + "/metalink/input5.metalink"; //< single source, multiple checksum (adler32 + md5 + zcrc32) + std::string meta4URL = address + "/" + dataPath + "/metalink/input5.meta4"; //< single source, multiple checksum (adler32 + md5 + zcrc32) + std::string targetURL = address + "/" + dataPath + "/output.dat"; //< use metalink file name (will be automatically replaced) + + //---------------------------------------------------------------------------- + // Run the test + //---------------------------------------------------------------------------- + + // metalink 3.0 + PropertyList properties1; + properties1.Set( "source", metalinkURL ); + properties1.Set( "target", targetURL ); + properties1.Set( "checkSumMode", "end2end" ); + properties1.Set( "checkSumType", "zcrc32" ); + DoTest( properties1, dataPath, address ); + + // metalink 4.0 + PropertyList properties2; + properties2.Set( "source", meta4URL ); + properties2.Set( "target", targetURL ); + properties2.Set( "checkSumMode", "end2end" ); + properties2.Set( "checkSumType", "zcrc32" ); + DoTest( properties2, dataPath, address ); +} +