Skip to content

Commit

Permalink
Merge branch 'master' into pssasync
Browse files Browse the repository at this point in the history
  • Loading branch information
@abh3
abh3 committed Mar 18, 2016
2 parents 38ea7f9 + 422ae9c commit c4dd4d8
Show file tree
Hide file tree
Showing 14 changed files with 751 additions and 31 deletions.
16 changes: 15 additions & 1 deletion docs/man/xrdcp.1
View file
Expand Up @@ -12,7 +12,7 @@ xrdcp - copy files
[\fB--recursive\fR] [\fB--retry\fR \fItime\fR] [\fB--server\fR]
[\fB--silent\fR] [\fB--sources\fR \fInum\fR] [\fB--streams\fR \fInum\fR]
[\fB--tpc\fR \fIfirst\fR|\fIonly\fR] [\fB--verbose\fR] [\fB--version\fR]
[\fB--xrate\fR \fIrate\fR]
[\fB--xrate\fR \fIrate\fR] [\fB--metalink\fR]

\fIlegacy options\fR: [\fB-adler\fR] [\fB-DS\fR\fIparm string\fR] [\fB-DI\fR\fIparm number\fR]
[\fB-md5\fR] [\fB-np\fR] [\fB-OD\fR\fIcgi\fR] [\fB-OS\fR\fIcgi\fR] [\fB-x\fR]
Expand Down Expand Up @@ -133,6 +133,11 @@ displays version information and immediately exits.
.RS 5
[NOT YET IMPLEMENTED]

.RE
\fB-M\fR | \fB--metalink\fR
.RS 5
treats source as a metalink.

limits the copy speed to the specified \fIrate\fB. The rate may be qualified
with the letter \fBk\fR, \fBm\fR, or \fBg\fR to indicate kilo, mega, or giga
bytes, respectively. The option only applies when the source or destination is
Expand Down Expand Up @@ -248,6 +253,15 @@ XRD_PARALLELEVTLOOP
The number of event loops.
.RE

XRD_READRECOVERY
.RS 5
Determines if read recovery should be enabled or disabled (enabled by default).
.RE

XRD_WRITERECOVERY
.RS 5
Determines if write recovery should be enabled or disabled (enabled by default).
.RE

XRD_CONNECTIONWINDOW (-DIConnectionWindow)
.RS 5
Expand Down
48 changes: 31 additions & 17 deletions src/XrdApps/XrdCpConfig.cc
View file
Expand Up @@ -81,7 +81,7 @@ static XrdSysError eDest(&Logger, "");

XrdSysError *XrdCpConfig::Log = &XrdCpConfiguration::eDest;

const char *XrdCpConfig::opLetters = ":C:d:D:fFhHI:NpPrRsS:t:T:vVX:y:Z";
const char *XrdCpConfig::opLetters = ":C:d:D:fFhHI:NpPrRsS:t:T:vVX:y:Z:M";

struct option XrdCpConfig::opVec[] = // For getopt_long()
{
Expand All @@ -108,6 +108,7 @@ struct option XrdCpConfig::opVec[] = // For getopt_long()
{OPT_TYPE "version", 0, 0, XrdCpConfig::OpVersion},
{OPT_TYPE "xrate", 1, 0, XrdCpConfig::OpXrate},
{OPT_TYPE "parallel", 1, 0, XrdCpConfig::OpParallel},
{OPT_TYPE "metalink", 0, 0, XrdCpConfig::OpMetalink},
{0, 0, 0, 0}
};

Expand Down Expand Up @@ -221,6 +222,8 @@ do{while(optind < Argc && Legacy(optind)) {}
break;
case OpForce: OpSpec |= DoForce;
break;
case OpMetalink: OpSpec |= DoMetalink;
break;
case OpHelp: Usage(0);
break;
case OpIfile: if (inFile) free(inFile);
Expand Down Expand Up @@ -288,7 +291,7 @@ do{while(optind < Argc && Legacy(optind)) {}
//
if (inFile) {if (!parmCnt ) UMSG("Destination not specified.");}
else { if (!parmCnt ) UMSG("No files specified.");
if ( parmCnt == 1) UMSG("Destination not specified.");
if ( parmCnt == 1 && !( OpSpec & DoMetalink ) ) UMSG("Destination not specified.");
}

// Check for conflicts wit third party copy
Expand All @@ -307,26 +310,36 @@ do{while(optind < Argc && Legacy(optind)) {}
//
if (getenv("XRD_MAKEPATH")) OpSpec |= DoPath;

if( parmCnt > 1 )
{
// Process the destination first as it is special
//
dstFile = new XrdCpFile(parmVal[--parmCnt], rc);
if (rc) FMSG("Invalid url, '" <<dstFile->Path <<"'.", 22);
dstFile = new XrdCpFile(parmVal[--parmCnt], rc);
if (rc) FMSG("Invalid url, '" <<dstFile->Path <<"'.", 22);

// Do a protocol check
//
if (dstFile->Protocol != XrdCpFile::isFile
&& dstFile->Protocol != XrdCpFile::isStdIO
&& dstFile->Protocol != XrdCpFile::isXroot)
{FMSG(dstFile->ProtName <<"file protocol is not supported.", 22)}
if (dstFile->Protocol != XrdCpFile::isFile
&& dstFile->Protocol != XrdCpFile::isStdIO
&& dstFile->Protocol != XrdCpFile::isXroot)
{FMSG(dstFile->ProtName <<"file protocol is not supported.", 22)}

// Resolve this file if it is a local file
//
isLcl = (dstFile->Protocol == XrdCpFile::isFile)
| (dstFile->Protocol == XrdCpFile::isStdIO);
if (isLcl && (rc = dstFile->Resolve()))
{if (rc != ENOENT || (Argc - optind - 1) > 1 || OpSpec & DoRecurse)
FMSG(strerror(rc) <<" processing " <<dstFile->Path, 2);
}
isLcl = (dstFile->Protocol == XrdCpFile::isFile)
| (dstFile->Protocol == XrdCpFile::isStdIO);
if (isLcl && (rc = dstFile->Resolve()))
{if (rc != ENOENT || (Argc - optind - 1) > 1 || OpSpec & DoRecurse)
FMSG(strerror(rc) <<" processing " <<dstFile->Path, 2);
}
}
else
{
// Create an empty destination file
//
dstFile = new XrdCpFile();
dstFile->Path = strdup( "" );
}

// Now pick up all the source files from the command line
//
Expand All @@ -353,7 +366,7 @@ do{while(optind < Argc && Legacy(optind)) {}

// Check if we have an appropriate destination
//
if (dstFile->Protocol == XrdCpFile::isFile && (numFiles > 1
if (dstFile->Protocol == XrdCpFile::isFile && (numFiles > 1
|| (OpSpec & DoRecurse && srcFile->Protocol != XrdCpFile::isFile)))
FMSG("Destination is neither remote nor a directory.", 2);

Expand Down Expand Up @@ -846,7 +859,7 @@ void XrdCpConfig::Usage(int rc)
" [--path] [--posc] [--proxy <host>:<port>] [--recursive]\n"
" [--retry <n>] [--server] [--silent] [--sources <n>] [--streams <n>]\n"
" [--tpc {first|only}] [--verbose] [--version] [--xrate <rate>]\n"
" [--parallel <n>]";
" [--parallel <n>] [--metalink]";

static const char *Syntax2= "\n"
"<src>: [[x]root://<host>[:<port>]/]<path> | -";
Expand Down Expand Up @@ -890,7 +903,8 @@ void XrdCpConfig::Usage(int rc)
"-V | --version prints the version number\n"
"-X | --xrate <rate> limits the transfer to the specified rate. You can\n"
" suffix the value with 'k', 'm', or 'g'\n"
" --parallel <n> number of copy jobs to be run simultaneously\n\n"
" --parallel <n> number of copy jobs to be run simultaneously\n"
"-M | --metalink treats source as a metalink\n\n"
"Legacy options: [-adler] [-DI<var> <val>] [-DS<var> <val>] [-np]\n"
" [-md5] [-OD<cgi>] [-OS<cgi>] [-version] [-x]";

Expand Down
3 changes: 3 additions & 0 deletions src/XrdApps/XrdCpConfig.hh
View file
Expand Up @@ -155,6 +155,9 @@ static const int DoParallel = 0x00200000; // --parallel
static const int OpDynaSrc = 'Z';
static const int DoDynaSrc = 0x00400000; // --dynamic-src

static const int OpMetalink = 'M';
static const int DoMetalink = 0x01000000; // -M | --metalink

// Call Config with the parameters passed to main() to fill out this object. If
// the method returns then no errors have been found. Otherwise, it exits.
// The following options may be passed (largely to support legacy stuff):
Expand Down
2 changes: 2 additions & 0 deletions src/XrdCl/CMakeLists.txt
View file
Expand Up @@ -62,10 +62,12 @@ add_library(
XrdClCopyJob.hh
XrdClFileSystemUtils.cc XrdClFileSystemUtils.hh
XrdClTPFallBackCopyJob.cc XrdClTPFallBackCopyJob.hh
XrdClMetalinkCopyJob.cc XrdClMetalinkCopyJob.hh
)

target_link_libraries(
XrdCl
XrdXml
XrdUtils
pthread
dl)
Expand Down
32 changes: 21 additions & 11 deletions src/XrdCl/XrdClCopy.cc
View file
Expand Up @@ -547,6 +547,7 @@ int main( int argc, char **argv )
bool coerce = false;
bool makedir = false;
bool dynSrc = false;
bool metalink = false;
std::string thirdParty = "none";

if( config.Want( XrdCpConfig::DoPosc ) ) posc = true;
Expand All @@ -557,6 +558,7 @@ int main( int argc, char **argv )
if( config.Want( XrdCpConfig::DoRecurse ) ) makedir = true;
if( config.Want( XrdCpConfig::DoPath ) ) makedir = true;
if( config.Want( XrdCpConfig::DoDynaSrc ) ) dynSrc = true;
if( config.Want( XrdCpConfig::DoMetalink ) ) metalink = true;

//----------------------------------------------------------------------------
// Checksums
Expand Down Expand Up @@ -650,17 +652,24 @@ int main( int argc, char **argv )
else if( config.dstFile->Protocol == XrdCpFile::isXroot )
{
URL target( dest );
FileSystem fs( target );
StatInfo *statInfo = 0;
XRootDStatus st = fs.Stat( target.GetPath(), statInfo );
if( st.IsOK() )
{if (statInfo->TestFlags( StatInfo::IsDir ) ) targetIsDir = true;}
else if (st.errNo == kXR_NotFound && config.Want( XrdCpConfig::DoPath ))
{int n = strlen(config.dstFile->Path);
if (config.dstFile->Path[n-1] == '/') targetIsDir = true;
}

delete statInfo;
//----------------------------------------------------------------------------
// In case of metalink we accept an empty path, otherwise we do a stat
// (the remaining part of the URL might be given in the metalink file)
//----------------------------------------------------------------------------
if( !( target.GetPath().empty() && config.Want( XrdCpConfig::DoMetalink ) ) )
{
FileSystem fs( target );
StatInfo *statInfo = 0;
XRootDStatus st = fs.Stat( target.GetPath(), statInfo );
if( st.IsOK() )
{if (statInfo->TestFlags( StatInfo::IsDir ) ) targetIsDir = true;}
else if (st.errNo == kXR_NotFound && config.Want( XrdCpConfig::DoPath ))
{int n = strlen(config.dstFile->Path);
if (config.dstFile->Path[n-1] == '/') targetIsDir = true;
}

delete statInfo;
}
}

//----------------------------------------------------------------------------
Expand Down Expand Up @@ -754,6 +763,7 @@ int main( int argc, char **argv )
properties.Set( "checkSumPreset", checkSumPreset );
properties.Set( "chunkSize", chunkSize );
properties.Set( "parallelChunks", parallelChunks );
properties.Set( "metalink", metalink );

XRootDStatus st = process.AddJob( properties, results );
if( !st.IsOK() )
Expand Down
20 changes: 20 additions & 0 deletions src/XrdCl/XrdClCopyProcess.cc
View file
Expand Up @@ -31,6 +31,7 @@
#include "XrdCl/XrdClFileSystem.hh"
#include "XrdCl/XrdClMonitor.hh"
#include "XrdCl/XrdClCopyJob.hh"
#include "XrdClMetalinkCopyJob.hh"
#include "XrdCl/XrdClUtils.hh"
#include "XrdCl/XrdClJobManager.hh"
#include "XrdCl/XrdClUglyHacks.hh"
Expand Down Expand Up @@ -257,6 +258,25 @@ namespace XrdCl
if( !source.IsValid() )
return XRootDStatus( stError, errInvalidArgs, 0, "invalid source" );

bool metalink = false;
props.Get( "metalink", metalink );

if( metalink && !source.IsMetalink())
{
log->Debug( UtilityMsg, "CopyProcess (job #%d): metalink transfer requested, but no metalink found.",
i );
CleanUpJobs();
XRootDStatus st = XRootDStatus( stError, errInvalidArgs );
res->Set( "status", st );
return st;
}

if( metalink && source.IsMetalink() )
{
pJobs.push_back( new MetalinkCopyJob( i+1, &props, res ) );
continue;
}

props.Get( "target", tmp );
URL target = tmp;
if( !target.IsValid() )
Expand Down

0 comments on commit c4dd4d8

Please sign in to comment.