Skip to content

Commit

Permalink
[XrdCl] Enable metalink processing on default.
Browse files Browse the repository at this point in the history
  • Loading branch information
@simonmichal
simonmichal committed Jun 10, 2016
1 parent 4beafb0 commit 365a383
Show file tree
Hide file tree
Showing 18 changed files with 72 additions and 104 deletions.
7 changes: 1 addition & 6 deletions docs/man/xrdcp.1
View file
Expand Up @@ -12,7 +12,7 @@ xrdcp - copy files
[\fB--recursive\fR] [\fB--retry\fR \fItime\fR] [\fB--server\fR]
[\fB--silent\fR] [\fB--sources\fR \fInum\fR] [\fB--streams\fR \fInum\fR]
[\fB--tpc\fR \fIfirst\fR|\fIonly\fR] [\fB--verbose\fR] [\fB--version\fR]
[\fB--xrate\fR \fIrate\fR] [\fB--metalink\fR]
[\fB--xrate\fR \fIrate\fR]

\fIlegacy options\fR: [\fB-adler\fR] [\fB-DS\fR\fIparm string\fR] [\fB-DI\fR\fIparm number\fR]
[\fB-md5\fR] [\fB-np\fR] [\fB-OD\fR\fIcgi\fR] [\fB-OS\fR\fIcgi\fR] [\fB-x\fR]
Expand Down Expand Up @@ -133,11 +133,6 @@ displays version information and immediately exits.
.RS 5
[NOT YET IMPLEMENTED]

.RE
\fB-M\fR | \fB--metalink\fR
.RS 5
treats source as a metalink.

limits the copy speed to the specified \fIrate\fB. The rate may be qualified
with the letter \fBk\fR, \fBm\fR, or \fBg\fR to indicate kilo, mega, or giga
bytes, respectively. The option only applies when the source or destination is
Expand Down
23 changes: 6 additions & 17 deletions src/XrdApps/XrdCpConfig.cc
View file
Expand Up @@ -81,7 +81,7 @@ static XrdSysError eDest(&Logger, "");

XrdSysError *XrdCpConfig::Log = &XrdCpConfiguration::eDest;

const char *XrdCpConfig::opLetters = ":C:d:D:fFhHI:NpPrRsS:t:T:vVX:y:Z:M";
const char *XrdCpConfig::opLetters = ":C:d:D:fFhHI:NpPrRsS:t:T:vVX:y:Z";

struct option XrdCpConfig::opVec[] = // For getopt_long()
{
Expand All @@ -108,7 +108,6 @@ struct option XrdCpConfig::opVec[] = // For getopt_long()
{OPT_TYPE "version", 0, 0, XrdCpConfig::OpVersion},
{OPT_TYPE "xrate", 1, 0, XrdCpConfig::OpXrate},
{OPT_TYPE "parallel", 1, 0, XrdCpConfig::OpParallel},
{OPT_TYPE "metalink", 0, 0, XrdCpConfig::OpMetalink},
{0, 0, 0, 0}
};

Expand Down Expand Up @@ -222,8 +221,6 @@ do{while(optind < Argc && Legacy(optind)) {}
break;
case OpForce: OpSpec |= DoForce;
break;
case OpMetalink: OpSpec |= DoMetalink;
break;
case OpHelp: Usage(0);
break;
case OpIfile: if (inFile) free(inFile);
Expand Down Expand Up @@ -289,9 +286,9 @@ do{while(optind < Argc && Legacy(optind)) {}

// Make sure we have the right number of files
//
if (inFile) {if (!parmCnt ) UMSG("Destination not specified.");}
else { if (!parmCnt ) UMSG("No files specified.");
if ( parmCnt == 1 && !( OpSpec & DoMetalink ) ) UMSG("Destination not specified.");
if (inFile) {if (!parmCnt ) UMSG("Destination not specified.");}
else { if (!parmCnt ) UMSG("No files specified.");
if ( parmCnt == 1 ) UMSG("Destination not specified.");
}

// Check for conflicts wit third party copy
Expand Down Expand Up @@ -333,13 +330,6 @@ do{while(optind < Argc && Legacy(optind)) {}
FMSG(strerror(rc) <<" processing " <<dstFile->Path, 2);
}
}
else
{
// Create an empty destination file
//
dstFile = new XrdCpFile();
dstFile->Path = strdup( "" );
}

// Now pick up all the source files from the command line
//
Expand Down Expand Up @@ -859,7 +849,7 @@ void XrdCpConfig::Usage(int rc)
" [--path] [--posc] [--proxy <host>:<port>] [--recursive]\n"
" [--retry <n>] [--server] [--silent] [--sources <n>] [--streams <n>]\n"
" [--tpc {first|only}] [--verbose] [--version] [--xrate <rate>]\n"
" [--parallel <n>] [--metalink]";
" [--parallel <n>]";

static const char *Syntax2= "\n"
"<src>: [[x]root://<host>[:<port>]/]<path> | -";
Expand Down Expand Up @@ -903,8 +893,7 @@ void XrdCpConfig::Usage(int rc)
"-V | --version prints the version number\n"
"-X | --xrate <rate> limits the transfer to the specified rate. You can\n"
" suffix the value with 'k', 'm', or 'g'\n"
" --parallel <n> number of copy jobs to be run simultaneously\n"
"-M | --metalink treats source as a metalink\n\n"
" --parallel <n> number of copy jobs to be run simultaneously\n\n"
"Legacy options: [-adler] [-DI<var> <val>] [-DS<var> <val>] [-np]\n"
" [-md5] [-OD<cgi>] [-OS<cgi>] [-version] [-x]";

Expand Down
3 changes: 0 additions & 3 deletions src/XrdApps/XrdCpConfig.hh
View file
Expand Up @@ -155,9 +155,6 @@ static const int DoParallel = 0x00200000; // --parallel
static const int OpDynaSrc = 'Z';
static const int DoDynaSrc = 0x00400000; // --dynamic-src

static const int OpMetalink = 'M';
static const int DoMetalink = 0x01000000; // -M | --metalink

// Call Config with the parameters passed to main() to fill out this object. If
// the method returns then no errors have been found. Otherwise, it exits.
// The following options may be passed (largely to support legacy stuff):
Expand Down
2 changes: 1 addition & 1 deletion src/XrdCl/XrdClChannel.cc
View file
Expand Up @@ -288,7 +288,7 @@ namespace XrdCl

if( redirector )
{
XRootDStatus st = redirector->Redirect( msg, pStreams[path.down] );
XRootDStatus st = redirector->HandleRequest( msg, pStreams[path.down] );
if( st.IsOK() )
handler->OnStatusReady( msg, Status() );
return st;
Expand Down
29 changes: 12 additions & 17 deletions src/XrdCl/XrdClClassicCopyJob.cc
View file
Expand Up @@ -549,11 +549,10 @@ namespace
//------------------------------------------------------------------------
XRootDSource( const XrdCl::URL *url,
uint32_t chunkSize,
uint8_t parallelChunks,
bool virtRedirector ):
uint8_t parallelChunks ):
pUrl( url ), pFile( new XrdCl::File() ), pSize( -1 ),
pCurrentOffset( 0 ), pChunkSize( chunkSize ),
pParallel( parallelChunks ), pVirtRedirector( virtRedirector )
pParallel( parallelChunks )
{
}

Expand Down Expand Up @@ -581,7 +580,7 @@ namespace
DefaultEnv::GetEnv()->GetString( "ReadRecovery", value );
pFile->SetProperty( "ReadRecovery", value );

XRootDStatus st = pFile->Open( pUrl->GetURL(), OpenFlags::Read, Access::None, 0, pVirtRedirector );
XRootDStatus st = pFile->Open( pUrl->GetURL(), OpenFlags::Read );
if( !st.IsOK() )
return st;

Expand Down Expand Up @@ -693,7 +692,7 @@ namespace
virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum,
std::string &checkSumType )
{
if( pVirtRedirector )
if( pUrl->IsMetalink() )
{
XrdCl::RedirectorRegistry &registry = XrdCl::RedirectorRegistry::Instance();
XrdCl::VirtualRedirector *redirector = registry.Get( *pUrl );
Expand Down Expand Up @@ -746,7 +745,6 @@ namespace
uint32_t pChunkSize;
uint8_t pParallel;
std::queue<ChunkHandler *> pChunks;
bool pVirtRedirector;
};

//----------------------------------------------------------------------------
Expand All @@ -759,10 +757,9 @@ namespace
//! Constructor
//------------------------------------------------------------------------
XRootDSourceDynamic( const XrdCl::URL *url,
uint32_t chunkSize,
bool virtRedirector ):
uint32_t chunkSize ):
pUrl( url ), pFile( new XrdCl::File() ), pCurrentOffset( 0 ),
pChunkSize( chunkSize ), pDone( false ), pVirtRedirector( virtRedirector )
pChunkSize( chunkSize ), pDone( false )
{
}

Expand All @@ -789,7 +786,7 @@ namespace
DefaultEnv::GetEnv()->GetString( "ReadRecovery", value );
pFile->SetProperty( "ReadRecovery", value );

XRootDStatus st = pFile->Open( pUrl->GetURL(), OpenFlags::Read, Access::None, 0, pVirtRedirector );
XRootDStatus st = pFile->Open( pUrl->GetURL(), OpenFlags::Read );
if( !st.IsOK() )
return st;

Expand Down Expand Up @@ -865,7 +862,7 @@ namespace
virtual XrdCl::XRootDStatus GetCheckSum( std::string &checkSum,
std::string &checkSumType )
{
if( pVirtRedirector )
if( pUrl->IsMetalink() )
{
XrdCl::RedirectorRegistry &registry = XrdCl::RedirectorRegistry::Instance();
XrdCl::VirtualRedirector *redirector = registry.Get( *pUrl );
Expand All @@ -887,7 +884,6 @@ namespace
int64_t pCurrentOffset;
uint32_t pChunkSize;
bool pDone;
bool pVirtRedirector;
};

//----------------------------------------------------------------------------
Expand Down Expand Up @@ -1249,7 +1245,7 @@ namespace

Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;

return pFile->Open( pUrl->GetURL(), flags, mode );
return pFile->Open( pUrl->GetURL(), flags, mode, 0, false );
}

//------------------------------------------------------------------------
Expand Down Expand Up @@ -1424,7 +1420,7 @@ namespace XrdCl
std::string checkSumPreset;
uint16_t parallelChunks;
uint32_t chunkSize;
bool posc, force, coerce, makeDir, dynamicSource, virtRedirector = false;
bool posc, force, coerce, makeDir, dynamicSource;

pProperties->Get( "checkSumMode", checkSumMode );
pProperties->Get( "checkSumType", checkSumType );
Expand All @@ -1436,7 +1432,6 @@ namespace XrdCl
pProperties->Get( "coerce", coerce );
pProperties->Get( "makeDir", makeDir );
pProperties->Get( "dynamicSource", dynamicSource );
pProperties->Get( "metalink", virtRedirector );

//--------------------------------------------------------------------------
// Initialize the source and the destination
Expand All @@ -1449,9 +1444,9 @@ namespace XrdCl
else
{
if( dynamicSource )
src.reset( new XRootDSourceDynamic( &GetSource(), chunkSize, virtRedirector ) );
src.reset( new XRootDSourceDynamic( &GetSource(), chunkSize ) );
else
src.reset( new XRootDSource( &GetSource(), chunkSize, parallelChunks, virtRedirector ) );
src.reset( new XRootDSource( &GetSource(), chunkSize, parallelChunks ) );
}

XRootDStatus st = src->Initialize();
Expand Down
1 change: 1 addition & 0 deletions src/XrdCl/XrdClConstants.hh
View file
Expand Up @@ -64,6 +64,7 @@ namespace XrdCl
const int DefaultTCPKeepAliveProbes = 9;
const int DefaultMultiProtocol = 0;
const int DefaultParallelEvtLoop = 1;
const int DefaultMetalinkProcessing = 1;

const char * const DefaultPollerPreference = "built-in";
const char * const DefaultNetworkStack = "IPAuto";
Expand Down
39 changes: 15 additions & 24 deletions src/XrdCl/XrdClCopy.cc
View file
Expand Up @@ -548,7 +548,6 @@ int main( int argc, char **argv )
bool coerce = false;
bool makedir = false;
bool dynSrc = false;
bool metalink = false;
std::string thirdParty = "none";

if( config.Want( XrdCpConfig::DoPosc ) ) posc = true;
Expand All @@ -559,7 +558,6 @@ int main( int argc, char **argv )
if( config.Want( XrdCpConfig::DoRecurse ) ) makedir = true;
if( config.Want( XrdCpConfig::DoPath ) ) makedir = true;
if( config.Want( XrdCpConfig::DoDynaSrc ) ) dynSrc = true;
if( config.Want( XrdCpConfig::DoMetalink ) ) metalink = true;

//----------------------------------------------------------------------------
// Checksums
Expand Down Expand Up @@ -653,24 +651,17 @@ int main( int argc, char **argv )
else if( config.dstFile->Protocol == XrdCpFile::isXroot )
{
URL target( dest );
//----------------------------------------------------------------------------
// In case of metalink we accept an empty path, otherwise we do a stat
// (the remaining part of the URL might be given in the metalink file)
//----------------------------------------------------------------------------
if( !( target.GetPath().empty() && config.Want( XrdCpConfig::DoMetalink ) ) )
{
FileSystem fs( target );
StatInfo *statInfo = 0;
XRootDStatus st = fs.Stat( target.GetPath(), statInfo );
if( st.IsOK() )
{if (statInfo->TestFlags( StatInfo::IsDir ) ) targetIsDir = true;}
else if (st.errNo == kXR_NotFound && config.Want( XrdCpConfig::DoPath ))
{int n = strlen(config.dstFile->Path);
if (config.dstFile->Path[n-1] == '/') targetIsDir = true;
}

delete statInfo;
}
FileSystem fs( target );
StatInfo *statInfo = 0;
XRootDStatus st = fs.Stat( target.GetPath(), statInfo );
if( st.IsOK() )
{if (statInfo->TestFlags( StatInfo::IsDir ) ) targetIsDir = true;}
else if (st.errNo == kXR_NotFound && config.Want( XrdCpConfig::DoPath ))
{int n = strlen(config.dstFile->Path);
if (config.dstFile->Path[n-1] == '/') targetIsDir = true;
}

delete statInfo;
}

//----------------------------------------------------------------------------
Expand Down Expand Up @@ -742,10 +733,11 @@ int main( int argc, char **argv )
//--------------------------------------------------------------------------
// Create a virtual redirector if it is a metalink file
//--------------------------------------------------------------------------
if( metalink )
URL src( source );
if( src.IsMetalink() )
{
RedirectorRegistry &registry = RedirectorRegistry::Instance();
XRootDStatus st = registry.Register( source );
XRootDStatus st = registry.Register( src );
if( !st.IsOK() )
{
std::cerr << "RedirectorRegistry::Register " << source << " -> " << dest << ": ";
Expand All @@ -764,7 +756,7 @@ int main( int argc, char **argv )
{
target = dest + "/";
// if it is a metalink we don't want to use the metalink name
if( metalink )
if( src.IsMetalink() )
{
XrdCl::RedirectorRegistry &registry = XrdCl::RedirectorRegistry::Instance();
VirtualRedirector *redirector = registry.Get( source );
Expand All @@ -788,7 +780,6 @@ int main( int argc, char **argv )
properties.Set( "checkSumPreset", checkSumPreset );
properties.Set( "chunkSize", chunkSize );
properties.Set( "parallelChunks", parallelChunks );
properties.Set( "metalink", metalink );

XRootDStatus st = process.AddJob( properties, results );
if( !st.IsOK() )
Expand Down
1 change: 1 addition & 0 deletions src/XrdCl/XrdClDefaultEnv.cc
View file
Expand Up @@ -201,6 +201,7 @@ namespace XrdCl
REGISTER_VAR_INT( varsInt, "TCPKeepProbes", DefaultTCPKeepAliveProbes );
REGISTER_VAR_INT( varsInt, "MultiProtocol", DefaultMultiProtocol );
REGISTER_VAR_INT( varsInt, "ParallelEvtLoop", DefaultParallelEvtLoop );
REGISTER_VAR_INT( varsInt, "MetalinkProcessing", DefaultMetalinkProcessing );

REGISTER_VAR_STR( varsStr, "PollerPreference", DefaultPollerPreference );
REGISTER_VAR_STR( varsStr, "ClientMonitor", DefaultClientMonitor );
Expand Down
4 changes: 2 additions & 2 deletions src/XrdCl/XrdClFile.hh
View file
Expand Up @@ -69,7 +69,7 @@ namespace XrdCl
Access::Mode mode,
ResponseHandler *handler,
uint16_t timeout = 0,
bool virtRedirector = false )
bool virtRedirector = true )
XRD_WARN_UNUSED_RESULT;

//------------------------------------------------------------------------
Expand All @@ -86,7 +86,7 @@ namespace XrdCl
OpenFlags::Flags flags,
Access::Mode mode = Access::None,
uint16_t timeout = 0,
bool virtRedirector = false )
bool virtRedirector = true )
XRD_WARN_UNUSED_RESULT;

//------------------------------------------------------------------------
Expand Down
7 changes: 4 additions & 3 deletions src/XrdCl/XrdClFileStateHandler.cc
View file
Expand Up @@ -448,6 +448,8 @@ namespace XrdCl
pOpenMode = mode;
pOpenFlags = flags;

pUseVirtRedirector = virtRedirector && pFileUrl->IsMetalink();

Message *msg;
ClientOpenRequest *req;
std::string path = pFileUrl->GetPathWithParams();
Expand All @@ -458,7 +460,7 @@ namespace XrdCl
req->options = flags | kXR_async | kXR_retstat;
req->dlen = path.length();
msg->Append( path.c_str(), path.length(), 24 );
msg->SetVirtualRedirections( virtRedirector );
msg->SetVirtualRedirections( pUseVirtRedirector );

XRootDTransport::SetDescription( msg );
OpenHandler *openHandler = new OpenHandler( this, handler );
Expand All @@ -469,7 +471,7 @@ namespace XrdCl
//--------------------------------------------------------------------------
// Register a virtual redirector
//--------------------------------------------------------------------------
if( virtRedirector )
if( pUseVirtRedirector )
{
RedirectorRegistry& registry = RedirectorRegistry::Instance();
XRootDStatus st = registry.Register( url, handler );
Expand All @@ -480,7 +482,6 @@ namespace XrdCl
params.loadBalancer = info;
params.hostList = list;
}
pUseVirtRedirector = virtRedirector;

Status st = MessageUtils::SendMessage( *pFileUrl, msg, openHandler, params );

Expand Down

0 comments on commit 365a383

Please sign in to comment.