Merge pull request #5 from xrootd/master

sync with xrootd master

docs/PreReleaseNotes.txt

@@ -6,14 +6,24 @@ Prerelease Notes
 ================
 
 + **New Features**
+  * [Server] Add new [no]rpipa option to xrd.network directive.
+  * [Server] Allow objectid's to be specified in the authorization file.
+  * [Server] Add new logging plugin interface.
   * [Server] Fixes #345 - add sid to TOD structure (ABI compliant).
   * [Server] Implement resource selection affinity (primarily for ssi).
 
 + **Major bug fixes**
+  * [Posix] Remove double unlock of a mutex.
+  * [Client] Serialize security protocol manager to allow MT loads.
   * [Authentication/sss] Fix dynamic id incompatability introduced in 4.0.
 
 + **Minor bug fixes**
+  * [Proxy]  Avoid SEGV when printing memory cache statistics.
+  * [Server] Avoid XrdNetIF static initialization issues.
+  * [Server] Honor DFS setting when forwarding operations.
+  * [Server] Make sure lockfile time is updated in deprecated runmodeold.
   * [Server] Fixes #344 - squash path before checking for static redirect.
   * [Server] Free Entity before replacing it from the cache (memleak).  
 
 + **Miscellaneous**
+  * [Apps] Add xrdacctest to the tools set to test access control databases.

docs/man/xrdcp.1

@@ -12,7 +12,7 @@ xrdcp - copy files
 [\fB--recursive\fR] [\fB--retry\fR \fItime\fR] [\fB--server\fR]
 [\fB--silent\fR] [\fB--sources\fR \fInum\fR] [\fB--streams\fR \fInum\fR]
 [\fB--tpc\fR \fIfirst\fR|\fIonly\fR] [\fB--verbose\fR] [\fB--version\fR]
-[\fB--xrate\fR \fIrate\fR] [\fB--metalink\fR]
+[\fB--xrate\fR \fIrate\fR]
 
 \fIlegacy options\fR: [\fB-adler\fR] [\fB-DS\fR\fIparm string\fR] [\fB-DI\fR\fIparm number\fR]
 [\fB-md5\fR] [\fB-np\fR] [\fB-OD\fR\fIcgi\fR] [\fB-OS\fR\fIcgi\fR] [\fB-x\fR]
@@ -133,11 +133,6 @@ displays version information and immediately exits.
 .RS 5
 [NOT YET IMPLEMENTED]
 
-.RE
-\fB-M\fR | \fB--metalink\fR
-.RS 5
-treats source as a metalink.
-
 limits the copy speed to the specified \fIrate\fB. The rate may be qualified
 with the letter \fBk\fR, \fBm\fR, or \fBg\fR to indicate kilo, mega, or giga
 bytes, respectively. The option only applies when the source or destination is
@@ -191,7 +186,7 @@ equivalent to "\fB--sources 12\fR".
 .SH OPERANDS
 \fIsource\fR
 .RS 5
-a local file, a local directory name suffixed by /, or
+a dash (i.e. \fB-\fR) indicating stanard in, a local file, a local directory name suffixed by /, or
 an xrootd URL in the form of
 .ce 1
 \fBxroot://[\fIuser\fB@\fR]\fIhost[\fB:\fIport\fR]\fB/\fIabsolutepath\fR
@@ -431,6 +426,21 @@ Number of unacknowledged probes before considering the connection dead
 (Linux only)
 .RE
 
+XRD_METALINKPROCESSING
+.RS 5
+Enable/Disable Metalink processing (enabled by default)
+.RE
+
+XRD_LOCALMETALINKFILE
+.RS 5
+Enable/Disable local Metalink file processing (by convention the following URL schema has to be used: root://localfile//path/filename.meta4) 
+.RE
+
+XRD_GLFNREDIRECTOR
+.RS 5
+The redirector will be used as a last resort if the GLFN tag is specified in a Metalink file.
+.RE
+
 .SH NOTES
 Documentation for all components associated with \fBxrdcp\fR can be found at
 http://xrootd.org/docs.html

docs/man/xrdfs.1

@@ -4,7 +4,7 @@ xrdfs - xrootd file and directory meta-data utility
 .SH SYNOPSIS
 .nf
 
-\fBxrdfs\fR \fIhost[:port]\fR \fI[command [args]]\fR
+\fBxrdfs\fR \fI[--no-cwd]\fR \fIhost[:port]\fR \fI[command [args]]\fR
 
 \fBcommand\fR: help, chmod, ls, locate, mkdir, mv, stat, statvfs, query, rm, rmdir,
            truncate, prepare, cat, tail, spaceinfo
@@ -18,6 +18,11 @@ Command help is available by invoking \fBxrdfs\fR with no command
 line options or parameters and then typing "help" in response to the
 input prompt.
 
+.SH OPTIONS
+\fB--no-cwd\fR
+.RS 3
+No CWD is being preset in interactive mode.
+
 .SH COMMANDS
 \fBchmod\fR \fIpath\fR \fI<user><group><other>\fR
 .RS 3

packaging/common/cmsd@.service

@@ -13,6 +13,7 @@ Type=simple
 Restart=on-abort
 RestartSec=0
 KillMode=control-group
+LimitNOFILE=65536
 
 [Install]
 RequiredBy=multi-user.target

packaging/common/frm_purged@.service

@@ -13,6 +13,7 @@ Type=simple
 Restart=on-abort
 RestartSec=0
 KillMode=control-group
+LimitNOFILE=65536
 
 [Install]
 RequiredBy=multi-user.target

packaging/common/frm_xfrd@.service

@@ -13,6 +13,7 @@ Type=simple
 Restart=on-abort
 RestartSec=0
 KillMode=control-group
+LimitNOFILE=65536
 
 [Install]
 RequiredBy=multi-user.target

packaging/common/xrootd@.service

@@ -13,6 +13,7 @@ Type=simple
 Restart=on-abort
 RestartSec=0
 KillMode=control-group
+LimitNOFILE=65536
 
 [Install]
 RequiredBy=multi-user.target

src/XProtocol/XProtocol.hh

@@ -294,6 +294,7 @@ enum XErrorCode {
    kXR_ChkLenErr,
    kXR_ChkSumErr,
    kXR_inProgress,
+   kXR_overQuota,
    kXR_noErrorYet = 10000
 };
 
@@ -757,8 +758,41 @@ static int mapError(int rc)
            case ETXTBSY:      return kXR_inProgress;
            case ENODEV:       return kXR_FSError;
            case EFAULT:       return kXR_ServerError;
+           case EDQUOT:       return kXR_overQuota;
            default:           return kXR_FSError;
           }
       }
+
+static int toErrno( int xerr )
+{
+    switch(xerr)
+       {case kXR_ArgInvalid:    return EINVAL;
+        case kXR_ArgMissing:    return EINVAL;
+        case kXR_ArgTooLong:    return ENAMETOOLONG;
+        case kXR_FileLocked:    return EDEADLK;
+        case kXR_FileNotOpen:   return EBADF;
+        case kXR_FSError:       return EIO;
+        case kXR_InvalidRequest:return EEXIST;
+        case kXR_IOError:       return EIO;
+        case kXR_NoMemory:      return ENOMEM;
+        case kXR_NoSpace:       return ENOSPC;
+        case kXR_NotAuthorized: return EACCES;
+        case kXR_NotFound:      return ENOENT;
+        case kXR_ServerError:   return ENOMSG;
+        case kXR_Unsupported:   return ENOSYS;
+        case kXR_noserver:      return EHOSTUNREACH;
+        case kXR_NotFile:       return ENOTBLK;
+        case kXR_isDirectory:   return EISDIR;
+        case kXR_Cancelled:     return ECANCELED;
+        case kXR_ChkLenErr:     return EDOM;
+        case kXR_ChkSumErr:     return EDOM;
+        case kXR_inProgress:    return EINPROGRESS;
+        case kXR_overQuota:     return EDQUOT;
+        default:                return ENOMSG;
+       }
+}
+
 };
+
+
 #endif

src/Xrd/XrdConfig.cc

@@ -60,6 +60,7 @@
 
 #include "XrdOuc/XrdOuca2x.hh"
 #include "XrdOuc/XrdOucEnv.hh"
+#include "XrdOuc/XrdOucLogging.hh"
 #include "XrdOuc/XrdOucSiteName.hh"
 #include "XrdOuc/XrdOucStream.hh"
 #include "XrdOuc/XrdOucUtils.hh"
@@ -215,19 +216,20 @@ int XrdConfig::Configure(int argc, char **argv)
 
    int retc, NoGo = 0, clPort = -1, optbg = 0;
    const char *temp;
-   char c, buff[512], *dfltProt, *libProt = 0, *logfn = 0;
+   char c, buff[512], *dfltProt, *libProt = 0;
    uid_t myUid = 0;
    gid_t myGid = 0;
    extern char *optarg;
    extern int optind, opterr;
+   struct XrdOucLogging::configLogInfo LogInfo;
    int pipeFD[2] = {-1, -1};
    const char *pidFN = 0;
    static const int myMaxc = 80;
    char **urArgv, *myArgv[myMaxc], argBuff[myMaxc*3+8];
    char *argbP = argBuff, *argbE = argbP+sizeof(argBuff)-4;
    char *ifList = 0;
-   int   myArgc = 1, bindArg = 1, urArgc = argc, i;
-   bool noV6, ipV4 = false, ipV6 = false, pureLFN = false;
+   int   myArgc = 1, urArgc = argc, i;
+   bool noV6, ipV4 = false, ipV6 = false;
 
 // Obtain the protocol name we will be using
 //
@@ -303,18 +305,12 @@ int XrdConfig::Configure(int argc, char **argv)
                        Usage(1);
                       }
                  break;
-       case 'k': if (!(bindArg = Log.logger()->ParseKeep(optarg)))
+       case 'k': if (!(LogInfo.keepV = Log.logger()->ParseKeep(optarg)))
                     {Log.Emsg("Config","Invalid -k argument -",optarg);
                      Usage(1);
                     }
                  break;
-       case 'l': if ((pureLFN = *optarg == '=')) optarg++;
-                 if (!*optarg)
-                    {Log.Emsg("Config", "Logfile name not specified.");
-                     Usage(1);
-                    }
-                 if (logfn) free(logfn);
-                 logfn = strdup(optarg);
+       case 'l': LogInfo.logArg = optarg;
                  break;
        case 'L': if (!*optarg)
                     {Log.Emsg("Config", "Protocol library path not specified.");
@@ -343,7 +339,7 @@ int XrdConfig::Configure(int argc, char **argv)
        case 'v': cerr <<XrdVSTRING <<endl;
                  _exit(0);
                  break;
-       case 'z': Log.logger()->setHiRes();
+       case 'z': LogInfo.hiRes = true;
                  break;
 
        default: if (optopt == '-' && *(argv[optind]+1) == '-')
@@ -401,25 +397,22 @@ int XrdConfig::Configure(int argc, char **argv)
    if (optbg)
    {
 #ifdef WIN32
-      XrdOucUtils::Undercover(&Log, !logfn);
+      XrdOucUtils::Undercover(&Log, !LogInfo.logArg);
 #else
       if (pipe( pipeFD ) == -1)
          {Log.Emsg("Config", errno, "create a pipe"); exit(17);}
-      XrdOucUtils::Undercover(Log, !logfn, pipeFD);
+      XrdOucUtils::Undercover(Log, !LogInfo.logArg, pipeFD);
 #endif
    }
 
 // Bind the log file if we have one
 //
-   if (logfn)
-      {char *lP;
-       if (!pureLFN && !(logfn = XrdOucUtils::subLogfn(Log,myInsName,logfn)))
-          _exit(16);
+   if (LogInfo.logArg)
+      {LogInfo.xrdEnv = &theEnv;
+       LogInfo.iName  = myInsName;
+       LogInfo.cfgFn  = ConfigFN;
+       if (!XrdOucLogging::configLog(Log, LogInfo)) _exit(16);
        Log.logger()->AddMsg(XrdBANNER);
-       if (Log.logger()->Bind(logfn, bindArg)) exit(19);
-       if ((lP = rindex(logfn,'/'))) {*(lP+1) = '\0'; lP = logfn;}
-          else lP = (char *)"./";
-       XrdOucEnv::Export("XRDLOGDIR", lP);
       }
 
 // Get the full host name. In theory, we should always get some kind of name.
@@ -545,12 +538,11 @@ int XrdConfig::Configure(int argc, char **argv)
    temp = (NoGo ? " initialization failed." : " initialization completed.");
    sprintf(buff, "%s:%d", myInstance, PortTCP);
    Log.Say("------ ", buff, temp);
-   if (logfn)
+   if (LogInfo.logArg)
       {strcat(buff, " running ");
        retc = strlen(buff);
        XrdSysUtils::FmtUname(buff+retc, sizeof(buff)-retc);
        Log.logger()->AddMsg(buff);
-       free(logfn);
       }
    return NoGo;
 }
@@ -855,8 +847,8 @@ int XrdConfig::setFDL()
 
 // Set the limit to the maximum allowed
 //
-   if (rlim.rlim_max == RLIM_INFINITY) rlim.rlim_max = maxFD;
-   rlim.rlim_cur = rlim.rlim_max;
+   if (rlim.rlim_max == RLIM_INFINITY) rlim.rlim_cur = maxFD;
+      else rlim.rlim_cur = rlim.rlim_max;
 #if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_5))
    if (rlim.rlim_cur > OPEN_MAX) rlim.rlim_max = rlim.rlim_cur = OPEN_MAX;
 #endif
@@ -1220,6 +1212,7 @@ int XrdConfig::xbuf(XrdSysError *eDest, XrdOucStream &Config)
    Purpose:  To parse directive: network [wan] [[no]keepalive] [buffsz <blen>]
                                          [kaparms parms] [cache <ct>] [[no]dnr]
                                          [routes <rtype> [use <ifn1>,<ifn2>]]
+                                         [[no]rpipa]
 
              <rtype>: split | common | local
 
@@ -1230,6 +1223,7 @@ int XrdConfig::xbuf(XrdSysError *eDest, XrdOucStream &Config)
              <ct>      Seconds to cache address to name resolutions.
              [no]dnr   do [not] perform a reverse DNS lookup if not needed.
              routes    specifies the network configuration (see reference)
+             [no]rpipa do [not] resolve private IP addresses.
 
    Output: 0 upon success or !0 upon failure.
 */
@@ -1238,6 +1232,7 @@ int XrdConfig::xnet(XrdSysError *eDest, XrdOucStream &Config)
 {
     char *val;
     int  i, n, V_keep = -1, V_nodnr = 0, V_iswan = 0, V_blen = -1, V_ct = -1;
+    int  v_rpip = -1;
     long long llp;
     struct netopts {const char *opname; int hasarg; int opval;
                            int *oploc;  const char *etxt;}
@@ -1251,6 +1246,8 @@ int XrdConfig::xnet(XrdSysError *eDest, XrdOucStream &Config)
         {"dnr",        0, 0, &V_nodnr,  "option"},
         {"nodnr",      0, 1, &V_nodnr,  "option"},
         {"routes",     3, 1, 0,         "routes"},
+        {"rpipa",      0, 1, &v_rpip,   "rpipa"},
+        {"norpipa",    0, 0, &v_rpip,   "norpipa"},
         {"wan",        0, 1, &V_iswan,  "option"}
        };
     int numopts = sizeof(ntopts)/sizeof(struct netopts);
@@ -1322,6 +1319,7 @@ int XrdConfig::xnet(XrdSysError *eDest, XrdOucStream &Config)
         }
 
      if (V_ct >= 0) XrdNetAddr::SetCache(V_ct);
+     if (v_rpip >= 0) XrdInet::netIF.SetRPIPA(v_rpip != 0);
      return 0;
 }
 

src/Xrd/XrdScheduler.cc

@@ -28,9 +28,11 @@
 /******************************************************************************/
 
 #include <errno.h>
+#include <fcntl.h>
 #include <signal.h>
 #include <stdio.h>
 #include <sys/resource.h>
+#include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 #ifdef __APPLE__
@@ -119,20 +121,41 @@ XrdScheduler::XrdScheduler(XrdSysError *eP, XrdOucTrace *tP,
 //
 #if defined(__linux__) && defined(RLIMIT_NPROC)
 
+// First determine the absolute maximum we can have
+//
+   rlim_t theMax = MAX_SCHED_PROCS;
+   int pdFD, rdsz;
+   if ((pdFD = open("/proc/sys/kernel/pid_max", O_RDONLY)) >= 0)
+      {char pmBuff[32];
+       if ((rdsz = read(pdFD, pmBuff, sizeof(pmBuff))) > 0)
+          {rdsz = atoi(pmBuff);
+           if (rdsz < 16384) theMax = 16384; // This is unlikely
+              else if (rdsz < MAX_SCHED_PROCS)
+                      theMax = static_cast<rlim_t>(rdsz-2000);
+          }
+       close(pdFD);
+      }
+
 // Get the resource thread limit and set to maximum. In Linux this may be -1
 // to indicate useless infnity, so we have to come up with a number, sigh.
 //
    if (!getrlimit(RLIMIT_NPROC, &rlim))
-      {     if (rlim.rlim_max == RLIM_INFINITY) rlim.rlim_cur = MAX_SCHED_PROCS;
-       else if (rlim.rlim_cur  < rlim.rlim_max) rlim.rlim_cur = rlim.rlim_max;
-       if (rlim.rlim_cur != rlim.rlim_max) setrlimit(RLIMIT_NPROC, &rlim);
+      {if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max > theMax)
+          {rlim.rlim_cur = theMax;
+           setrlimit(RLIMIT_NPROC, &rlim);
+          } else {
+           if (rlim.rlim_cur != rlim.rlim_max)
+              {rlim.rlim_cur = rlim.rlim_max;
+               setrlimit(RLIMIT_NPROC, &rlim);
+              }
+          }
       }
 
 // Readjust our internal maximum to be the actual maximum
 //
    if (!getrlimit(RLIMIT_NPROC, &rlim))
-      {if (rlim.rlim_cur == RLIM_INFINITY || rlim.rlim_cur > MAX_SCHED_PROCS)
-          max_Workers = MAX_SCHED_PROCS;
+      {if (rlim.rlim_cur == RLIM_INFINITY || rlim.rlim_cur > theMax)
+               max_Workers = static_cast<int>(theMax);
           else max_Workers = static_cast<int>(rlim.rlim_cur);
       }
 #endif