Merge branch 'master' into xrdcl_lib_only_py

.gitlab-ci.yml

@@ -33,6 +33,35 @@ stages:
   tags:
     - docker_node
 
+build:centos8:
+  stage: build:rpm
+  image: centos:8
+  script:
+    - dnf install -y epel-release
+    - dnf install --nogpg -y gcc-c++ rpm-build which tar dnf-plugins-core git
+    - dnf config-manager --set-enabled PowerTools
+    - cd packaging
+    - ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_with_xrdclhttp 1"
+    - dnf builddep --nogpgcheck -y *.src.rpm
+    - mkdir RPMS
+    - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_tests 1" --define "_with_xrdclhttp 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm
+    - cd ..
+    - mkdir centos-8
+    - cp packaging/*.src.rpm centos-8
+    - cp packaging/RPMS/*    centos-8
+  artifacts:
+    expire_in: 1 day
+    paths:
+    - centos-8/
+  tags:
+    - docker_node
+  only:
+    - master
+    - /^stable-.*$/
+  except:
+    - tags
+    - schedules
+
 build:cc7:
   stage: build:rpm
   image: gitlab-registry.cern.ch/linuxsupport/cc7-base
@@ -261,6 +290,38 @@ build:macosx:
     - schedules
   allow_failure: true
 
+release:centos8-x86_64:
+  stage: build:rpm
+  image: centos:8
+  script:
+    - dnf install -y epel-release
+    - dnf install --nogpg -y gcc-c++ rpm-build which tar dnf-plugins-core git python-macros
+    - dnf config-manager --set-enabled PowerTools
+    - dnf install -y cppunit-devel
+    - mkdir centos-8-x86_64
+    - git archive --prefix=xrootd-${CI_COMMIT_TAG#"v"}/ --format=tar tags/$CI_COMMIT_TAG | gzip -9fn > xrootd-${CI_COMMIT_TAG#"v"}.tar.gz
+    - mv xrootd-${CI_COMMIT_TAG#"v"}.tar.gz centos-8-x86_64
+    - cd packaging/
+    - git checkout tags/${CI_COMMIT_TAG}
+    - ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" -D "dist .el8"
+    - dnf builddep --nogpgcheck -y *.src.rpm
+    - mkdir RPMS
+    - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" -D "dist .el8" *.src.rpm
+    - cd ..
+    - cp packaging/RPMS/*.rpm centos-8-x86_64
+    - cp packaging/*src.rpm   centos-8-x86_64
+  artifacts:
+    expire_in: 1 day
+    paths:
+    - centos-8-x86_64/
+  tags:
+    - docker_node
+  only:
+    - web
+  except:
+    - branches
+
+
 release:cc7-x86_64:
   stage: build:rpm
   image: gitlab-registry.cern.ch/linuxsupport/cc7-base
@@ -351,7 +412,8 @@ release:pypi:
   image: gitlab-registry.cern.ch/linuxsupport/cc7-base
   script:
     - yum install --nogpg -y git python-pip which
-    - pip install setuptools --upgrade
+    - pip install --upgrade pip
+    - pip install setuptools wheel --upgrade
     - cp packaging/wheel/* .
     - ./publish.sh
   artifacts:
@@ -382,6 +444,40 @@ publish:pypi:
   allow_failure: true
 
 
+weekly:centos8:
+  stage: build:rpm
+  image: centos:8
+  script:
+    - dnf install -y epel-release
+    - dnf install --nogpg -y gcc-c++ rpm-build which tar dnf-plugins-core git python-macros
+    - dnf config-manager --set-enabled PowerTools
+    - dnf install -y cppunit-devel
+    - xrootd_version=$(git for-each-ref --sort=taggerdate --format '%(refname)' refs/tags | grep '^refs/tags/v' | grep -v 'rc.*$' | grep -v 'CERN$' | sed -e '$!d')
+    - xrootd_version=${xrootd_version:11}
+    - short_hash=$(git rev-parse --verify HEAD | awk '{print substr($0, 0, 8)}')
+    - a=( ${xrootd_version//./ } )
+    - ((a[2]++)) || true
+    - experimental_version="${a[0]}.${a[1]}.${a[2]}-0.experimental."${CI_PIPELINE_ID}.$short_hash
+    - cd packaging/
+    - ./makesrpm.sh --version $experimental_version --define "_with_python3 1" --define "_with_xrdclhttp 1"
+    - dnf builddep --nogpgcheck -y *.src.rpm
+    - mkdir RPMS
+    - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_tests 1" --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm
+    - cd ..
+    - mkdir epel-8/
+    - cp packaging/*.src.rpm epel-8
+    - cp packaging/RPMS/* epel-8
+  artifacts:
+    expire_in: 1 day
+    paths:
+    - epel-8/
+  tags:
+    - docker_node
+  only:
+    - schedules
+  except:
+    - tags
+
 weekly:cc7:
   stage: build:rpm
   image: gitlab-registry.cern.ch/linuxsupport/cc7-base
@@ -538,7 +634,7 @@ publish:rhel:
   script:
     - yum install --nogpg -y sssd-client sudo createrepo
     - prefix=/eos/project/s/storage-ci/www/xrootd
-    - "for platform in cc-7 slc-6 fc-rawhide fc-30 fc-29 fc-28; do 
+    - "for platform in centos-8 cc-7 slc-6 fc-rawhide fc-30 fc-29 fc-28; do 
          repo=$prefix/${CI_COMMIT_REF_NAME}/$platform/x86_64
          path=$repo/$(date +'%Y%m%d');
          sudo -u stci -H mkdir -p $path;
@@ -587,7 +683,7 @@ publish:rhel:release:
     - yum install --nogpg -y sssd-client sudo createrepo
     - prefix=/eos/project/s/storage-ci/www/xrootd
     - tarball=cc-7-x86_64/xrootd-*.tar.gz
-    - "for platform in cc-7-x86_64 slc-6-x86_64; do 
+    - "for platform in centos-8-x86_64 cc-7-x86_64 slc-6-x86_64; do 
          path=$prefix/release/$platform/$CI_COMMIT_TAG/;
          sudo -u stci -H mkdir -p $path/{source,tarball};
          sudo -u stci -H cp $platform/*.rpm $path;
@@ -632,7 +728,7 @@ publish:weekly:
   script:
     - yum install --nogpg -y sssd-client sudo createrepo
     - prefix=/eos/project/s/storage-ci/www/xrootd
-    - "for platform in epel-7 epel-6; do 
+    - "for platform in epel-8 epel-7 epel-6; do 
          path=$prefix/experimental/$platform/x86_64/;
          sudo -u stci -H mkdir -p $path;
          sudo -u stci -H cp $platform/* $path;

CMakeLists.txt

@@ -1,11 +1,7 @@
 #-------------------------------------------------------------------------------
 # Project description
 #-------------------------------------------------------------------------------
-cmake_minimum_required( VERSION 2.6 )
-
-IF(CMAKE_VERSION VERSION_GREATER "2.8.12")
-   CMAKE_POLICY(SET CMP0022 OLD)
-ENDIF()
+cmake_minimum_required( VERSION 3.1 )
 
 project( XRootD )
 
@@ -17,10 +13,6 @@ if(NOT (CMAKE_VERSION VERSION_LESS "3.1"))
   cmake_policy(SET CMP0054 OLD)
 endif()
 
-if(CMAKE_VERSION VERSION_LESS "3.0")
-  SET(CMAKE_DL_LIBS "-ldl")
-endif()
-
 include( XRootDUtils )
 CheckBuildDirectory()
 

bindings/python/setup.py.in

@@ -60,7 +60,7 @@ print('XRootD src include dir:', xrdsrcincdir)
 print('XRootD bin include dir:', xrdbinincdir)
 print('Version:               ', version)
 
-setup( name             = 'pyxrootd',
+setup( name             = 'xrootd',
        version          = version,
        author           = 'XRootD Developers',
        author_email     = 'xrootd-dev@slac.stanford.edu',

bindings/python/setup_pypi.py

@@ -42,7 +42,7 @@ def getincdir():
 print 'XRootD include dir: ', xrdincdir
 print 'Version:            ', version
 
-setup( name             = 'pyxrootd',
+setup( name             = 'xrootd',
        version          = version,
        author           = 'XRootD Developers',
        author_email     = 'xrootd-dev@slac.stanford.edu',

bindings/python/setup_standalone.py

@@ -42,7 +42,7 @@
 print('XRootD include dir: ', xrdincdir)
 print('Version:            ', version)
 
-setup( name             = 'pyxrootd',
+setup( name             = 'xrootd',
        version          = version,
        author           = 'XRootD Developers',
        author_email     = 'xrootd-dev@slac.stanford.edu',

cmake/FindLibUuid.cmake

@@ -1,25 +1,56 @@
-# This code sets the following variables:
-# 
-#  LIBUUID_FOUND               True if libuuid got found
-#  LIBUUID_INCLUDE_DIRS        Location of libuuid headers
-#  LIBUUID_LIBRARIES           List of libraries to use libuuid 
+#.rst:
+# Finduuid
+# -----------
+#
+# Find libuuid, DCE compatible Universally Unique Identifier library.
+#
+# Result Variables
+# ^^^^^^^^^^^^^^^^
+#
+# This module will set the following variables in your project:
+#
+# ``UUID_FOUND``
+#   True if libuuid has been found.
+# ``UUID_INCLUDE_DIRS``
+#   Where to find uuid/uuid.h.
+# ``UUID_LIBRARIES``
+#   The libraries to link against to use libuuid.
+#
+# Obsolete variables
+# ^^^^^^^^^^^^^^^^^^
+#
+# The following variables may also be set, for backwards compatibility:
+#
+# ``UUID_LIBRARY``
+#   where to find the libuuid library (same as UUID_LIBRARIES).
+# ``UUID_INCLUDE_DIR``
+#   where to find the uuid/uuid.h header (same as UUID_INCLUDE_DIRS).
 
-FIND_PATH(LIBUUID_INCLUDE_DIRS uuid/uuid.h)
-FIND_LIBRARY(LIBUUID_LIBRARIES uuid)
+include(CheckCXXSymbolExists)
+include(CheckLibraryExists)
+include(FindPackageHandleStandardArgs)
 
-IF (LIBUUID_LIBRARIES AND LIBUUID_INCLUDE_DIRS)       
-	SET(LIBUUID_FOUND 1)
-	IF (NOT LibUuid_FIND_QUIETLY)
-		MESSAGE(STATUS "Found libuuid: ${LIBUUID_LIBRARIES}")
-	ENDIF ( NOT LibUuid_FIND_QUIETLY )
-ELSE (LIBUUID_LIBRARIES AND LIBUUID_INCLUDE_DIRS)     
-	IF (LibUuid_FIND_REQUIRED)
-		MESSAGE(SEND_ERROR "Could NOT find libuuid")
-	ELSE (LibUuid_FIND_REQUIRED)
-		IF (NOT LIBUUID_FIND_QUIETLY)
-			MESSAGE(STATUS "Could NOT find libuuid")     
-		ENDIF (NOT LIBUUID_FIND_QUIETLY)
-	ENDIF (LibUuid_FIND_REQUIRED)
-ENDIF (LIBUUID_LIBRARIES AND LIBUUID_INCLUDE_DIRS)
+if(NOT UUID_INCLUDE_DIR)
+  find_path(UUID_INCLUDE_DIR uuid/uuid.h)
+endif()
 
-MARK_AS_ADVANCED(LIBUUID_LIBRARIES LIBUUID_INCLUDE_DIRS)
+if(EXISTS UUID_INCLUDE_DIR)
+  set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR})
+  set(CMAKE_REQUIRED_INCLUDES ${UUID_INCLUDE_DIRS})
+  check_cxx_symbol_exists("uuid_generate_random" "uuid/uuid.h" _uuid_header_only)
+endif()
+
+if(NOT _uuid_header_only AND NOT UUID_LIBRARY)
+  check_library_exists("uuid" "uuid_generate_random" "" _have_libuuid)
+  if(_have_libuuid)
+    set(UUID_LIBRARY "uuid")
+    set(UUID_LIBRARIES ${UUID_LIBRARY})
+  endif()
+endif()
+
+unset(CMAKE_REQUIRED_INCLUDES)
+unset(_uuid_header_only)
+unset(_have_libuuid)
+
+find_package_handle_standard_args(uuid DEFAULT_MSG UUID_INCLUDE_DIR)
+mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY)

cmake/XRootDOSDefs.cmake

@@ -66,6 +66,13 @@ if( CMAKE_COMPILER_IS_GNUCXX )
       set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=address-of-packed-member" )
   endif()
 
+  # we need at least gcc 4.6 in order to support crc32c as the compiler needs
+  # to support constexpr
+  if( GCC_VERSION VERSION_LESS 4.6 )
+      set( ENABLE_CRC32C FALSE CACHE BOOL "Disable CRC32C submodule." )
+      message( WARNING "Disabling CRC32C submodule, insufficient GCC version!" )
+  endif()
+
 endif()
 
 #-------------------------------------------------------------------------------

docs/PreReleaseNotes.txt

@@ -6,6 +6,11 @@ Prerelease Notes
 ================
 
 + **New Features**
+  * **[Server]** Provide a way to see the actual server config when running.
+  * **[Server]i** Provide fallback when an IPv6 address is missing a ptr record.
+  * **[Server]** Allow redirect differentiation for delegated and undelegated TPC.
+  * **[Xcache]** Provide monitoring summary statistics.
+  * **[All]** Add hardware assisted CRC32C checksum based on Mark Adler's code.
   * **[All]** Implement new thread-safe strerror() replacement.
   * **[Server]** Allow xrootd.fslib plugin to be generally stacked.
   * **[Server]** Add getfile and putfile SFS interface.
@@ -32,11 +37,20 @@ Prerelease Notes
   * **[Server]** Add additional fields to he SecEntity structure (breaks ABI).
 
 + **Major bug fixes**
+  * **[Proxy]** Remove offending CGI elements before passing URL.
+  * **[Server]** Accommodate changes in epoll handling in CentOS 7.10 plus.
   * **[SSI]** Fix double delete due to extraneous code addition.
 
 + **Minor bug fixes**
+  * **[Server]** Correct action determination for debug message to avoid strife.
+  * **[Server]** Use correct mechanism to determine DNS registration.
 
 + **Miscellaneous**
+  * **[Server]** Turn off IP address caching when dynamic DNS is enabled.
+  * **[Server]** Log unusual stat() errors affecting file availability.
+  * **[Server]** Manage reference counters in a more timely fashion.
+  * **[Server]** Fallback using known DNS name when reverse translation fails.
+  * **[Xcache]** Provide config backward compatibility for "pss.cachelib" directive.
   * **[All]** Use thread-safe strerror() replacement, part 1.
   * **[Server]** Provide interface for checksum handler to report progress.
   * **[Server]** Improve SFS wrapping and add "how to" documentation.

docs/ReleaseNotes.txt

@@ -5,6 +5,8 @@ XRootD
 Release Notes
 =============
 
++ **Miscellaneous**
+  **[XCache]** Switch from usage of XrdFileCache prefix to XrdPfc.
 
 --------------
 Version 4.11.0

docs/man/xrdcp.1

@@ -14,7 +14,7 @@ xrdcp - copy files
 [\fB--tpc\fR [\fBdelegate\fR] \fBfirst\fR|\fBonly\fR] [\fB--verbose\fR]
 [\fB--version\fR] [\fB--xrate\fR \fIrate\fR] [\fB--zip\fR \fIfile\fR]
 [\fB--dynamic-src\fR] [\fB--infiles\fR \fIfn\fR] [\fB--path\fR]
-[\fB--parallel\fR \fIn\fR] [\fB--allow-http\fR]
+[\fB--parallel\fR \fIn\fR] [\fB--allow-http\fR] [\fB--xattr\fR]
 
 \fIlegacy options\fR: [\fB-adler\fR] [\fB-DS\fR\fIparm string\fR] [\fB-DI\fR\fIparm number\fR]
 [\fB-md5\fR] [\fB-np\fR] [\fB-OD\fR\fIcgi\fR] [\fB-OS\fR\fIcgi\fR] [\fB-x\fR]
@@ -172,6 +172,11 @@ number of copy jobs to be run simultaneously
 .RS 5
 allow HTTP as source or destination protocol. Requires the XrdClHttp client plugin
 
+.RE
+\fB--xattr\fR
+.RS 5
+preserve extended attributes
+
 .SH LEGACY OPTIONS
 Legacy options are provided for backward compatability. These are now
 deprecated and should be avoided.
@@ -512,6 +517,12 @@ If set to 1 XRootD client will preserve tired/triedrc cgi opaque info for kXR_lo
 if set to 0 XRootD client will treat kXR_locate as any other passive request.
 .RE
 
+XRD_PRESERVEXATTRS
+.RS 5
+If set to 1 (default) xrdcp will preverve file extended attribues,
+if set to 0 file extended attributes wont be preserved.
+.RE
+
 .SH RETURN CODES
 .RE
 \fB50\fR  : generic error (e.g. config, internal, data, OS, command line option)

packaging/common/xrootd-filecache-clustered.cfg

@@ -67,7 +67,7 @@ all.role server
 # For xrootd, load the proxy plugin and the disk caching plugin.
 #
 ofs.osslib   libXrdPss.so
-pss.cachelib libFileCache.so
+pss.cachelib libXrdPfc.so
 
 # The server needs to write to disk, stage not relevant
 #

packaging/common/xrootd-filecache-standalone.cfg

@@ -19,8 +19,8 @@ all.export  /test/
 
 # Load the proxy plugin and the disk caching plugin.
 #
-ofs.osslib  libXrdPss.so
-pss.cachelib libXrdFileCache.so
+ofs.osslib   libXrdPss.so
+pss.cachelib libXrdPfc.so
 
 # Tell the proxy where the data is coming from (arbitrary).
 #