From 646aab7d2298df702fc7d8994d1e5e23835ec118 Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 2 May 2023 14:12:01 +0200 Subject: [PATCH 1/9] Update XRootD to 5.x in CMSSW 10_6_X --- rpm-preamble.file | 7 +++++ xrootd.spec | 79 +++++++++++++++++++++++++++-------------------- 2 files changed, 53 insertions(+), 33 deletions(-) diff --git a/rpm-preamble.file b/rpm-preamble.file index 298c7203770..04fe5313ce8 100644 --- a/rpm-preamble.file +++ b/rpm-preamble.file @@ -262,3 +262,10 @@ if [ -f $RPM_INSTALL_PREFIX/%{pkgrel}/etc/profile.d/.autodependencies ] ; then \ %{relocateConfig}etc/profile.d/dependencies-setup.sh \ %{relocateConfig}etc/profile.d/dependencies-setup.csh \ fi + +%define relocatePy2SitePackages \ + for py in $(grep -RlI -m1 '^#\!.*python' %{i}/${PYTHON_LIB_SITE_PACKAGES} | grep -v '\.pyc') ; do \ + lnum=$(grep -n -m1 '^#\!.*python' $py | sed 's|:.*||') \ + sed -i -e "${lnum}c#!/usr/bin/env python" $py \ + done + diff --git a/xrootd.spec b/xrootd.spec index cdcb5ec2fde..637bd834233 100644 --- a/xrootd.spec +++ b/xrootd.spec @@ -1,57 +1,70 @@ -### RPM external xrootd 4.8.5 +### RPM external xrootd 5.5.3 ## INITENV +PATH LD_LIBRARY_PATH %i/lib64 -## INITENV +PATH PYTHON27PATH %{i}/${PYTHON_LIB_SITE_PACKAGES} +## INITENV +PATH PYTHONPATH %{i}/${PYTHON_LIB_SITE_PACKAGES} -%define tag 72b7fd30afaec0328b4f3693ee60346f8e13802e -%define branch cms/v%{realversion} -%define github_user cms-externals -Source: git+https://github.com/%github_user/xrootd.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}.tgz +%define strip_files %i/lib +%define tag %{realversion} +%define branch master +%define github_user xrootd +Source: git+https://github.com/%github_user/xrootd.git?obj=%{branch}/v%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}.tgz -BuildRequires: cmake -Requires: zlib +BuildRequires: cmake gmake autotools +Requires: zlib libuuid curl davix +Requires: python py2-setuptools +Requires: libxml2 +#Requires: scitokens-cpp Requires: openssl -Requires: python + +%define soext so +%ifarch darwin +%define soext dylib +%endif %prep %setup -n %n-%{realversion} - -# need to fix these from xrootd git -perl -p -i -e 's|^#!.*perl(.*)|#!/usr/bin/env perl$1|' src/XrdMon/cleanup.pl -perl -p -i -e 's|^#!.*perl(.*)|#!/usr/bin/env perl$1|' src/XrdMon/loadRTDataToMySQL.pl -perl -p -i -e 's|^#!.*perl(.*)|#!/usr/bin/env perl$1|' src/XrdMon/xrdmonCollector.pl -perl -p -i -e 's|^#!.*perl(.*)|#!/usr/bin/env perl$1|' src/XrdMon/prepareMySQLStats.pl -perl -p -i -e 's|^#!.*perl(.*)|#!/usr/bin/env perl$1|' src/XrdMon/xrdmonCreateMySQL.pl -perl -p -i -e 's|^#!.*perl(.*)|#!/usr/bin/env perl$1|' src/XrdMon/xrdmonLoadMySQL.pl -perl -p -i -e 's|^#!.*perl(.*)|#!/usr/bin/env perl$1|' src/XrdMon/xrdmonPrepareStats.pl +sed -i -e 's|UUID REQUIRED|UUID |' cmake/XRootDFindLibs.cmake %build -mkdir build -cd build # By default xrootd has perl, fuse, krb5, readline, and crypto enabled. # libfuse and libperl are not produced by CMSDIST. -cmake ../ \ + +rm -rf ../build; mkdir ../build; cd ../build + +PYTHONPATH=%{i}/${PYTHON_LIB_SITE_PACKAGES}:$PYTHONPATH \ +cmake ../%n-%{realversion} \ -DCMAKE_INSTALL_PREFIX=%{i} \ - -DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT} \ - -DZLIB_ROOT:PATH=${ZLIB_ROOT} \ - -DENABLE_PYTHON=FALSE \ + -DUSER_VERSION=%{realversion} \ + -DCMAKE_BUILD_TYPE=Release \ + -DFORCE_ENABLED=ON \ -DENABLE_FUSE=FALSE \ + -DENABLE_VOMS=FALSE \ + -DXRDCL_ONLY=TRUE \ -DENABLE_KRB5=TRUE \ - -DENABLE_READLINE=FALSE \ + -DENABLE_READLINE=TRUE \ -DENABLE_CRYPTO=TRUE \ -DCMAKE_SKIP_RPATH=TRUE \ -DENABLE_PYTHON=TRUE \ - -DCMAKE_PREFIX_PATH="${PYTHON_ROOT}" + -DENABLE_HTTP=TRUE \ + -DENABLE_SCITOKENS=OFF \ + -DXRD_PYTHON_REQ_VERSION=2 \ + -DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT} \ + -DCMAKE_CXX_FLAGS="-I${LIBUUID_ROOT}/include -I${DAVIX_ROOT}/include" \ + -DUUID_INCLUDE_DIR="${LIBUUID_ROOT}/include" \ + -DUUID_LIBRARY="${LIBUUID_ROOT}/lib64/libuuid.%{soext}" \ + -DSCITOKENS_CPP_DIR="${SCITOKENS_CPP_ROOT}" \ + -DCMAKE_PREFIX_PATH="${ZLIB_ROOT};${PYTHON_ROOT};${LIBXML2_ROOT};${LIBUUID_ROOT};${SCITOKENS_CPP_ROOT};${CURL_ROOT};${DAVIX_ROOT}" -# Use makeprocess macro, it uses compiling_processes defined by -# build configuration file or build argument +PYTHONPATH=%{i}/${PYTHON_LIB_SITE_PACKAGES}:$PYTHONPATH \ make %makeprocesses VERBOSE=1 %install -cd build +cd ../build +mkdir -p %{i}/${PYTHON_LIB_SITE_PACKAGES} +PYTHONPATH=%{i}/${PYTHON_LIB_SITE_PACKAGES}:$PYTHONPATH \ make install -cd .. - -%define strip_files %i/lib -%define keep_archives true +%{relocatePy2SitePackages} +%post +%{relocateConfig}bin/xrootd-config +%{relocateConfig}${PYTHON_LIB_SITE_PACKAGES}/xrootd-%{realvesion}-*.egg/EGG-INFO/SOURCES.txt From ffa7bb3cd6d126b2e5f675b70ba52801bfee2b8e Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 2 May 2023 15:41:46 +0200 Subject: [PATCH 2/9] Add scitokens --- scitokens-cpp.patch | 46 +++++++++++++++++++++++++++++++++++++++++++++ scitokens-cpp.spec | 25 ++++++++++++++++++++++++ xrootd.spec | 3 +-- 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 scitokens-cpp.patch create mode 100644 scitokens-cpp.spec diff --git a/scitokens-cpp.patch b/scitokens-cpp.patch new file mode 100644 index 00000000000..b63f9f32339 --- /dev/null +++ b/scitokens-cpp.patch @@ -0,0 +1,46 @@ +--- CMakeLists.txt.orig 2021-09-14 22:52:06.000000000 +0200 ++++ CMakeLists.txt 2022-02-01 11:46:11.698151800 +0100 +@@ -25,27 +25,23 @@ + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g") + + +-if( APPLE ) +- + find_package( OpenSSL REQUIRED ) + find_package( Sqlite3 REQUIRED ) ++ ++if( APPLE ) ++ + set(LIBCRYPTO_INCLUDE_DIRS ${OPENSSL_INCLUDE_DIR}) + set(LIBCRYPTO_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY}) + set(CMAKE_MACOSX_RPATH ON) + + elseif( UNIX ) + +-include (FindPkgConfig) +-pkg_check_modules(LIBCRYPTO REQUIRED libcrypto) +-pkg_check_modules(OPENSSL REQUIRED openssl) +-pkg_check_modules(SQLITE REQUIRED sqlite3) +- + # pkg_check_modules fails to return an absolute path on RHEL7. Set the + # link directories accordingly. + link_directories(${OPENSSL_LIBRARY_DIRS} ${LIBCRYPTO_LIBRARY_DIRS}) + endif() + +-include_directories( "${PROJECT_SOURCE_DIR}" ${JWT_CPP_INCLUDES} ${CURL_INCLUDES} ${OPENSSL_INCLUDE_DIRS} ${LIBCRYPTO_INCLUDE_DIRS} ${SQLITE_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS} ) ++include_directories( "${PROJECT_SOURCE_DIR}" ${JWT_CPP_INCLUDES} ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} ${LIBCRYPTO_INCLUDE_DIRS} ${SQLITE_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS} ) + + add_library(SciTokens SHARED src/scitokens.cpp src/scitokens_internal.cpp src/scitokens_cache.cpp) + target_link_libraries(SciTokens ${OPENSSL_LIBRARIES} ${LIBCRYPTO_LIBRARIES} ${CURL_LIBRARIES} ${SQLITE_LIBRARIES} ${UUID_LIBRARIES}) +--- vendor/jwt-cpp/CMakeLists.txt.orig 2023-05-02 15:31:50.008285546 +0200 ++++ vendor/jwt-cpp/CMakeLists.txt 2023-05-02 15:32:36.039856604 +0200 +@@ -17,6 +17,8 @@ + find_package(picojson REQUIRED) + endif() + ++include_directories(${OPENSSL_INCLUDE_DIRS}) ++ + add_library(jwt-cpp INTERFACE) + add_library(jwt-cpp::jwt-cpp ALIAS jwt-cpp) + + diff --git a/scitokens-cpp.spec b/scitokens-cpp.spec new file mode 100644 index 00000000000..672003ac910 --- /dev/null +++ b/scitokens-cpp.spec @@ -0,0 +1,25 @@ +### RPM external scitokens-cpp 0.7.0 +Source: https://github.com/scitokens/%{n}/archive/refs/tags/v%{realversion}.tar.gz + +BuildRequires: cmake gmake +Requires: libuuid curl sqlite openssl +Patch0: scitokens-cpp + +%prep +%setup -n %{n}-%{realversion} +%patch0 -p0 +sed -i -e 's/ -Werror//' CMakeLists.txt + +%build +rm -rf ../build; mkdir ../build; cd ../build +cmake ../%{n}-%{realversion} \ + -DCMAKE_INSTALL_PREFIX=%{i} \ + -DCMAKE_BUILD_TYPE=Release \ + -DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT} \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ + -DCMAKE_PREFIX_PATH="${CURL_ROOT};${LIBUUID_ROOT};${SQLITE_ROOT}" + +%install +cd ../build +make %{makeprocesses} +make install diff --git a/xrootd.spec b/xrootd.spec index 637bd834233..68bdd24aa01 100644 --- a/xrootd.spec +++ b/xrootd.spec @@ -12,7 +12,7 @@ BuildRequires: cmake gmake autotools Requires: zlib libuuid curl davix Requires: python py2-setuptools Requires: libxml2 -#Requires: scitokens-cpp +Requires: scitokens-cpp Requires: openssl %define soext so @@ -46,7 +46,6 @@ cmake ../%n-%{realversion} \ -DCMAKE_SKIP_RPATH=TRUE \ -DENABLE_PYTHON=TRUE \ -DENABLE_HTTP=TRUE \ - -DENABLE_SCITOKENS=OFF \ -DXRD_PYTHON_REQ_VERSION=2 \ -DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT} \ -DCMAKE_CXX_FLAGS="-I${LIBUUID_ROOT}/include -I${DAVIX_ROOT}/include" \ From e887819b85178fa6f6396de0a8584f35674d9d96 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Fri, 5 May 2023 11:14:26 +0200 Subject: [PATCH 3/9] update xrootd 5.5.4 --- xrootd-toolfile.spec | 2 +- xrootd.spec | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/xrootd-toolfile.spec b/xrootd-toolfile.spec index aa2b52b901a..8f02af79d0c 100644 --- a/xrootd-toolfile.spec +++ b/xrootd-toolfile.spec @@ -10,7 +10,7 @@ mkdir -p %i/etc/scram.d cat << \EOF_TOOLFILE >%i/etc/scram.d/xrootd.xml - + diff --git a/xrootd.spec b/xrootd.spec index 68bdd24aa01..7c66cf439e4 100644 --- a/xrootd.spec +++ b/xrootd.spec @@ -1,4 +1,4 @@ -### RPM external xrootd 5.5.3 +### RPM external xrootd 5.5.4 ## INITENV +PATH LD_LIBRARY_PATH %i/lib64 ## INITENV +PATH PYTHONPATH %{i}/${PYTHON_LIB_SITE_PACKAGES} @@ -12,7 +12,6 @@ BuildRequires: cmake gmake autotools Requires: zlib libuuid curl davix Requires: python py2-setuptools Requires: libxml2 -Requires: scitokens-cpp Requires: openssl %define soext so @@ -51,8 +50,7 @@ cmake ../%n-%{realversion} \ -DCMAKE_CXX_FLAGS="-I${LIBUUID_ROOT}/include -I${DAVIX_ROOT}/include" \ -DUUID_INCLUDE_DIR="${LIBUUID_ROOT}/include" \ -DUUID_LIBRARY="${LIBUUID_ROOT}/lib64/libuuid.%{soext}" \ - -DSCITOKENS_CPP_DIR="${SCITOKENS_CPP_ROOT}" \ - -DCMAKE_PREFIX_PATH="${ZLIB_ROOT};${PYTHON_ROOT};${LIBXML2_ROOT};${LIBUUID_ROOT};${SCITOKENS_CPP_ROOT};${CURL_ROOT};${DAVIX_ROOT}" + -DCMAKE_PREFIX_PATH="${ZLIB_ROOT};${PYTHON_ROOT};${LIBXML2_ROOT};${LIBUUID_ROOT};${CURL_ROOT};${DAVIX_ROOT}" PYTHONPATH=%{i}/${PYTHON_LIB_SITE_PACKAGES}:$PYTHONPATH \ make %makeprocesses VERBOSE=1 From 2d68d17298469f54a9d65e6bec19d41ae86c3e59 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 8 May 2023 11:58:16 +0200 Subject: [PATCH 4/9] Drop py2-fs and py2-xrootdpyfs --- pip/requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/pip/requirements.txt b/pip/requirements.txt index c25c5f58044..0564c99dbb9 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -40,7 +40,6 @@ entrypoints==0.3 enum34==1.1.6 flake8==3.7.7 flawfinder==2.0.8 -fs==0.5.5a1 funcsigs==1.0.2 functools32==3.2.3-2 future==0.17.1 @@ -192,4 +191,3 @@ Werkzeug==0.15.2 wheel==0.33.1 widgetsnbextension==3.4.2 xgboost==0.82 -xrootdpyfs==0.1.5 From a2cb058ecb33abdf73b75add2b57d45935abcbdc Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 8 May 2023 13:26:20 +0200 Subject: [PATCH 5/9] Remove py2-xrootdpyfs from python_tools --- python_tools.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tools.spec b/python_tools.spec index c0321f56447..a98d7c8f862 100644 --- a/python_tools.spec +++ b/python_tools.spec @@ -40,7 +40,7 @@ Requires: py2-uproot Requires: py2-oamap #this DOES NOT depend on numpy.. -Requires: py2-xrootdpyfs +# Requires: py2-xrootdpyfs Requires: root curl python openldap From 66e22a5b0dcc2243d9426b2163e04bed6f029b84 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 8 May 2023 15:17:16 +0200 Subject: [PATCH 6/9] Remove py2-fs from python_tools --- python_tools.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tools.spec b/python_tools.spec index a98d7c8f862..15aff8cbec6 100644 --- a/python_tools.spec +++ b/python_tools.spec @@ -127,7 +127,7 @@ Requires: py2-idna Requires: py2-Werkzeug Requires: py2-pytest Requires: py2-avro -Requires: py2-fs +# Requires: py2-fs Requires: py2-lizard Requires: py2-flawfinder Requires: py2-python-ldap From bb95df3e8454cff0a97c535986b4c09c7e06fddf Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 9 May 2023 16:37:52 +0200 Subject: [PATCH 7/9] Delete scitokens-cpp.patch --- scitokens-cpp.patch | 46 --------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 scitokens-cpp.patch diff --git a/scitokens-cpp.patch b/scitokens-cpp.patch deleted file mode 100644 index b63f9f32339..00000000000 --- a/scitokens-cpp.patch +++ /dev/null @@ -1,46 +0,0 @@ ---- CMakeLists.txt.orig 2021-09-14 22:52:06.000000000 +0200 -+++ CMakeLists.txt 2022-02-01 11:46:11.698151800 +0100 -@@ -25,27 +25,23 @@ - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g") - - --if( APPLE ) -- - find_package( OpenSSL REQUIRED ) - find_package( Sqlite3 REQUIRED ) -+ -+if( APPLE ) -+ - set(LIBCRYPTO_INCLUDE_DIRS ${OPENSSL_INCLUDE_DIR}) - set(LIBCRYPTO_LIBRARIES ${OPENSSL_CRYPTO_LIBRARY}) - set(CMAKE_MACOSX_RPATH ON) - - elseif( UNIX ) - --include (FindPkgConfig) --pkg_check_modules(LIBCRYPTO REQUIRED libcrypto) --pkg_check_modules(OPENSSL REQUIRED openssl) --pkg_check_modules(SQLITE REQUIRED sqlite3) -- - # pkg_check_modules fails to return an absolute path on RHEL7. Set the - # link directories accordingly. - link_directories(${OPENSSL_LIBRARY_DIRS} ${LIBCRYPTO_LIBRARY_DIRS}) - endif() - --include_directories( "${PROJECT_SOURCE_DIR}" ${JWT_CPP_INCLUDES} ${CURL_INCLUDES} ${OPENSSL_INCLUDE_DIRS} ${LIBCRYPTO_INCLUDE_DIRS} ${SQLITE_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS} ) -+include_directories( "${PROJECT_SOURCE_DIR}" ${JWT_CPP_INCLUDES} ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} ${LIBCRYPTO_INCLUDE_DIRS} ${SQLITE_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS} ) - - add_library(SciTokens SHARED src/scitokens.cpp src/scitokens_internal.cpp src/scitokens_cache.cpp) - target_link_libraries(SciTokens ${OPENSSL_LIBRARIES} ${LIBCRYPTO_LIBRARIES} ${CURL_LIBRARIES} ${SQLITE_LIBRARIES} ${UUID_LIBRARIES}) ---- vendor/jwt-cpp/CMakeLists.txt.orig 2023-05-02 15:31:50.008285546 +0200 -+++ vendor/jwt-cpp/CMakeLists.txt 2023-05-02 15:32:36.039856604 +0200 -@@ -17,6 +17,8 @@ - find_package(picojson REQUIRED) - endif() - -+include_directories(${OPENSSL_INCLUDE_DIRS}) -+ - add_library(jwt-cpp INTERFACE) - add_library(jwt-cpp::jwt-cpp ALIAS jwt-cpp) - - From 49b6bfa80eea28d2418242bd94ab22b6e10b8355 Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 9 May 2023 16:38:01 +0200 Subject: [PATCH 8/9] Delete scitokens-cpp.spec --- scitokens-cpp.spec | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 scitokens-cpp.spec diff --git a/scitokens-cpp.spec b/scitokens-cpp.spec deleted file mode 100644 index 672003ac910..00000000000 --- a/scitokens-cpp.spec +++ /dev/null @@ -1,25 +0,0 @@ -### RPM external scitokens-cpp 0.7.0 -Source: https://github.com/scitokens/%{n}/archive/refs/tags/v%{realversion}.tar.gz - -BuildRequires: cmake gmake -Requires: libuuid curl sqlite openssl -Patch0: scitokens-cpp - -%prep -%setup -n %{n}-%{realversion} -%patch0 -p0 -sed -i -e 's/ -Werror//' CMakeLists.txt - -%build -rm -rf ../build; mkdir ../build; cd ../build -cmake ../%{n}-%{realversion} \ - -DCMAKE_INSTALL_PREFIX=%{i} \ - -DCMAKE_BUILD_TYPE=Release \ - -DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT} \ - -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ - -DCMAKE_PREFIX_PATH="${CURL_ROOT};${LIBUUID_ROOT};${SQLITE_ROOT}" - -%install -cd ../build -make %{makeprocesses} -make install From f968e874c48a08c9c6d3429d1865ca27ce6dd0af Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Thu, 11 May 2023 19:08:26 +0200 Subject: [PATCH 9/9] update root commit --- root.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/root.spec b/root.spec index 21a1eef8e56..62599909a6b 100644 --- a/root.spec +++ b/root.spec @@ -2,7 +2,7 @@ ## INITENV +PATH PYTHON27PATH %{i}/lib ## INITENV +PATH PYTHON3PATH %{i}/lib ## INITENV SET ROOTSYS %{i} -%define tag cfdde2734930ff94b0539a22c0d6fcbd43e7cfc9 +%define tag c5b9d2c2dffbe7c71b210a331b343d1a56b1bbac %define branch cms/v6-14-00-patches/8ba6b0f %define github_user cms-sw Source: git+https://github.com/%{github_user}/root.git?obj=%{branch}/%{tag}&export=%{n}-%{realversion}&output=/%{n}-%{realversion}-%{tag}.tgz