create single crab to deploy helper scripts

cms-common.spec

@@ -1,8 +1,8 @@
 ### RPM cms cms-common 1.0
-## REVISION 1208
+## REVISION 1209
 ## NOCOMPILER
 
-%define tag cf4dd43612a2feb26f8a06b566bea485f888a010
+%define tag da521d6544b6cca0c90e8b95ea562a84170c79a0
 
 Source:  git+https://github.com/cms-sw/cms-common.git?obj=master/%{tag}&export=%{n}-%{realversion}-%{tag}&output=/%{n}-%{realversion}-%{tag}.tgz
 

cmssw-tool-conf.spec

@@ -11,7 +11,7 @@
 %define isnotaarch64 %(case %{cmsplatf} in (*_aarch64_*) echo 0 ;; (*) echo 1 ;; esac)
 %define isslc7 %(case %{cmsplatf} in (slc7_amd64*) echo 1 ;; (*) echo 0 ;; esac)
 
-Requires: crab-prod crab-pre crab-dev
+Requires: crab
 Requires: google-benchmark-toolfile
 Requires: catch2-toolfile
 Requires: starlight-toolfile

crab-build.file

@@ -1,10 +1,6 @@
 ## NOCOMPILER
-%define copy_revision_file() \
-  OLD_REV=0 \
-  if [ -f %1 ] ; then OLD_REV=$(grep '^\s*#\s*CMSDIST_FILE_REVISION\s*=' %{1} | tail -1 | sed 's|.*=||;s| ||g') ; fi \
-  NEW_REV=$(grep '^\s*#\s*CMSDIST_FILE_REVISION\s*=' %{2} | tail -1 | sed 's|.*=||;s| ||g') \
-  if [ ${OLD_REV} -lt ${NEW_REV} ] ; then rm -f %1 ; cp %2 %1 ; fi
-
+## NO_AUTO_DEPENDENCY
+%define crab_type %(echo %n | sed -e 's|crab-||')
 %if "%{?wmcore_packages:set}" != "set"
   %define wmcore_packages PSetTweaks Utils WMCore
 %endif
@@ -19,10 +15,6 @@ Source0: git://github.com/dmwm/CRABClient.git?obj=master/%{realversion}&export=C
 Source1: git://github.com/dmwm/WMCore.git?obj=master/%{wmcore_version}&export=WMCore&output=/WMCore-%{wmcore_version}.tar.gz
 Source2: git://github.com/dmwm/CRABServer.git?obj=master/%{crabserver_version}&export=CRABServer&output=/CRABServer-%{crabserver_version}.tar.gz
 Source3: git://github.com/dmwm/DBS.git?obj=master/%{dbs_version}&export=DBS&output=/DBS-%{dbs_version}.tar.gz
-#Needed for CRAB Python API
-Source98: crab-proxy-package
-#Crab startup script which finds latest crab client, sets env and run it
-Source99: crab.sh
 
 %prep
 %setup -D -T -b 0 -n CRABClient
@@ -34,15 +26,14 @@ Source99: crab.sh
 
 %install
 
-mkdir -p %{i}/bin %{i}/lib %{i}/common
-
-#List of CRAB python pakcages for which we need to create ProxyPackage symlink
-ls %{_builddir}/CRABClient/src/python/*/__init__.py | sed 's|/__init__.py$||;s|.*/||' > %{i}/common/crab_pkgs.txt
-
 #Copy CRABClient
-rsync -a %{_builddir}/CRABClient/src/python/ %{i}/lib
+rsync -a %{_builddir}/CRABClient/src/python/ %{i}/lib/
 sed -i -e 's|"development"|"%{realversion}"|' %{i}/lib/CRABClient/__init__.py
 rsync -a %{_builddir}/CRABClient/bin/  %{i}/bin/
+rsync -a %{_builddir}/CRABClient/etc/  %{i}/etc/
+
+#List of CRAB python pakcages for which we need to create ProxyPackage symlink
+ls %{_builddir}/CRABClient/src/python/*/__init__.py | sed 's|/__init__.py$||;s|.*/||' > %{i}/etc/crab_py_pkgs.txt
 
 #Copy WMCore
 for pkg in %{wmcore_packages} ; do
@@ -71,30 +62,16 @@ for pkg in %{dbs_packages} ; do
   fi
 done
 
-#Copy scripts needed for crab startup and api
-cp %{_sourcedir}/crab-proxy-package %{i}/common/
-cp %{_sourcedir}/crab.sh            %{i}/common/
-chmod +x %{i}/common/crab.sh
-sed -i -e 's|@CMS_PATH@|%{cmsroot}|g' %{i}/common/crab.sh
-sed -i -e 's|@CMS_PATH@|%{cmsroot}|g' %{i}/common/crab-proxy-package
+if [ $(grep '_UseCrab' %{i}/etc/crab-bash-completion.sh | wc -l) -gt 0 ] && [ $(grep ' filenames  *crab' %{i}/etc/crab-bash-completion.sh |wc -l) -gt 0 ] ; then
+  sed -i -e 's|_UseCrab|_UseCrab_%{crab_type}|g;s| filenames  *crab| filenames crab-%{crab_type}|' %{i}/etc/crab-bash-completion.sh
+else
+  echo "ERROR: Unable to fix crab use function _UseCrab"
+  exit 1
+fi
 
 %post
-%{relocateConfig}/common/crab.sh
-%{relocateConfig}/common/crab-proxy-package
-
 cd ${RPM_INSTALL_PREFIX}
 mkdir -p share/%{pkgdir}
-for dir in bin lib ; do rsync -a %{pkgrel}/${dir}/ share/%{pkgdir}/${dir}/ ; done
-crab_common=share/%{pkgcategory}/crab-common/1.0
-mkdir -p ${crab_common}/bin ${crab_common}/lib
-for p in $(cat %{pkgrel}/common/crab_pkgs.txt); do
-  mkdir -p ${crab_common}/lib/${p}
-  rm -f ${crab_common}/lib/${p}/__init__.py*
-  ln -s ../crab-proxy-package ${crab_common}/lib/${p}/__init__.py
+for dir in bin lib etc ; do 
+  rsync -a %{pkgrel}/${dir}/ share/%{pkgdir}/${dir}/
 done
-%copy_revision_file ${crab_common}/bin/crab.sh            %{pkgrel}/common/crab.sh
-%copy_revision_file ${crab_common}/lib/crab-proxy-package %{pkgrel}/common/crab-proxy-package
-ls -d share/%{pkgcategory}/%{n}/*/bin/crab | sed -e 's|/bin/crab$||;s|.*/||' | sort -n | tail -1 > ${crab_common}/%{n}.latest
-
-rm -f common/_crab-startup common/%{n}
-ln -s ../${crab_common}/bin/crab.sh common/%{n}

crab.spec

@@ -1,6 +1,63 @@
-### RPM cms crab 3.3.2001
-%define wmcore_version     1.2.8
-%define crabserver_version 3.3.2001.rc1
-%define dbs_version        3.10.0
+### RPM cms crab 1.0
+## NOCOMPILER
+## NO_AUTO_DEPENDENCY
+Requires: crab-prod crab-pre crab-dev
 
-## IMPORT crab-build
+#Crab startup script which finds latest crab client, sets env and run it
+Source0: crab/crab.sh
+#Needed for CRAB Python API
+Source1: crab/crab-proxy-package
+#CRAB Setup script
+Source2: crab/crab-setup.csh
+Source3: crab/crab-setup.sh
+#CRAB env script
+Source4: crab/crab-env.csh
+Source5: crab/crab-env.sh
+
+%define copy_revision_file() \
+  OLD_REV=0 \
+  if [ -f %2 ] ; then OLD_REV=$(grep '^\s*#\s*CMSDIST_FILE_REVISION\s*=' %{2} | tail -1 | sed 's|.*=||;s| ||g') ; fi \
+  NEW_REV=$(grep '^\s*#\s*CMSDIST_FILE_REVISION\s*=' %{1} | tail -1 | sed 's|.*=||;s| ||g') \
+  if [ ${OLD_REV} -lt ${NEW_REV} ] ; then rm -f %2 ; cp %1 %2 ; fi
+
+%prep
+%build
+%install
+
+#Copy scripts needed for crab startup and api
+cp %{_sourcedir}/crab-proxy-package %{i}/
+cp %{_sourcedir}/crab.sh            %{i}/
+cp %{_sourcedir}/crab-env.*sh       %{i}/
+cp %{_sourcedir}/crab-setup.*sh     %{i}/
+chmod +x %{i}/crab.sh
+sed -i -e 's|@CMS_PATH@|%{cmsroot}|g' %{i}/crab*
+sed -i -e 's|@CRAB_COMMON_VERSION@|%{realversion}|g' %{i}/crab*
+
+%post
+%{relocateConfig}/crab*
+
+cd ${RPM_INSTALL_PREFIX}
+crab=share/%{pkgcategory}/%{n}/%{realversion}
+mkdir -p ${crab}/bin ${crab}/lib ${crab}/etc share/etc/profile.d
+for f in crab-env.csh crab-env.sh ; do
+  %copy_revision_file %{pkgrel}/$f share/etc/profile.d/S99$f
+done
+for f in crab-setup.csh crab-setup.sh ; do
+  %copy_revision_file %{pkgrel}/$f common/$f
+done
+%copy_revision_file %{pkgrel}/crab.sh            ${crab}/bin/crab.sh
+%copy_revision_file %{pkgrel}/crab-proxy-package ${crab}/lib/crab-proxy-package
+
+for pkg in $(echo %{directpkgreqs} | tr ' ' '\n' | grep '^cms/crab-') ; do
+  crab_name=$(echo $pkg | cut -d/ -f2)
+  crab_type=$(echo $crab_name | sed -e 's|^crab-||')
+  for p in $(cat share/${pkg}/etc/crab_py_pkgs.txt); do
+    mkdir -p ${crab}/lib/${crab_type}/$p
+    rm -rf ${crab}/lib/${crab_type}/$p/__init__.py*
+    ln -s ../../crab-proxy-package ${crab}/lib/${crab_type}/$p/__init__.py
+  done
+  #Find latest version; extra .zzzz are added so that version 3.3.2001 becomes > 3.3.2001.rcX
+  ls -d share/cms/${crab_name}/*/bin/crab | sed -e 's|/bin/crab$|.zzzz|;s|.*/||' | sort -n | sed -e 's|.zzzz$||' | tail -1 > ${crab}/etc/${crab_name}.latest
+  rm -f common/_crab-startup common/${crab_name}
+  ln -s ../${crab}/bin/crab.sh common/${crab_name}
+done

crab/crab-env.csh.file

@@ -0,0 +1,2 @@
+#!/bin/tcsh
+#CMSDIST_FILE_REVISION=1

crab/crab-env.sh.file

@@ -0,0 +1,13 @@
+#!/bin/bash
+#CMSDIST_FILE_REVISION=1
+cms_basedir="@CMS_PATH@/share/cms"
+crab_shared_dir="${cms_basedir}/crab/@CRAB_COMMON_VERSION@"
+for crab_latest in $(ls ${crab_shared_dir}/etc/crab-*.latest 2>/dev/null) ; do
+  crab_type=$(basename ${crab_latest} | sed 's|.latest$||;s|^crab-||')
+  crab_version=$(cat ${crab_latest})
+  if [ -e ${cms_basedir}/crab-${crab_type}/${crab_version}/etc/crab-bash-completion.sh ]; then
+    source ${cms_basedir}/crab-${crab_type}/${crab_version}/etc/crab-bash-completion.sh >/dev/null 2>&1
+  fi
+  unset crab_version crab_type
+done
+unset cms_basedir crab_shared_dir crab_latest

crab/crab-proxy-package.file

@@ -0,0 +1,15 @@
+#CMSDIST_FILE_REVISION=2
+import sys
+from os.path import join, exists
+CMSPath="@CMS_PATH@/share/cms"
+_crab_type="crab-%s" % __file__.split("/")[-3]
+_crab_ver=None
+with open(join(CMSPath,"crab/@CRAB_COMMON_VERSION@/etc","%s.latest" % _crab_type)) as ref:
+  _crab_ver = ref.readlines()[0].strip()
+_crablib = join(CMSPath,_crab_type,_crab_ver,"lib")
+sys.path.insert(0,_crablib)
+_crabpkg = join(_crablib, __name__)
+_pkg_init_file = join(_crabpkg, '__init__.py')
+if exists(_pkg_init_file):
+  __path__.insert(0,_crabpkg)
+  exec(open(_pkg_init_file).read())

crab/crab-setup.csh.file

@@ -0,0 +1,26 @@
+#!/bin/tcsh
+#CMSDIST_FILE_REVISION=1
+setenv CRABCLINET_TYPE ""
+switch ("X$1Y")
+  case XprodY:
+  case XpreY:
+  case XdevY:
+    setenv CRABCLINET_TYPE "$1"
+    breaksw
+  case XY:
+    setenv CRABCLINET_TYPE "prod"
+    breaksw
+  default:
+    echo "ERROR: Invalid CRAB type '$1' provided. Valid values are prod, pre and dev."
+    echo "Usage: $0 [prod|pre|dev]"
+    breaksw
+endsw
+if (${?CRABCLINET_TYPE}) then
+  set crab_shared_dir="@CMS_PATH@/share/cms/crab/@CRAB_COMMON_VERSION@"
+  if (${?PYTHONPATH}) then
+    setenv PYTHONPATH "${crab_shared_dir}/lib/${CRABCLINET_TYPE}:$PYTHONPATH"
+  else
+    setenv PYTHONPATH "${crab_shared_dir}/lib/${CRABCLINET_TYPE}"
+  endif
+  unset crab_shared_dir
+endif

crab/crab-setup.sh.file

@@ -0,0 +1,16 @@
+#!/bin/bash
+#CMSDIST_FILE_REVISION=1
+case "X$1Y" in
+  XprodY|XdevY|XpreY) CRABCLINET_TYPE="$1" ;;
+  XY ) CRABCLINET_TYPE="prod" ;;
+  * ) 
+    echo "ERROR: Invalid CRAB type '$1' provided, valid values are prod, pre and dev."
+    echo "Usage: $0 [prod|pre|dev]"
+    return 1
+    ;;
+esac
+export CRABCLINET_TYPE
+crab_shared_dir="@CMS_PATH@/share/cms/crab/@CRAB_COMMON_VERSION@"
+export PYTHONPATH="${crab_shared_dir}/lib/${CRABCLINET_TYPE}${PYTHONPATH:+:$PYTHONPATH}"
+complete -F _UseCrab_${CRABCLINET_TYPE} -o filenames crab
+unset crab_shared_dir

crab.sh.file → crab/crab.sh.file

@@ -3,15 +3,18 @@
 # **************** IMPORTANT NOTE ***************** #
 # Increament Crab Startup Revision for every change #
 #####################################################
-#CMSDIST_FILE_REVISION=1
+#CMSDIST_FILE_REVISION=2
 
 #Use crab-prod for default crab command
 crab_pkg=$(basename $0)
-if [ "${crab_pkg}" = "crab" ] ; then crab_pkg="crab-prod" ; fi
+if [ "${crab_pkg}" = "crab" ] ; then
+  if [ "$CRABCLINET_TYPE" = "" ] ; then CRABCLINET_TYPE="prod" ; fi
+  crab_pkg="crab-${CRABCLINET_TYPE}"
+fi
 
 #Search for latest crab version for cmsos
-cms_basedir="@CMS_PATH@"
-crab_dir="${cms_basedir}/share/cms/${crab_pkg}/$(cat ${cms_basedir}/share/cms/crab-common/1.0/${crab_pkg}.latest)"
+cms_basedir="@CMS_PATH@/share/cms"
+crab_dir="${cms_basedir}/${crab_pkg}/$(cat ${cms_basedir}/crab/@CRAB_COMMON_VERSION@/etc/${crab_pkg}.latest)"
 
 #Set crab runtime env and run crab
 export PATH="${crab_dir}/bin${PATH:+:$PATH}"