diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..d9a042c6ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# Ignore all compiled files +*.pyc +*.o +*.mod + +# Ignore exec folder +exec/ + +# Ignore sorc folders from externals +sorc/logs/ +sorc/fv3gfs.fd/ +sorc/gfs_post.fd/ +sorc/gsi.fd/ +sorc/ufs_utils.fd/ +sorc/gfs_wafs.fd/ +sorc/verif-global.fd/ diff --git a/Externals.cfg b/Externals.cfg new file mode 100644 index 0000000000..9fb968a378 --- /dev/null +++ b/Externals.cfg @@ -0,0 +1,46 @@ +# External sub-modules of global-workflow + +[NEMSfv3gfs] +tag = gfs.v16_PhysicsUpdate +local_path = sorc/fv3gfs.fd +repo_url = ssh://vlab.ncep.noaa.gov:29418/NEMSfv3gfs +protocol = git +required = True + +[GSI] +hash = cb8f69d82f38dcf85669b45aaf95dad068f0103c +local_path = sorc/gsi.fd +repo_url = ssh://vlab.ncep.noaa.gov:29418/ProdGSI +protocol = git +required = True + +[EMC_post] +hash = ba7e59b290c8149ff1c2fee98d01e99e4ef92ee6 +local_path = sorc/gfs_post.fd +repo_url = https://github.com/NOAA-EMC/EMC_post.git +protocol = git +required = True + +[UFS_UTILS] +tag = v1.1.0 +local_path = sorc/ufs_utils.fd +repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git +protocol = git +required = True + +[EMC_verif-global] +tag = verif_global_v1.2.2 +local_path = sorc/verif-global.fd +repo_url = ssh://vlab.ncep.noaa.gov:29418/EMC_verif-global +protocol = git +required = True + +[EMC_gfs_wafs] +tag = gfs_wafs.v5.0.11 +local_path = sorc/gfs_wafs.fd +repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git +protocol = git +required = False + +[externals_description] +schema_version = 1.0.0 diff --git a/modulefiles/module_nemsutil.hera b/modulefiles/module_nemsutil.hera deleted file mode 100644 index f1908fdf6e..0000000000 --- a/modulefiles/module_nemsutil.hera +++ /dev/null @@ -1,10 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles -module load w3nco/2.0.6 -module load bacio/2.0.3 -module load nemsio/2.2.3 - -export FCMP=ifort diff --git a/modulefiles/module_nemsutil.theia b/modulefiles/module_nemsutil.theia deleted file mode 100644 index 45685118eb..0000000000 --- a/modulefiles/module_nemsutil.theia +++ /dev/null @@ -1,15 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -# Loading Intel Compiler Suite -module load intel/14.0.2 -module load impi/5.1.2.150 - -# Loding nceplibs modules -module use -a $MOD_PATH -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load nemsio/v2.2.1 - -export FCMP=ifort diff --git a/modulefiles/module_nemsutil.wcoss b/modulefiles/module_nemsutil.wcoss deleted file mode 100644 index f421c1a88b..0000000000 --- a/modulefiles/module_nemsutil.wcoss +++ /dev/null @@ -1,13 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -# Loading Intel Compiler Suite -module load ics/14.0.1 - -# Loding nceplibs modules -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load nemsio/v2.2.1 - -export FCMP=ifort diff --git a/modulefiles/module_nemsutil.wcoss_cray b/modulefiles/module_nemsutil.wcoss_cray deleted file mode 100644 index 371c8e0245..0000000000 --- a/modulefiles/module_nemsutil.wcoss_cray +++ /dev/null @@ -1,17 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -module purge -module load modules -module load PrgEnv-intel -module load cray-mpich -module load craype-sandybridge - -module load w3nco-intel/2.0.6 -module load bacio-intel/2.0.1 - -export NEMSIO_INC=/usrx/local/nceplibs/nemsio/nemsio_v2.2.3/incmod -export NEMSIO_LIB=/usrx/local/nceplibs/nemsio/nemsio_v2.2.3/libnemsio_v2.2.3.a - -export FCMP=ftn diff --git a/modulefiles/module_nemsutil.wcoss_cray_userlib b/modulefiles/module_nemsutil.wcoss_cray_userlib deleted file mode 100644 index 53fad475a5..0000000000 --- a/modulefiles/module_nemsutil.wcoss_cray_userlib +++ /dev/null @@ -1,19 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -# Load Intel environment -module purge -module load modules -module load PrgEnv-intel -module load cray-mpich -module load craype-sandybridge - -# Load NCEPLIBS modules -module unuse /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module use $MOD_PATH -module load w3nco/v2.0.6 -module load bacio/v2.0.2 -module load nemsio/v2.2.3 - -export FCMP=ftn diff --git a/modulefiles/module_nemsutil.wcoss_dell_p3 b/modulefiles/module_nemsutil.wcoss_dell_p3 deleted file mode 100644 index e93d581651..0000000000 --- a/modulefiles/module_nemsutil.wcoss_dell_p3 +++ /dev/null @@ -1,12 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -module load ips/18.0.1.163 -module load impi/18.0.1 - -module load bacio/2.0.2 -module load w3nco/2.0.6 -module load nemsio/2.2.3 - -export FCMP=ifort diff --git a/modulefiles/modulefile.grib_util.theia b/modulefiles/modulefile.grib_util.theia deleted file mode 100644 index ecdd678049..0000000000 --- a/modulefiles/modulefile.grib_util.theia +++ /dev/null @@ -1,20 +0,0 @@ -#%Module###################################################################### -module use -a $MOD_PATH - -module load intel/14.0.2 - -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 - - -module load bacio/v2.0.2 -module load w3emc/v2.2.0 -module load w3nco/v2.0.6 -module load ip/v3.0.0 -module load sp/v2.0.2 -module load g2/v3.1.0 - -export FCMP=ifort -export CCMP=icc - diff --git a/modulefiles/modulefile.grib_util.wcoss b/modulefiles/modulefile.grib_util.wcoss deleted file mode 100644 index 0ae2e8f49e..0000000000 --- a/modulefiles/modulefile.grib_util.wcoss +++ /dev/null @@ -1,32 +0,0 @@ -#%Module###################################################################### -proc ModulesHelp { } { - puts stderr "Load modules for building GRIB utilities" -} -module-whatis "This module loads the modules and libraries for building\ - the GRIB utilities, including jasper, png, zlib, bacio, g2,\ - w3emc, w3nco, ip, sp, and iobuf." - -conflict build_grib_util - -# -# Loading required system modules -# - module load ics - module switch ics/15.0.6 - module load jasper/v1.900.1 - module load png/v1.2.44 - module load z/v1.2.6 - -# Loading Intel-Compiled NCEP Libraries - module load bacio/v2.0.1 - module load w3emc/v2.2.0 - module load w3nco/v2.0.6 - module load ip/v3.0.0 - module load sp/v2.0.2 - - # pre-implemented g2 v3.1.0 - module use /nwtest2/lib/modulefiles - module load g2/v3.1.0 - -setenv FCMP ifort -setenv CCMP icc diff --git a/modulefiles/modulefile.grib_util.wcoss_cray b/modulefiles/modulefile.grib_util.wcoss_cray deleted file mode 100644 index 191baa15df..0000000000 --- a/modulefiles/modulefile.grib_util.wcoss_cray +++ /dev/null @@ -1,22 +0,0 @@ -#%Module###################################################################### -module unload craype-haswell -module load craype-sandybridge -module unload PrgEnv-cray -module load PrgEnv-intel/5.2.56 -module switch intel/15.0.6.233 -module load iobuf/2.0.7 - -module load bacio-intel/2.0.1 -module load w3emc-intel/2.2.0 -module load w3nco-intel/2.0.6 -module load ip-intel/3.0.0 -module load sp-intel/2.0.2 -module load jasper-gnu-sandybridge/1.900.1 -module load png-intel-sandybridge/1.2.49 -module load zlib-intel-sandybridge/1.2.7 - -module use /gpfs/hps/nco/ops/nwtest/lib/modulefiles -module load g2-intel/3.1.0 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.grib_util.wcoss_cray_userlib b/modulefiles/modulefile.grib_util.wcoss_cray_userlib deleted file mode 100644 index 56ebe0c336..0000000000 --- a/modulefiles/modulefile.grib_util.wcoss_cray_userlib +++ /dev/null @@ -1,22 +0,0 @@ -#%Module###################################################################### -module unload craype-haswell -module load craype-sandybridge -module unload PrgEnv-cray -module load PrgEnv-intel/5.2.56 -module switch intel/15.0.6.233 -module load iobuf/2.0.7 - -module unuse /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module use -a $MOD_PATH -module load bacio/v2.0.2 -module load w3emc/v2.2.0 -module load w3nco/v2.0.6 -module load ip/v3.0.0 -module load sp/v2.0.2 -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 -module load g2/v3.1.0 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.grib_util.wcoss_dell_p3 b/modulefiles/modulefile.grib_util.wcoss_dell_p3 deleted file mode 100644 index bbebec9cbf..0000000000 --- a/modulefiles/modulefile.grib_util.wcoss_dell_p3 +++ /dev/null @@ -1,15 +0,0 @@ -#%Module###################################################################### - -module load bacio/2.0.2 -module load w3emc/2.3.0 -module load w3nco/2.0.6 -module load ip/3.0.1 -module load sp/2.0.2 - -module load jasper/1.900.1 -module load libpng/1.2.59 -module load zlib/1.2.11 -module load g2/3.1.0 - -export FCMP=ifort -export CCMP=icc diff --git a/modulefiles/modulefile.prod_util.theia b/modulefiles/modulefile.prod_util.theia deleted file mode 100644 index 3ad4d37bf6..0000000000 --- a/modulefiles/modulefile.prod_util.theia +++ /dev/null @@ -1,8 +0,0 @@ -#%Module###################################################################### -## -module load intel/16.1.150 -module use -a $MOD_PATH -module load w3nco/v2.0.6 - -export FCMP=ifort -export CCMP=icc diff --git a/modulefiles/modulefile.prod_util.wcoss_cray b/modulefiles/modulefile.prod_util.wcoss_cray deleted file mode 100644 index 51031c6bdd..0000000000 --- a/modulefiles/modulefile.prod_util.wcoss_cray +++ /dev/null @@ -1,11 +0,0 @@ -#%Module##################################################### -module purge -module load modules -module load PrgEnv-intel -module load cray-mpich -module load craype-sandybridge - -module load w3nco-intel/2.0.6 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.prod_util.wcoss_cray_userlib b/modulefiles/modulefile.prod_util.wcoss_cray_userlib deleted file mode 100644 index dd5209fcf0..0000000000 --- a/modulefiles/modulefile.prod_util.wcoss_cray_userlib +++ /dev/null @@ -1,13 +0,0 @@ -#%Module##################################################### -module purge -module load modules -module load PrgEnv-intel -module load cray-mpich -module load craype-sandybridge - -module unuse /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module use -a $MOD_PATH -module load w3nco/v2.0.6 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.prod_util.wcoss_dell_p3 b/modulefiles/modulefile.prod_util.wcoss_dell_p3 deleted file mode 100644 index 9186e13eb8..0000000000 --- a/modulefiles/modulefile.prod_util.wcoss_dell_p3 +++ /dev/null @@ -1,6 +0,0 @@ -#%Module##################################################### - -module load w3nco/2.0.6 - -export FCMP=ifort -export CCMP=icc diff --git a/modulefiles/modulefile.wgrib2.theia b/modulefiles/modulefile.wgrib2.theia deleted file mode 100644 index b1568e8cea..0000000000 --- a/modulefiles/modulefile.wgrib2.theia +++ /dev/null @@ -1,19 +0,0 @@ -#%Module###################################################################### - -module load intel/14.0.2 -module load netcdf/4.3.0 -module load hdf5/1.8.14 -module load szip/2.1 - -module use -a $MOD_PATH -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 -module load ip/v3.0.0 -module load sp/v2.0.2 -module load g2c/v1.5.0 - -export NETCDF_INCLUDE="-I${NETCDF}/include" - -export FCMP=ifort -export CCMP=icc diff --git a/modulefiles/modulefile.wgrib2.wcoss b/modulefiles/modulefile.wgrib2.wcoss deleted file mode 100644 index 0eea72e391..0000000000 --- a/modulefiles/modulefile.wgrib2.wcoss +++ /dev/null @@ -1,24 +0,0 @@ -#%Module###################################################################### -############################################################# -## Lin.Gan@noaa.gov -## EMC -## wgrib2 v2.0.5 -############################################################# -proc ModulesHelp { } { -puts stderr "Set environment veriables for wgrib2" -puts stderr "This module initializes the users environment" -puts stderr "to build the wgrib2 for WCOSS production.\n" -} -module-whatis "wgrib2" - -set ver v2.0.5 - -module load ics/15.0.6 -module load NetCDF/4.2/serial -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 -module load ip/v3.0.0 -module load sp/v2.0.2 -module load g2c/v1.5.0 - diff --git a/modulefiles/modulefile.wgrib2.wcoss_cray b/modulefiles/modulefile.wgrib2.wcoss_cray deleted file mode 100644 index 4d933141ad..0000000000 --- a/modulefiles/modulefile.wgrib2.wcoss_cray +++ /dev/null @@ -1,13 +0,0 @@ -#%Module###################################################################### -module load PrgEnv-gnu/5.2.56 -module load cray-netcdf/4.3.2 -module load craype/2.3.0 -module load craype-sandybridge -module load /gpfs/hps/nco/ops/nwtest/lib/modulefiles/g2c-gnu/1.5.0 - -module load jasper-gnu-sandybridge/1.900.1 -module load png-gnu-sandybridge/1.2.49 -module load zlib-gnu-sandybridge/1.2.7 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.wgrib2.wcoss_cray_userlib b/modulefiles/modulefile.wgrib2.wcoss_cray_userlib deleted file mode 100644 index 7b41a17f9f..0000000000 --- a/modulefiles/modulefile.wgrib2.wcoss_cray_userlib +++ /dev/null @@ -1,15 +0,0 @@ -#%Module###################################################################### -module load PrgEnv-gnu/5.2.56 -module load cray-netcdf/4.3.2 -module load craype/2.3.0 -module load craype-sandybridge - -module unuse /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module use -a $MOD_PATH -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 -module load g2c/v1.5.0 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.wgrib2.wcoss_dell_p3 b/modulefiles/modulefile.wgrib2.wcoss_dell_p3 deleted file mode 100644 index 9a43e3ffd9..0000000000 --- a/modulefiles/modulefile.wgrib2.wcoss_dell_p3 +++ /dev/null @@ -1,11 +0,0 @@ -#%Module###################################################################### - -module load ips/18.0.1.163 - -module load g2c/1.5.0 -module load jasper/1.900.1 -module load libpng/1.2.59 -module load zlib/1.2.11 - -export FCMP=ifort -export CCMP=icc diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 1af1b2b16b..34bd2d6d4b 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -51,40 +51,60 @@ echo " .... Library build not currently supported .... " # build fv3 #------------------------------------ $Build_fv3gfs && { -echo " .... Building fv3 .... " -./build_fv3.sh > $logs_dir/build_fv3.log 2>&1 +if [ ! -d fv3gfs.fd ]; then + echo " **** SKIPPING: FV3GFS sorc folder (fv3gfs.fd) not present" +else + echo " .... Building fv3 .... " + ./build_fv3.sh > $logs_dir/build_fv3.log 2>&1 +fi } #------------------------------------ # build gsi #------------------------------------ $Build_gsi && { -echo " .... Building gsi .... " -./build_gsi.sh > $logs_dir/build_gsi.log 2>&1 +if [ ! -d gsi.fd ]; then + echo " **** SKIPPING: GSI sorc folder (gsi.fd) not present" +else + echo " .... Building gsi .... " + ./build_gsi.sh > $logs_dir/build_gsi.log 2>&1 +fi } #------------------------------------ # build ncep_post #------------------------------------ $Build_ncep_post && { -echo " .... Building ncep_post .... " -./build_ncep_post.sh > $logs_dir/build_ncep_post.log 2>&1 +if [ ! -d gfs_post.fd ]; then + echo " **** SKIPPING: EMC_post sorc folder (gfs_post.fd) not present" +else + echo " .... Building ncep_post .... " + ./build_ncep_post.sh > $logs_dir/build_ncep_post.log 2>&1 +fi } #------------------------------------ # build ufs_utils #------------------------------------ $Build_ufs_utils && { -echo " .... Building ufs_utils .... " -./build_ufs_utils.sh > $logs_dir/build_ufs_utils.log 2>&1 +if [ ! -d ufs_utils.fd ]; then + echo " **** SKIPPING: UFS_UTILS sorc folder (ufs_utils.fd) not present" +else + echo " .... Building ufs_utils .... " + ./build_ufs_utils.sh > $logs_dir/build_ufs_utils.log 2>&1 +fi } #------------------------------------ # build gfs_wafs #------------------------------------ $Build_gfs_wafs && { -echo " .... Building gfs_wafs .... " -./build_gfs_wafs.sh > $logs_dir/build_gfs_wafs .log 2>&1 +if [ ! -d gfs_wafs.fd ]; then + echo " **** SKIPPING: GFS WAFS sorc folder (gfs_wafs.fd) not present" +else + echo " .... Building gfs_wafs .... " + ./build_gfs_wafs.sh > $logs_dir/build_gfs_wafs .log 2>&1 +fi } #------------------------------------ @@ -178,24 +198,5 @@ if [ $target = wcoss -o $target = wcoss_cray -o $target = wcoss_dell_p3 ]; then } fi -#------------------------------------ -# build prod_util -#------------------------------------ -$Build_prod_util && { -echo " .... prod_util build not currently supported .... " -#echo " .... Building prod_util .... " -#./build_prod_util.sh > $logs_dir/build_prod_util.log 2>&1 -} - -#------------------------------------ -# build grib_util -#------------------------------------ -$Build_grib_util && { -echo " .... grib_util build not currently supported .... " -#echo " .... Building grib_util .... " -#./build_grib_util.sh > $logs_dir/build_grib_util.log 2>&1 -} - -echo;echo " .... Build system finished .... " - +echo;echo "----- Build system finished -----" exit 0 diff --git a/sorc/build_grib_util.sh b/sorc/build_grib_util.sh deleted file mode 100755 index 6569cc22c0..0000000000 --- a/sorc/build_grib_util.sh +++ /dev/null @@ -1,88 +0,0 @@ -#! /usr/bin/env bash -set -eux - -source ./machine-setup.sh > /dev/null 2>&1 -cwd=`pwd` - -USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} -if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles - source ../modulefiles/modulefile.grib_util.$target > /dev/null 2>&1 -else - export MOD_PATH=${cwd}/lib/modulefiles - if [ $target = wcoss_cray ]; then - source ../modulefiles/modulefile.grib_util.${target}_userlib > /dev/null 2>&1 - else - source ../modulefiles/modulefile.grib_util.$target > /dev/null 2>&1 - fi -fi - -# Move to util/sorc folder -cd ../util/sorc - -# Check final exec folder exists -if [ ! -d "../exec" ]; then - mkdir ../exec -fi - -for grib_util in cnvgrib copygb2 degrib2 grbindex tocgrib2 tocgrib \ - copygb grb2index grib2grib tocgrib2super -do - cd $grib_util.fd - make -f makefile_$target clean - make -f makefile_$target - make -f makefile_$target install - make -f makefile_$target clean - cd .. -done - -# -# compile wgrib -# -cd wgrib.cd - make -f makefile_$target clean - make -f makefile_$target - make -f makefile_$target install - make -f makefile_$target clean -cd .. - -# -# compile wgrib2 -# -cd $cwd -source ./machine-setup.sh > /dev/null 2>&1 - -if [ $target = wcoss_cray -a $USE_PREINST_LIBS != true ]; then - source ../modulefiles/modulefile.wgrib2.${target}_userlib > /dev/null 2>&1 -else - source ../modulefiles/modulefile.wgrib2.$target > /dev/null 2>&1 -fi - -# Move to util/sorc folder -cd ../util/sorc -cwd=`pwd` - -#---------------------------------------------------------------- -export CPPFLAGS="-ffast-math -O3 -DGFORTRAN" -cd $cwd/wgrib2.cd/gctpc/ -make -f makefile.gctpc clean -make -f makefile.gctpc -rm -f *.o -#---------------------------------------------------------------- -if [ $target = wcoss_cray ]; then - export FFLAGS=-O2 - cd $cwd/wgrib2.cd/iplib/ - make clean - make - rm -f *.o -fi -#---------------------------------------------------------------- -cd $cwd/wgrib2.cd -module list -make -f makefile_$target clean -make -f makefile_$target -make -f makefile_$target install -make -f makefile_$target clean -#---------------------------------------------------------------- - -exit diff --git a/sorc/build_prod_util.sh b/sorc/build_prod_util.sh deleted file mode 100755 index e4220f7c26..0000000000 --- a/sorc/build_prod_util.sh +++ /dev/null @@ -1,47 +0,0 @@ -#! /usr/bin/env bash -set -eux - -source ./machine-setup.sh > /dev/null 2>&1 -cwd=`pwd` - -USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} -if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles - source ../modulefiles/modulefile.prod_util.$target > /dev/null 2>&1 -else - export MOD_PATH=${cwd}/lib/modulefiles - if [ $target = wcoss_cray ]; then - source ../modulefiles/modulefile.prod_util.${target}_userlib > /dev/null 2>&1 - else - source ../modulefiles/modulefile.prod_util.$target > /dev/null 2>&1 - fi -fi - -# Move to util/sorc folder -cd ../util/sorc - -# Check final exec folder exists -if [ ! -d "../exec" ]; then - mkdir ../exec -fi - -for prod_util in fsync_file -do - cd $prod_util.cd - make -f makefile clean - make -f makefile - make -f makefile install - make -f makefile clean - cd .. -done - -for prod_util in mdate ndate nhour -do - cd $prod_util.fd - make -f makefile clean - make -f makefile - make -f makefile install - make -f makefile clean - cd .. -done -exit diff --git a/sorc/fv3gfs_build.cfg b/sorc/fv3gfs_build.cfg index 8143eb0bd3..93fab78443 100644 --- a/sorc/fv3gfs_build.cfg +++ b/sorc/fv3gfs_build.cfg @@ -18,8 +18,6 @@ Building fv3nc2nemsio (fv3nc2nemsio) .................. yes Building regrid_nemsio (regrid_nemsio) ................ yes Building gfs_util (gfs_util) .......................... yes - Building prod_util (prod_util) ........................ no - Building grib_util (grib_util) ........................ no # -- END -- diff --git a/sorc/partial_build.sh b/sorc/partial_build.sh index 885548f052..da306969a0 100755 --- a/sorc/partial_build.sh +++ b/sorc/partial_build.sh @@ -14,9 +14,7 @@ "Build_gfs_bufrsnd" \ "Build_fv3nc2nemsio" \ "Build_regrid_nemsio" \ - "Build_gfs_util" \ - "Build_prod_util" \ - "Build_grib_util") + "Build_gfs_util") # # function parse_cfg: read config file and retrieve the values diff --git a/util/manage_externals/checkout_externals b/util/manage_externals/checkout_externals new file mode 100755 index 0000000000..a0698baef0 --- /dev/null +++ b/util/manage_externals/checkout_externals @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +"""Main driver wrapper around the manic/checkout utility. + +Tool to assemble external respositories represented in an externals +description file. + +""" +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import sys +import traceback + +import manic + +if sys.hexversion < 0x02070000: + print(70 * '*') + print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) + print('It appears that you are running python {0}'.format( + '.'.join(str(x) for x in sys.version_info[0:3]))) + print(70 * '*') + sys.exit(1) + + +if __name__ == '__main__': + ARGS = manic.checkout.commandline_arguments() + try: + RET_STATUS, _ = manic.checkout.main(ARGS) + sys.exit(RET_STATUS) + except Exception as error: # pylint: disable=broad-except + manic.printlog(str(error)) + if ARGS.backtrace: + traceback.print_exc() + sys.exit(1) diff --git a/util/manage_externals/manic/__init__.py b/util/manage_externals/manic/__init__.py new file mode 100644 index 0000000000..11badedd3b --- /dev/null +++ b/util/manage_externals/manic/__init__.py @@ -0,0 +1,9 @@ +"""Public API for the manage_externals library +""" + +from manic import checkout +from manic.utils import printlog + +__all__ = [ + 'checkout', 'printlog', +] diff --git a/util/manage_externals/manic/checkout.py b/util/manage_externals/manic/checkout.py new file mode 100755 index 0000000000..edc5655954 --- /dev/null +++ b/util/manage_externals/manic/checkout.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python + +""" +Tool to assemble repositories represented in a model-description file. + +If loaded as a module (e.g., in a component's buildcpp), it can be used +to check the validity of existing subdirectories and load missing sources. +""" +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import argparse +import logging +import os +import os.path +import sys + +from manic.externals_description import create_externals_description +from manic.externals_description import read_externals_description_file +from manic.externals_status import check_safe_to_update_repos +from manic.sourcetree import SourceTree +from manic.utils import printlog, fatal_error +from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME + +if sys.hexversion < 0x02070000: + print(70 * '*') + print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) + print('It appears that you are running python {0}'.format( + VERSION_SEPERATOR.join(str(x) for x in sys.version_info[0:3]))) + print(70 * '*') + sys.exit(1) + + +# --------------------------------------------------------------------- +# +# User input +# +# --------------------------------------------------------------------- +def commandline_arguments(args=None): + """Process the command line arguments + + Params: args - optional args. Should only be used during systems + testing. + + Returns: processed command line arguments + """ + description = ''' + +%(prog)s manages checking out groups of externals from revision +control based on an externals description file. By default only the +required externals are checkout out. + +Running %(prog)s without the '--status' option will always attempt to +synchronize the working copy to exactly match the externals description. +''' + + epilog = ''' +``` +NOTE: %(prog)s *MUST* be run from the root of the source tree it +is managing. For example, if you cloned a repository with: + + $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev + +Then the root of the source tree is /path/to/some-project-dev. If you +obtained a sub-project via a checkout of another project: + + $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev + +and you need to checkout the sub-project externals, then the root of the +source tree remains /path/to/some-project-dev. Do *NOT* run %(prog)s +from within /path/to/some-project-dev/sub-project + +The root of the source tree will be referred to as `${SRC_ROOT}` below. + + +# Supported workflows + + * Checkout all required components from the default externals + description file: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s + + * To update all required components to the current values in the + externals description file, re-run %(prog)s: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s + + If there are *any* modifications to *any* working copy according + to the git or svn 'status' command, %(prog)s + will not update any external repositories. Modifications + include: modified files, added files, removed files, or missing + files. + + To avoid this safety check, edit the externals description file + and comment out the modified external block. + + * Checkout all required components from a user specified externals + description file: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s --externals my-externals.cfg + + * Status summary of the repositories managed by %(prog)s: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s --status + + ./cime + s ./components/cism + ./components/mosart + e-o ./components/rtm + M ./src/fates + e-o ./tools/PTCLM + + + where: + * column one indicates the status of the repository in relation + to the externals description file. + * column two indicates whether the working copy has modified files. + * column three shows how the repository is managed, optional or required + + Column one will be one of these values: + * s : out-of-sync : repository is checked out at a different commit + compared with the externals description + * e : empty : directory does not exist - %(prog)s has not been run + * ? : unknown : directory exists but .git or .svn directories are missing + + Column two will be one of these values: + * M : Modified : modified, added, deleted or missing files + * : blank / space : clean + * - : dash : no meaningful state, for empty repositories + + Column three will be one of these values: + * o : optional : optionally repository + * : blank / space : required repository + + * Detailed git or svn status of the repositories managed by %(prog)s: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s --status --verbose + +# Externals description file + + The externals description contains a list of the external + repositories that are used and their version control locations. The + file format is the standard ini/cfg configuration file format. Each + external is defined by a section containing the component name in + square brackets: + + * name (string) : component name, e.g. [cime], [cism], etc. + + Each section has the following keyword-value pairs: + + * required (boolean) : whether the component is a required checkout, + 'true' or 'false'. + + * local_path (string) : component path *relative* to where + %(prog)s is called. + + * protoctol (string) : version control protocol that is used to + manage the component. Valid values are 'git', 'svn', + 'externals_only'. + + Switching an external between different protocols is not + supported, e.g. from svn to git. To switch protocols, you need to + manually move the old working copy to a new location. + + Note: 'externals_only' will only process the external's own + external description file without trying to manage a repository + for the component. This is used for retrieving externals for + standalone components like cam and ctsm which also serve as + sub-components within a larger project. If the source root of the + externals_only component is the same as the main source root, then + the local path must be set to '.', the unix current working + directory, e. g. 'local_path = .' + + * repo_url (string) : URL for the repository location, examples: + * https://svn-ccsm-models.cgd.ucar.edu/glc + * git@github.com:esmci/cime.git + * /path/to/local/repository + * . + + NOTE: To operate on only the local clone and and ignore remote + repositories, set the url to '.' (the unix current path), + i.e. 'repo_url = .' . This can be used to checkout a local branch + instead of the upstream branch. + + If a repo url is determined to be a local path (not a network url) + then user expansion, e.g. ~/, and environment variable expansion, + e.g. $HOME or $REPO_ROOT, will be performed. + + Relative paths are difficult to get correct, especially for mixed + use repos. It is advised that local paths expand to absolute paths. + If relative paths are used, they should be relative to one level + above local_path. If local path is 'src/foo', the the relative url + should be relative to 'src'. + + * tag (string) : tag to checkout + + * hash (string) : the git hash to checkout. Only applies to git + repositories. + + * branch (string) : branch to checkout from the specified + repository. Specifying a branch on a remote repository means that + %(prog)s will checkout the version of the branch in the remote, + not the the version in the local repository (if it exists). + + Note: one and only one of tag, branch hash must be supplied. + + * externals (string) : used to make manage_externals aware of + sub-externals required by an external. This is a relative path to + the external's root directory. For example, if LIBX is often used + as a sub-external, it might have an externals file (for its + externals) called Externals_LIBX.cfg. To use libx as a standalone + checkout, it would have another file, Externals.cfg with the + following entry: + + [ libx ] + local_path = . + protocol = externals_only + externals = Externals_LIBX.cfg + required = True + + Now, %(prog)s will process Externals.cfg and also process + Externals_LIBX.cfg as if it was a sub-external. + + * from_submodule (True / False) : used to pull the repo_url, local_path, + and hash properties for this external from the .gitmodules file in + this repository. Note that the section name (the entry in square + brackets) must match the name in the .gitmodules file. + If from_submodule is True, the protocol must be git and no repo_url, + local_path, hash, branch, or tag entries are allowed. + Default: False + + * sparse (string) : used to control a sparse checkout. This optional + entry should point to a filename (path relative to local_path) that + contains instructions on which repository paths to include (or + exclude) from the working tree. + See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree + Default: sparse checkout is disabled + + * Lines beginning with '#' or ';' are comments and will be ignored. + +# Obtaining this tool, reporting issues, etc. + + The master repository for manage_externals is + https://github.com/ESMCI/manage_externals. Any issues with this tool + should be reported there. + +# Troubleshooting + +Operations performed by manage_externals utilities are explicit and +data driven. %(prog)s will always attempt to make the working copy +*exactly* match what is in the externals file when modifying the +working copy of a repository. + +If %(prog)s is not doing what you expected, double check the contents +of the externals description file or examine the output of +./manage_externals/%(prog)s --status + +''' + + parser = argparse.ArgumentParser( + description=description, epilog=epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # + # user options + # + parser.add_argument("components", nargs="*", + help="Specific component(s) to checkout. By default, " + "all required externals are checked out.") + + parser.add_argument('-e', '--externals', nargs='?', + default='Externals.cfg', + help='The externals description filename. ' + 'Default: %(default)s.') + + parser.add_argument('-o', '--optional', action='store_true', default=False, + help='By default only the required externals ' + 'are checked out. This flag will also checkout the ' + 'optional externals.') + + parser.add_argument('-S', '--status', action='store_true', default=False, + help='Output the status of the repositories managed by ' + '%(prog)s. By default only summary information ' + 'is provided. Use the verbose option to see details.') + + parser.add_argument('-v', '--verbose', action='count', default=0, + help='Output additional information to ' + 'the screen and log file. This flag can be ' + 'used up to two times, increasing the ' + 'verbosity level each time.') + + parser.add_argument('--svn-ignore-ancestry', action='store_true', default=False, + help='By default, subversion will abort if a component is ' + 'already checked out and there is no common ancestry with ' + 'the new URL. This flag passes the "--ignore-ancestry" flag ' + 'to the svn switch call. (This is not recommended unless ' + 'you are sure about what you are doing.)') + + # + # developer options + # + parser.add_argument('--backtrace', action='store_true', + help='DEVELOPER: show exception backtraces as extra ' + 'debugging output') + + parser.add_argument('-d', '--debug', action='store_true', default=False, + help='DEVELOPER: output additional debugging ' + 'information to the screen and log file.') + + logging_group = parser.add_mutually_exclusive_group() + + logging_group.add_argument('--logging', dest='do_logging', + action='store_true', + help='DEVELOPER: enable logging.') + logging_group.add_argument('--no-logging', dest='do_logging', + action='store_false', default=False, + help='DEVELOPER: disable logging ' + '(this is the default)') + + if args: + options = parser.parse_args(args) + else: + options = parser.parse_args() + return options + + +# --------------------------------------------------------------------- +# +# main +# +# --------------------------------------------------------------------- +def main(args): + """ + Function to call when module is called from the command line. + Parse externals file and load required repositories or all repositories if + the --all option is passed. + + Returns a tuple (overall_status, tree_status). overall_status is 0 + on success, non-zero on failure. tree_status gives the full status + *before* executing the checkout command - i.e., the status that it + used to determine if it's safe to proceed with the checkout. + """ + if args.do_logging: + logging.basicConfig(filename=LOG_FILE_NAME, + format='%(levelname)s : %(asctime)s : %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=logging.DEBUG) + + program_name = os.path.basename(sys.argv[0]) + logging.info('Beginning of %s', program_name) + + load_all = False + if args.optional: + load_all = True + + root_dir = os.path.abspath(os.getcwd()) + external_data = read_externals_description_file(root_dir, args.externals) + external = create_externals_description( + external_data, components=args.components) + + for comp in args.components: + if comp not in external.keys(): + fatal_error( + "No component {} found in {}".format( + comp, args.externals)) + + source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) + printlog('Checking status of externals: ', end='') + tree_status = source_tree.status() + printlog('') + + if args.status: + # user requested status-only + for comp in sorted(tree_status.keys()): + tree_status[comp].log_status_message(args.verbose) + else: + # checkout / update the external repositories. + safe_to_update = check_safe_to_update_repos(tree_status) + if not safe_to_update: + # print status + for comp in sorted(tree_status.keys()): + tree_status[comp].log_status_message(args.verbose) + # exit gracefully + msg = """The external repositories labeled with 'M' above are not in a clean state. + +The following are two options for how to proceed: + +(1) Go into each external that is not in a clean state and issue either + an 'svn status' or a 'git status' command. Either revert or commit + your changes so that all externals are in a clean state. (Note, + though, that it is okay to have untracked files in your working + directory.) Then rerun {program_name}. + +(2) Alternatively, you do not have to rely on {program_name}. Instead, you + can manually update out-of-sync externals (labeled with 's' above) + as described in the configuration file {config_file}. + + +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before re-running the +checkout_externals tool. +""".format(program_name=program_name, config_file=args.externals) + + printlog('-' * 70) + printlog(msg) + printlog('-' * 70) + else: + if not args.components: + source_tree.checkout(args.verbose, load_all) + for comp in args.components: + source_tree.checkout(args.verbose, load_all, load_comp=comp) + printlog('') + + logging.info('%s completed without exceptions.', program_name) + # NOTE(bja, 2017-11) tree status is used by the systems tests + return 0, tree_status diff --git a/util/manage_externals/manic/externals_description.py b/util/manage_externals/manic/externals_description.py new file mode 100644 index 0000000000..b0c4f736a7 --- /dev/null +++ b/util/manage_externals/manic/externals_description.py @@ -0,0 +1,794 @@ +#!/usr/bin/env python + +"""Model description + +Model description is the representation of the various externals +included in the model. It processes in input data structure, and +converts it into a standard interface that is used by the rest of the +system. + +To maintain backward compatibility, externals description files should +follow semantic versioning rules, http://semver.org/ + + + +""" +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import logging +import os +import os.path +import re + +# ConfigParser in python2 was renamed to configparser in python3. +# In python2, ConfigParser returns byte strings, str, instead of unicode. +# We need unicode to be compatible with xml and json parser and python3. +try: + # python2 + from ConfigParser import SafeConfigParser as config_parser + from ConfigParser import MissingSectionHeaderError + from ConfigParser import NoSectionError, NoOptionError + + USE_PYTHON2 = True + + def config_string_cleaner(text): + """convert strings into unicode + """ + return text.decode('utf-8') +except ImportError: + # python3 + from configparser import ConfigParser as config_parser + from configparser import MissingSectionHeaderError + from configparser import NoSectionError, NoOptionError + + USE_PYTHON2 = False + + def config_string_cleaner(text): + """Python3 already uses unicode strings, so just return the string + without modification. + + """ + return text + +from .utils import printlog, fatal_error, str_to_bool, expand_local_url +from .utils import execute_subprocess +from .global_constants import EMPTY_STR, PPRINTER, VERSION_SEPERATOR + +# +# Globals +# +DESCRIPTION_SECTION = 'externals_description' +VERSION_ITEM = 'schema_version' + + +def read_externals_description_file(root_dir, file_name): + """Read a file containing an externals description and + create its internal representation. + + """ + root_dir = os.path.abspath(root_dir) + msg = 'In directory : {0}'.format(root_dir) + logging.info(msg) + printlog('Processing externals description file : {0}'.format(file_name)) + + file_path = os.path.join(root_dir, file_name) + if not os.path.exists(file_name): + if file_name.lower() == "none": + msg = ('INTERNAL ERROR: Attempt to read externals file ' + 'from {0} when not configured'.format(file_path)) + else: + msg = ('ERROR: Model description file, "{0}", does not ' + 'exist at path:\n {1}\nDid you run from the root of ' + 'the source tree?'.format(file_name, file_path)) + + fatal_error(msg) + + externals_description = None + if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: + externals_description = read_gitmodules_file(root_dir, file_name) + else: + try: + config = config_parser() + config.read(file_path) + externals_description = config + except MissingSectionHeaderError: + # not a cfg file + pass + + if externals_description is None: + msg = 'Unknown file format!' + fatal_error(msg) + + return externals_description + +class LstripReader(object): + "LstripReader formats .gitmodules files to be acceptable for configparser" + def __init__(self, filename): + with open(filename, 'r') as infile: + lines = infile.readlines() + self._lines = list() + self._num_lines = len(lines) + self._index = 0 + for line in lines: + self._lines.append(line.lstrip()) + + def readlines(self): + """Return all the lines from this object's file""" + return self._lines + + def readline(self, size=-1): + """Format and return the next line or raise StopIteration""" + try: + line = self.next() + except StopIteration: + line = '' + + if (size > 0) and (len(line) < size): + return line[0:size] + + return line + + def __iter__(self): + """Begin an iteration""" + self._index = 0 + return self + + def next(self): + """Return the next line or raise StopIteration""" + if self._index >= self._num_lines: + raise StopIteration + + self._index = self._index + 1 + return self._lines[self._index - 1] + + def __next__(self): + return self.next() + +def git_submodule_status(repo_dir): + """Run the git submodule status command to obtain submodule hashes. + """ + # This function is here instead of GitRepository to avoid a dependency loop + cwd = os.getcwd() + os.chdir(repo_dir) + cmd = ['git', 'submodule', 'status'] + git_output = execute_subprocess(cmd, output_to_caller=True) + submodules = {} + submods = git_output.split('\n') + for submod in submods: + if submod: + status = submod[0] + items = submod[1:].split(' ') + if len(items) > 2: + tag = items[2] + else: + tag = None + + submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} + + os.chdir(cwd) + return submodules + +def parse_submodules_desc_section(section_items, file_path): + """Find the path and url for this submodule description""" + path = None + url = None + for item in section_items: + name = item[0].strip().lower() + if name == 'path': + path = item[1].strip() + elif name == 'url': + url = item[1].strip() + else: + msg = 'WARNING: Ignoring unknown {} property, in {}' + msg = msg.format(item[0], file_path) # fool pylint + logging.warning(msg) + + return path, url + +def read_gitmodules_file(root_dir, file_name): + # pylint: disable=deprecated-method + # Disabling this check because the method is only used for python2 + """Read a .gitmodules file and convert it to be compatible with an + externals description. + """ + root_dir = os.path.abspath(root_dir) + msg = 'In directory : {0}'.format(root_dir) + logging.info(msg) + printlog('Processing submodules description file : {0}'.format(file_name)) + + file_path = os.path.join(root_dir, file_name) + if not os.path.exists(file_name): + msg = ('ERROR: submodules description file, "{0}", does not ' + 'exist at path:\n {1}'.format(file_name, file_path)) + fatal_error(msg) + + submodules_description = None + externals_description = None + try: + config = config_parser() + if USE_PYTHON2: + config.readfp(LstripReader(file_path), filename=file_name) + else: + config.read_file(LstripReader(file_path), source=file_name) + + submodules_description = config + except MissingSectionHeaderError: + # not a cfg file + pass + + if submodules_description is None: + msg = 'Unknown file format!' + fatal_error(msg) + else: + # Convert the submodules description to an externals description + externals_description = config_parser() + # We need to grab all the commit hashes for this repo + submods = git_submodule_status(root_dir) + for section in submodules_description.sections(): + if section[0:9] == 'submodule': + sec_name = section[9:].strip(' "') + externals_description.add_section(sec_name) + section_items = submodules_description.items(section) + path, url = parse_submodules_desc_section(section_items, + file_path) + + if path is None: + msg = 'Submodule {} missing path'.format(sec_name) + fatal_error(msg) + + if url is None: + msg = 'Submodule {} missing url'.format(sec_name) + fatal_error(msg) + + externals_description.set(sec_name, + ExternalsDescription.PATH, path) + externals_description.set(sec_name, + ExternalsDescription.PROTOCOL, 'git') + externals_description.set(sec_name, + ExternalsDescription.REPO_URL, url) + externals_description.set(sec_name, + ExternalsDescription.REQUIRED, 'True') + git_hash = submods[sec_name]['hash'] + externals_description.set(sec_name, + ExternalsDescription.HASH, git_hash) + + # Required items + externals_description.add_section(DESCRIPTION_SECTION) + externals_description.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0') + + return externals_description + +def create_externals_description( + model_data, model_format='cfg', components=None, parent_repo=None): + """Create the a externals description object from the provided data + """ + externals_description = None + if model_format == 'dict': + externals_description = ExternalsDescriptionDict( + model_data, components=components) + elif model_format == 'cfg': + major, _, _ = get_cfg_schema_version(model_data) + if major == 1: + externals_description = ExternalsDescriptionConfigV1( + model_data, components=components, parent_repo=parent_repo) + else: + msg = ('Externals description file has unsupported schema ' + 'version "{0}".'.format(major)) + fatal_error(msg) + else: + msg = 'Unknown model data format "{0}"'.format(model_format) + fatal_error(msg) + return externals_description + + +def get_cfg_schema_version(model_cfg): + """Extract the major, minor, patch version of the config file schema + + Params: + model_cfg - config parser object containing the externas description data + + Returns: + major = integer major version + minor = integer minor version + patch = integer patch version + """ + semver_str = '' + try: + semver_str = model_cfg.get(DESCRIPTION_SECTION, VERSION_ITEM) + except (NoSectionError, NoOptionError): + msg = ('externals description file must have the required ' + 'section: "{0}" and item "{1}"'.format(DESCRIPTION_SECTION, + VERSION_ITEM)) + fatal_error(msg) + + # NOTE(bja, 2017-11) Assume we don't care about the + # build/pre-release metadata for now! + version_list = re.split(r'[-+]', semver_str) + version_str = version_list[0] + version = version_str.split(VERSION_SEPERATOR) + try: + major = int(version[0].strip()) + minor = int(version[1].strip()) + patch = int(version[2].strip()) + except ValueError: + msg = ('Config file schema version must have integer digits for ' + 'major, minor and patch versions. ' + 'Received "{0}"'.format(version_str)) + fatal_error(msg) + return major, minor, patch + + +class ExternalsDescription(dict): + """Base externals description class that is independent of the user input + format. Different input formats can all be converted to this + representation to provide a consistent represtentation for the + rest of the objects in the system. + + NOTE(bja, 2018-03): do NOT define _schema_major etc at the class + level in the base class. The nested/recursive nature of externals + means different schema versions may be present in a single run! + + All inheriting classes must overwrite: + self._schema_major and self._input_major + self._schema_minor and self._input_minor + self._schema_patch and self._input_patch + + where _schema_x is the supported schema, _input_x is the user + input value. + + """ + # keywords defining the interface into the externals description data + EXTERNALS = 'externals' + BRANCH = 'branch' + SUBMODULE = 'from_submodule' + HASH = 'hash' + NAME = 'name' + PATH = 'local_path' + PROTOCOL = 'protocol' + REPO = 'repo' + REPO_URL = 'repo_url' + REQUIRED = 'required' + TAG = 'tag' + SPARSE = 'sparse' + + PROTOCOL_EXTERNALS_ONLY = 'externals_only' + PROTOCOL_GIT = 'git' + PROTOCOL_SVN = 'svn' + GIT_SUBMODULES_FILENAME = '.gitmodules' + KNOWN_PRROTOCOLS = [PROTOCOL_GIT, PROTOCOL_SVN, PROTOCOL_EXTERNALS_ONLY] + + # v1 xml keywords + _V1_TREE_PATH = 'TREE_PATH' + _V1_ROOT = 'ROOT' + _V1_TAG = 'TAG' + _V1_BRANCH = 'BRANCH' + _V1_REQ_SOURCE = 'REQ_SOURCE' + + _source_schema = {REQUIRED: True, + PATH: 'string', + EXTERNALS: 'string', + SUBMODULE : True, + REPO: {PROTOCOL: 'string', + REPO_URL: 'string', + TAG: 'string', + BRANCH: 'string', + HASH: 'string', + SPARSE: 'string', + } + } + + def __init__(self, parent_repo=None): + """Convert the xml into a standardized dict that can be used to + construct the source objects + + """ + dict.__init__(self) + + self._schema_major = None + self._schema_minor = None + self._schema_patch = None + self._input_major = None + self._input_minor = None + self._input_patch = None + self._parent_repo = parent_repo + + def _verify_schema_version(self): + """Use semantic versioning rules to verify we can process this schema. + + """ + known = '{0}.{1}.{2}'.format(self._schema_major, + self._schema_minor, + self._schema_patch) + received = '{0}.{1}.{2}'.format(self._input_major, + self._input_minor, + self._input_patch) + + if self._input_major != self._schema_major: + # should never get here, the factory should handle this correctly! + msg = ('DEV_ERROR: version "{0}" parser received ' + 'version "{1}" input.'.format(known, received)) + fatal_error(msg) + + if self._input_minor > self._schema_minor: + msg = ('Incompatible schema version:\n' + ' User supplied schema version "{0}" is too new."\n' + ' Can only process version "{1}" files and ' + 'older.'.format(received, known)) + fatal_error(msg) + + if self._input_patch > self._schema_patch: + # NOTE(bja, 2018-03) ignoring for now... Not clear what + # conditions the test is needed. + pass + + def _check_user_input(self): + """Run a series of checks to attempt to validate the user input and + detect errors as soon as possible. + + NOTE(bja, 2018-03) These checks are called *after* the file is + read. That means the schema check can not occur here. + + Note: the order is important. check_optional will create + optional with null data. run check_data first to ensure + required data was provided correctly by the user. + + """ + self._check_data() + self._check_optional() + self._validate() + + def _check_data(self): + # pylint: disable=too-many-branches,too-many-statements + """Check user supplied data is valid where possible. + """ + for ext_name in self.keys(): + if (self[ext_name][self.REPO][self.PROTOCOL] + not in self.KNOWN_PRROTOCOLS): + msg = 'Unknown repository protocol "{0}" in "{1}".'.format( + self[ext_name][self.REPO][self.PROTOCOL], ext_name) + fatal_error(msg) + + if (self[ext_name][self.REPO][self.PROTOCOL] == + self.PROTOCOL_SVN): + if self.HASH in self[ext_name][self.REPO]: + msg = ('In repo description for "{0}". svn repositories ' + 'may not include the "hash" keyword.'.format( + ext_name)) + fatal_error(msg) + + if ((self[ext_name][self.REPO][self.PROTOCOL] != self.PROTOCOL_GIT) + and (self.SUBMODULE in self[ext_name])): + msg = ('self.SUBMODULE is only supported with {0} protocol, ' + '"{1}" is defined as an {2} repository') + fatal_error(msg.format(self.PROTOCOL_GIT, ext_name, + self[ext_name][self.REPO][self.PROTOCOL])) + + if (self[ext_name][self.REPO][self.PROTOCOL] != + self.PROTOCOL_EXTERNALS_ONLY): + ref_count = 0 + found_refs = '' + if self.TAG in self[ext_name][self.REPO]: + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.TAG, self[ext_name][self.REPO][self.TAG], + found_refs) + if self.BRANCH in self[ext_name][self.REPO]: + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.BRANCH, self[ext_name][self.REPO][self.BRANCH], + found_refs) + if self.HASH in self[ext_name][self.REPO]: + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.HASH, self[ext_name][self.REPO][self.HASH], + found_refs) + if (self.SUBMODULE in self[ext_name] and + self[ext_name][self.SUBMODULE]): + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.SUBMODULE, + self[ext_name][self.SUBMODULE], found_refs) + + if ref_count > 1: + msg = 'Model description is over specified! ' + if self.SUBMODULE in self[ext_name]: + msg += ('from_submodule is not compatible with ' + '"tag", "branch", or "hash" ') + else: + msg += (' Only one of "tag", "branch", or "hash" ' + 'may be specified ') + + msg += 'for repo description of "{0}".'.format(ext_name) + msg = '{0}\nFound: {1}'.format(msg, found_refs) + fatal_error(msg) + elif ref_count < 1: + msg = ('Model description is under specified! One of ' + '"tag", "branch", or "hash" must be specified for ' + 'repo description of "{0}"'.format(ext_name)) + fatal_error(msg) + + if (self.REPO_URL not in self[ext_name][self.REPO] and + (self.SUBMODULE not in self[ext_name] or + not self[ext_name][self.SUBMODULE])): + msg = ('Model description is under specified! Must have ' + '"repo_url" in repo ' + 'description for "{0}"'.format(ext_name)) + fatal_error(msg) + + if (self.SUBMODULE in self[ext_name] and + self[ext_name][self.SUBMODULE]): + if self.REPO_URL in self[ext_name][self.REPO]: + msg = ('Model description is over specified! ' + 'from_submodule keyword is not compatible ' + 'with {0} keyword for'.format(self.REPO_URL)) + msg = '{0} repo description of "{1}"'.format(msg, + ext_name) + fatal_error(msg) + + if self.PATH in self[ext_name]: + msg = ('Model description is over specified! ' + 'from_submodule keyword is not compatible with ' + '{0} keyword for'.format(self.PATH)) + msg = '{0} repo description of "{1}"'.format(msg, + ext_name) + fatal_error(msg) + + if self.REPO_URL in self[ext_name][self.REPO]: + url = expand_local_url( + self[ext_name][self.REPO][self.REPO_URL], ext_name) + self[ext_name][self.REPO][self.REPO_URL] = url + + def _check_optional(self): + # pylint: disable=too-many-branches + """Some fields like externals, repo:tag repo:branch are + (conditionally) optional. We don't want the user to be + required to enter them in every externals description file, but + still want to validate the input. Check conditions and add + default values if appropriate. + + """ + submod_desc = None # Only load submodules info once + for field in self: + # truely optional + if self.EXTERNALS not in self[field]: + self[field][self.EXTERNALS] = EMPTY_STR + + # git and svn repos must tags and branches for validation purposes. + if self.TAG not in self[field][self.REPO]: + self[field][self.REPO][self.TAG] = EMPTY_STR + if self.BRANCH not in self[field][self.REPO]: + self[field][self.REPO][self.BRANCH] = EMPTY_STR + if self.HASH not in self[field][self.REPO]: + self[field][self.REPO][self.HASH] = EMPTY_STR + if self.REPO_URL not in self[field][self.REPO]: + self[field][self.REPO][self.REPO_URL] = EMPTY_STR + if self.SPARSE not in self[field][self.REPO]: + self[field][self.REPO][self.SPARSE] = EMPTY_STR + + # from_submodule has a complex relationship with other fields + if self.SUBMODULE in self[field]: + # User wants to use submodule information, is it available? + if self._parent_repo is None: + # No parent == no submodule information + PPRINTER.pprint(self[field]) + msg = 'No parent submodule for "{0}"'.format(field) + fatal_error(msg) + elif self._parent_repo.protocol() != self.PROTOCOL_GIT: + PPRINTER.pprint(self[field]) + msg = 'Parent protocol, "{0}", does not support submodules' + fatal_error(msg.format(self._parent_repo.protocol())) + else: + args = self._repo_config_from_submodule(field, submod_desc) + repo_url, repo_path, ref_hash, submod_desc = args + + if repo_url is None: + msg = ('Cannot checkout "{0}" as a submodule, ' + 'repo not found in {1} file') + fatal_error(msg.format(field, + self.GIT_SUBMODULES_FILENAME)) + # Fill in submodule fields + self[field][self.REPO][self.REPO_URL] = repo_url + self[field][self.REPO][self.HASH] = ref_hash + self[field][self.PATH] = repo_path + + if self[field][self.SUBMODULE]: + # We should get everything from the parent submodule + # configuration. + pass + # No else (from _submodule = False is the default) + else: + # Add the default value (not using submodule information) + self[field][self.SUBMODULE] = False + + def _repo_config_from_submodule(self, field, submod_desc): + """Find the external config information for a repository from + its submodule configuration information. + """ + if submod_desc is None: + repo_path = os.getcwd() # Is this always correct? + submod_file = self._parent_repo.submodules_file(repo_path=repo_path) + if submod_file is None: + msg = ('Cannot checkout "{0}" from submodule information\n' + ' Parent repo, "{1}" does not have submodules') + fatal_error(msg.format(field, self._parent_repo.name())) + + submod_file = read_gitmodules_file(repo_path, submod_file) + submod_desc = create_externals_description(submod_file) + + # Can we find our external? + repo_url = None + repo_path = None + ref_hash = None + for ext_field in submod_desc: + if field == ext_field: + ext = submod_desc[ext_field] + repo_url = ext[self.REPO][self.REPO_URL] + repo_path = ext[self.PATH] + ref_hash = ext[self.REPO][self.HASH] + break + + return repo_url, repo_path, ref_hash, submod_desc + + def _validate(self): + """Validate that the parsed externals description contains all necessary + fields. + + """ + def print_compare_difference(data_a, data_b, loc_a, loc_b): + """Look through the data structures and print the differences. + + """ + for item in data_a: + if item in data_b: + if not isinstance(data_b[item], type(data_a[item])): + printlog(" {item}: {loc} = {val} ({val_type})".format( + item=item, loc=loc_a, val=data_a[item], + val_type=type(data_a[item]))) + printlog(" {item} {loc} = {val} ({val_type})".format( + item=' ' * len(item), loc=loc_b, val=data_b[item], + val_type=type(data_b[item]))) + else: + printlog(" {item}: {loc} = {val} ({val_type})".format( + item=item, loc=loc_a, val=data_a[item], + val_type=type(data_a[item]))) + printlog(" {item} {loc} missing".format( + item=' ' * len(item), loc=loc_b)) + + def validate_data_struct(schema, data): + """Compare a data structure against a schema and validate all required + fields are present. + + """ + is_valid = False + in_ref = True + valid = True + if isinstance(schema, dict) and isinstance(data, dict): + # Both are dicts, recursively verify that all fields + # in schema are present in the data. + for key in schema: + in_ref = in_ref and (key in data) + if in_ref: + valid = valid and ( + validate_data_struct(schema[key], data[key])) + + is_valid = in_ref and valid + else: + # non-recursive structure. verify data and schema have + # the same type. + is_valid = isinstance(data, type(schema)) + + if not is_valid: + printlog(" Unmatched schema and input:") + if isinstance(schema, dict): + print_compare_difference(schema, data, 'schema', 'input') + print_compare_difference(data, schema, 'input', 'schema') + else: + printlog(" schema = {0} ({1})".format( + schema, type(schema))) + printlog(" input = {0} ({1})".format(data, type(data))) + + return is_valid + + for field in self: + valid = validate_data_struct(self._source_schema, self[field]) + if not valid: + PPRINTER.pprint(self._source_schema) + PPRINTER.pprint(self[field]) + msg = 'ERROR: source for "{0}" did not validate'.format(field) + fatal_error(msg) + + +class ExternalsDescriptionDict(ExternalsDescription): + """Create a externals description object from a dictionary using the API + representations. Primarily used to simplify creating model + description files for unit testing. + + """ + + def __init__(self, model_data, components=None): + """Parse a native dictionary into a externals description. + """ + ExternalsDescription.__init__(self) + self._schema_major = 1 + self._schema_minor = 0 + self._schema_patch = 0 + self._input_major = 1 + self._input_minor = 0 + self._input_patch = 0 + self._verify_schema_version() + if components: + for key in model_data.items(): + if key not in components: + del model_data[key] + + self.update(model_data) + self._check_user_input() + + +class ExternalsDescriptionConfigV1(ExternalsDescription): + """Create a externals description object from a config_parser object, + schema version 1. + + """ + + def __init__(self, model_data, components=None, parent_repo=None): + """Convert the config data into a standardized dict that can be used to + construct the source objects + + """ + ExternalsDescription.__init__(self, parent_repo=parent_repo) + self._schema_major = 1 + self._schema_minor = 1 + self._schema_patch = 0 + self._input_major, self._input_minor, self._input_patch = \ + get_cfg_schema_version(model_data) + self._verify_schema_version() + self._remove_metadata(model_data) + self._parse_cfg(model_data, components=components) + self._check_user_input() + + @staticmethod + def _remove_metadata(model_data): + """Remove the metadata section from the model configuration file so + that it is simpler to look through the file and construct the + externals description. + + """ + model_data.remove_section(DESCRIPTION_SECTION) + + def _parse_cfg(self, cfg_data, components=None): + """Parse a config_parser object into a externals description. + """ + def list_to_dict(input_list, convert_to_lower_case=True): + """Convert a list of key-value pairs into a dictionary. + """ + output_dict = {} + for item in input_list: + key = config_string_cleaner(item[0].strip()) + value = config_string_cleaner(item[1].strip()) + if convert_to_lower_case: + key = key.lower() + output_dict[key] = value + return output_dict + + for section in cfg_data.sections(): + name = config_string_cleaner(section.lower().strip()) + if components and name not in components: + continue + self[name] = {} + self[name].update(list_to_dict(cfg_data.items(section))) + self[name][self.REPO] = {} + loop_keys = self[name].copy().keys() + for item in loop_keys: + if item in self._source_schema: + if isinstance(self._source_schema[item], bool): + self[name][item] = str_to_bool(self[name][item]) + elif item in self._source_schema[self.REPO]: + self[name][self.REPO][item] = self[name][item] + del self[name][item] + else: + msg = ('Invalid input: "{sect}" contains unknown ' + 'item "{item}".'.format(sect=name, item=item)) + fatal_error(msg) diff --git a/util/manage_externals/manic/externals_status.py b/util/manage_externals/manic/externals_status.py new file mode 100644 index 0000000000..d3d238f289 --- /dev/null +++ b/util/manage_externals/manic/externals_status.py @@ -0,0 +1,164 @@ +"""ExternalStatus + +Class to store status and state information about repositories and +create a string representation. + +""" +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +from .global_constants import EMPTY_STR +from .utils import printlog, indent_string +from .global_constants import VERBOSITY_VERBOSE, VERBOSITY_DUMP + + +class ExternalStatus(object): + """Class to represent the status of a given source repository or tree. + + Individual repositories determine their own status in the + Repository objects. This object is just resposible for storing the + information and passing it up to a higher level for reporting or + global decisions. + + There are two states of concern: + + * If the repository is in-sync with the externals description file. + + * If the repostiory working copy is clean and there are no pending + transactions (e.g. add, remove, rename, untracked files). + + """ + DEFAULT = '-' + UNKNOWN = '?' + EMPTY = 'e' + MODEL_MODIFIED = 's' # a.k.a. out-of-sync + DIRTY = 'M' + + STATUS_OK = ' ' + STATUS_ERROR = '!' + + # source types + OPTIONAL = 'o' + STANDALONE = 's' + MANAGED = ' ' + + def __init__(self): + self.sync_state = self.DEFAULT + self.clean_state = self.DEFAULT + self.source_type = self.DEFAULT + self.path = EMPTY_STR + self.current_version = EMPTY_STR + self.expected_version = EMPTY_STR + self.status_output = EMPTY_STR + + def log_status_message(self, verbosity): + """Write status message to the screen and log file + """ + self._default_status_message() + if verbosity >= VERBOSITY_VERBOSE: + self._verbose_status_message() + if verbosity >= VERBOSITY_DUMP: + self._dump_status_message() + + def _default_status_message(self): + """Return the default terse status message string + """ + msg = '{sync}{clean}{src_type} {path}'.format( + sync=self.sync_state, clean=self.clean_state, + src_type=self.source_type, path=self.path) + printlog(msg) + + def _verbose_status_message(self): + """Return the verbose status message string + """ + clean_str = self.DEFAULT + if self.clean_state == self.STATUS_OK: + clean_str = 'clean sandbox' + elif self.clean_state == self.DIRTY: + clean_str = 'modified sandbox' + + sync_str = 'on {0}'.format(self.current_version) + if self.sync_state != self.STATUS_OK: + sync_str = '{current} --> {expected}'.format( + current=self.current_version, expected=self.expected_version) + msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) + printlog(msg) + + def _dump_status_message(self): + """Return the dump status message string + """ + msg = indent_string(self.status_output, 12) + printlog(msg) + + def safe_to_update(self): + """Report if it is safe to update a repository. Safe is defined as: + + * If a repository is empty, it is safe to update. + + * If a repository exists and has a clean working copy state + with no pending transactions. + + """ + safe_to_update = False + repo_exists = self.exists() + if not repo_exists: + safe_to_update = True + else: + # If the repo exists, it must be in ok or modified + # sync_state. Any other sync_state at this point + # represents a logic error that should have been handled + # before now! + sync_safe = ((self.sync_state == ExternalStatus.STATUS_OK) or + (self.sync_state == ExternalStatus.MODEL_MODIFIED)) + if sync_safe: + # The clean_state must be STATUS_OK to update. Otherwise we + # are dirty or there was a missed error previously. + if self.clean_state == ExternalStatus.STATUS_OK: + safe_to_update = True + return safe_to_update + + def exists(self): + """Determine if the repo exists. This is indicated by: + + * sync_state is not EMPTY + + * if the sync_state is empty, then the valid states for + clean_state are default, empty or unknown. Anything else + and there was probably an internal logic error. + + NOTE(bja, 2017-10) For the moment we are considering a + sync_state of default or unknown to require user intervention, + but we may want to relax this convention. This is probably a + result of a network error or internal logic error but more + testing is needed. + + """ + is_empty = (self.sync_state == ExternalStatus.EMPTY) + clean_valid = ((self.clean_state == ExternalStatus.DEFAULT) or + (self.clean_state == ExternalStatus.EMPTY) or + (self.clean_state == ExternalStatus.UNKNOWN)) + + if is_empty and clean_valid: + exists = False + else: + exists = True + return exists + + +def check_safe_to_update_repos(tree_status): + """Check if *ALL* repositories are in a safe state to update. We don't + want to do a partial update of the repositories then die, leaving + the model in an inconsistent state. + + Note: if there is an update to do, the repositories will by + definiation be out of synce with the externals description, so we + can't use that as criteria for updating. + + """ + safe_to_update = True + for comp in tree_status: + stat = tree_status[comp] + safe_to_update &= stat.safe_to_update() + + return safe_to_update diff --git a/util/manage_externals/manic/global_constants.py b/util/manage_externals/manic/global_constants.py new file mode 100644 index 0000000000..0e91cffc90 --- /dev/null +++ b/util/manage_externals/manic/global_constants.py @@ -0,0 +1,18 @@ +"""Globals shared across modules +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import pprint + +EMPTY_STR = '' +LOCAL_PATH_INDICATOR = '.' +VERSION_SEPERATOR = '.' +LOG_FILE_NAME = 'manage_externals.log' +PPRINTER = pprint.PrettyPrinter(indent=4) + +VERBOSITY_DEFAULT = 0 +VERBOSITY_VERBOSE = 1 +VERBOSITY_DUMP = 2 diff --git a/util/manage_externals/manic/repository.py b/util/manage_externals/manic/repository.py new file mode 100644 index 0000000000..ea4230fb7b --- /dev/null +++ b/util/manage_externals/manic/repository.py @@ -0,0 +1,98 @@ +"""Base class representation of a repository +""" + +from .externals_description import ExternalsDescription +from .utils import fatal_error +from .global_constants import EMPTY_STR + + +class Repository(object): + """ + Class to represent and operate on a repository description. + """ + + def __init__(self, component_name, repo): + """ + Parse repo externals description + """ + self._name = component_name + self._protocol = repo[ExternalsDescription.PROTOCOL] + self._tag = repo[ExternalsDescription.TAG] + self._branch = repo[ExternalsDescription.BRANCH] + self._hash = repo[ExternalsDescription.HASH] + self._url = repo[ExternalsDescription.REPO_URL] + self._sparse = repo[ExternalsDescription.SPARSE] + + if self._url is EMPTY_STR: + fatal_error('repo must have a URL') + + if ((self._tag is EMPTY_STR) and (self._branch is EMPTY_STR) and + (self._hash is EMPTY_STR)): + fatal_error('{0} repo must have a branch, tag or hash element') + + ref_count = 0 + if self._tag is not EMPTY_STR: + ref_count += 1 + if self._branch is not EMPTY_STR: + ref_count += 1 + if self._hash is not EMPTY_STR: + ref_count += 1 + if ref_count != 1: + fatal_error('repo {0} must have exactly one of ' + 'tag, branch or hash.'.format(self._name)) + + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the source. + If the repo destination directory does not exist, checkout the correce + branch or tag. + NB: is include as an argument for compatibility with + git functionality (repository_git.py) + """ + msg = ('DEV_ERROR: checkout method must be implemented in all ' + 'repository classes! {0}'.format(self.__class__.__name__)) + fatal_error(msg) + + def status(self, stat, repo_dir_path): # pylint: disable=unused-argument + """Report the status of the repo + + """ + msg = ('DEV_ERROR: status method must be implemented in all ' + 'repository classes! {0}'.format(self.__class__.__name__)) + fatal_error(msg) + + def submodules_file(self, repo_path=None): + # pylint: disable=no-self-use,unused-argument + """Stub for use by non-git VC systems""" + return None + + def url(self): + """Public access of repo url. + """ + return self._url + + def tag(self): + """Public access of repo tag + """ + return self._tag + + def branch(self): + """Public access of repo branch. + """ + return self._branch + + def hash(self): + """Public access of repo hash. + """ + return self._hash + + def name(self): + """Public access of repo name. + """ + return self._name + + def protocol(self): + """Public access of repo protocol. + """ + return self._protocol diff --git a/util/manage_externals/manic/repository_factory.py b/util/manage_externals/manic/repository_factory.py new file mode 100644 index 0000000000..80a92a9d8a --- /dev/null +++ b/util/manage_externals/manic/repository_factory.py @@ -0,0 +1,29 @@ +"""Factory for creating and initializing the appropriate repository class +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +from .repository_git import GitRepository +from .repository_svn import SvnRepository +from .externals_description import ExternalsDescription +from .utils import fatal_error + + +def create_repository(component_name, repo_info, svn_ignore_ancestry=False): + """Determine what type of repository we have, i.e. git or svn, and + create the appropriate object. + + """ + protocol = repo_info[ExternalsDescription.PROTOCOL].lower() + if protocol == 'git': + repo = GitRepository(component_name, repo_info) + elif protocol == 'svn': + repo = SvnRepository(component_name, repo_info, ignore_ancestry=svn_ignore_ancestry) + elif protocol == 'externals_only': + repo = None + else: + msg = 'Unknown repo protocol "{0}"'.format(protocol) + fatal_error(msg) + return repo diff --git a/util/manage_externals/manic/repository_git.py b/util/manage_externals/manic/repository_git.py new file mode 100644 index 0000000000..f986051001 --- /dev/null +++ b/util/manage_externals/manic/repository_git.py @@ -0,0 +1,819 @@ +"""Class for interacting with git repositories +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import copy +import os + +from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR +from .global_constants import VERBOSITY_VERBOSE +from .repository import Repository +from .externals_status import ExternalStatus +from .externals_description import ExternalsDescription, git_submodule_status +from .utils import expand_local_url, split_remote_url, is_remote_url +from .utils import fatal_error, printlog +from .utils import execute_subprocess + + +class GitRepository(Repository): + """Class to represent and operate on a repository description. + + For testing purpose, all system calls to git should: + + * be isolated in separate functions with no application logic + * of the form: + - cmd = ['git', ...] + - value = execute_subprocess(cmd, output_to_caller={T|F}, + status_to_caller={T|F}) + - return value + * be static methods (not rely on self) + * name as _git_subcommand_args(user_args) + + This convention allows easy unit testing of the repository logic + by mocking the specific calls to return predefined results. + + """ + + def __init__(self, component_name, repo): + """ + Parse repo (a XML element). + """ + Repository.__init__(self, component_name, repo) + self._gitmodules = None + self._submods = None + + # ---------------------------------------------------------------- + # + # Public API, defined by Repository + # + # ---------------------------------------------------------------- + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the source. + If the repo destination directory does not exist, checkout the correct + branch or tag. + """ + repo_dir_path = os.path.join(base_dir_path, repo_dir_name) + repo_dir_exists = os.path.exists(repo_dir_path) + if (repo_dir_exists and not os.listdir( + repo_dir_path)) or not repo_dir_exists: + self._clone_repo(base_dir_path, repo_dir_name, verbosity) + self._checkout_ref(repo_dir_path, verbosity, recursive) + gmpath = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + if os.path.exists(gmpath): + self._gitmodules = gmpath + self._submods = git_submodule_status(repo_dir_path) + else: + self._gitmodules = None + self._submods = None + + def status(self, stat, repo_dir_path): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the source. + If the repo destination directory does not exist, checkout the correct + branch or tag. + """ + self._check_sync(stat, repo_dir_path) + if os.path.exists(repo_dir_path): + self._status_summary(stat, repo_dir_path) + + def submodules_file(self, repo_path=None): + if repo_path is not None: + gmpath = os.path.join(repo_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + if os.path.exists(gmpath): + self._gitmodules = gmpath + self._submods = git_submodule_status(repo_path) + + return self._gitmodules + + # ---------------------------------------------------------------- + # + # Internal work functions + # + # ---------------------------------------------------------------- + def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): + """Prepare to execute the clone by managing directory location + """ + cwd = os.getcwd() + os.chdir(base_dir_path) + self._git_clone(self._url, repo_dir_name, verbosity) + os.chdir(cwd) + + def _current_ref(self): + """Determine the *name* associated with HEAD. + + If we're on a branch, then returns the branch name; otherwise, + if we're on a tag, then returns the tag name; otherwise, returns + the current hash. Returns an empty string if no reference can be + determined (e.g., if we're not actually in a git repository). + """ + ref_found = False + + # If we're on a branch, then use that as the current ref + branch_found, branch_name = self._git_current_branch() + if branch_found: + current_ref = branch_name + ref_found = True + + if not ref_found: + # Otherwise, if we're exactly at a tag, use that as the + # current ref + tag_found, tag_name = self._git_current_tag() + if tag_found: + current_ref = tag_name + ref_found = True + + if not ref_found: + # Otherwise, use current hash as the current ref + hash_found, hash_name = self._git_current_hash() + if hash_found: + current_ref = hash_name + ref_found = True + + if not ref_found: + # If we still can't find a ref, return empty string. This + # can happen if we're not actually in a git repo + current_ref = '' + + return current_ref + + def _check_sync(self, stat, repo_dir_path): + """Determine whether a git repository is in-sync with the model + description. + + Because repos can have multiple remotes, the only criteria is + whether the branch or tag is the same. + + """ + if not os.path.exists(repo_dir_path): + # NOTE(bja, 2017-10) condition should have been determined + # by _Source() object and should never be here! + stat.sync_state = ExternalStatus.STATUS_ERROR + else: + git_dir = os.path.join(repo_dir_path, '.git') + if not os.path.exists(git_dir): + # NOTE(bja, 2017-10) directory exists, but no git repo + # info.... Can't test with subprocess git command + # because git will move up directory tree until it + # finds the parent repo git dir! + stat.sync_state = ExternalStatus.UNKNOWN + else: + self._check_sync_logic(stat, repo_dir_path) + + def _check_sync_logic(self, stat, repo_dir_path): + """Compare the underlying hashes of the currently checkout ref and the + expected ref. + + Output: sets the sync_state as well as the current and + expected ref in the input status object. + + """ + def compare_refs(current_ref, expected_ref): + """Compare the current and expected ref. + + """ + if current_ref == expected_ref: + status = ExternalStatus.STATUS_OK + else: + status = ExternalStatus.MODEL_MODIFIED + return status + + cwd = os.getcwd() + os.chdir(repo_dir_path) + + # get the full hash of the current commit + _, current_ref = self._git_current_hash() + + if self._branch: + if self._url == LOCAL_PATH_INDICATOR: + expected_ref = self._branch + else: + remote_name = self._determine_remote_name() + if not remote_name: + # git doesn't know about this remote. by definition + # this is a modified state. + expected_ref = "unknown_remote/{0}".format(self._branch) + else: + expected_ref = "{0}/{1}".format(remote_name, self._branch) + elif self._hash: + expected_ref = self._hash + elif self._tag: + expected_ref = self._tag + else: + msg = 'In repo "{0}": none of branch, hash or tag are set'.format( + self._name) + fatal_error(msg) + + # record the *names* of the current and expected branches + stat.current_version = self._current_ref() + stat.expected_version = copy.deepcopy(expected_ref) + + if current_ref == EMPTY_STR: + stat.sync_state = ExternalStatus.UNKNOWN + else: + # get the underlying hash of the expected ref + revparse_status, expected_ref_hash = self._git_revparse_commit( + expected_ref) + if revparse_status: + # We failed to get the hash associated with + # expected_ref. Maybe we should assign this to some special + # status, but for now we're just calling this out-of-sync to + # remain consistent with how this worked before. + stat.sync_state = ExternalStatus.MODEL_MODIFIED + else: + # compare the underlying hashes + stat.sync_state = compare_refs(current_ref, expected_ref_hash) + + os.chdir(cwd) + + def _determine_remote_name(self): + """Return the remote name. + + Note that this is for the *future* repo url and branch, not + the current working copy! + + """ + git_output = self._git_remote_verbose() + git_output = git_output.splitlines() + remote_name = '' + for line in git_output: + data = line.strip() + if not data: + continue + data = data.split() + name = data[0].strip() + url = data[1].strip() + if self._url == url: + remote_name = name + break + return remote_name + + def _create_remote_name(self): + """The url specified in the externals description file was not known + to git. We need to add it, which means adding a unique and + safe name.... + + The assigned name needs to be safe for git to use, e.g. can't + look like a path 'foo/bar' and work with both remote and local paths. + + Remote paths include but are not limited to: git, ssh, https, + github, gitlab, bitbucket, custom server, etc. + + Local paths can be relative or absolute. They may contain + shell variables, e.g. ${REPO_ROOT}/repo_name, or username + expansion, i.e. ~/ or ~someuser/. + + Relative paths must be at least one layer of redirection, i.e. + container/../ext_repo, but may be many layers deep, e.g. + container/../../../../../ext_repo + + NOTE(bja, 2017-11) + + The base name below may not be unique, for example if the + user has local paths like: + + /path/to/my/repos/nice_repo + /path/to/other/repos/nice_repo + + But the current implementation should cover most common + use cases for remotes and still provide usable names. + + """ + url = copy.deepcopy(self._url) + if is_remote_url(url): + url = split_remote_url(url) + else: + url = expand_local_url(url, self._name) + url = url.split('/') + repo_name = url[-1] + base_name = url[-2] + # repo name should nominally already be something that git can + # deal with. We need to remove other possibly troublesome + # punctuation, e.g. /, $, from the base name. + unsafe_characters = '!@#$%^&*()[]{}\\/,;~' + for unsafe in unsafe_characters: + base_name = base_name.replace(unsafe, '') + remote_name = "{0}_{1}".format(base_name, repo_name) + return remote_name + + def _checkout_ref(self, repo_dir, verbosity, submodules): + """Checkout the user supplied reference + if is True, recursively initialize and update + the repo's submodules + """ + # import pdb; pdb.set_trace() + cwd = os.getcwd() + os.chdir(repo_dir) + if self._url.strip() == LOCAL_PATH_INDICATOR: + self._checkout_local_ref(verbosity, submodules) + else: + self._checkout_external_ref(verbosity, submodules) + + if self._sparse: + self._sparse_checkout(repo_dir, verbosity) + os.chdir(cwd) + + + def _checkout_local_ref(self, verbosity, submodules): + """Checkout the reference considering the local repo only. Do not + fetch any additional remotes or specify the remote when + checkout out the ref. + if is True, recursively initialize and update + the repo's submodules + """ + if self._tag: + ref = self._tag + elif self._branch: + ref = self._branch + else: + ref = self._hash + + self._check_for_valid_ref(ref) + self._git_checkout_ref(ref, verbosity, submodules) + + def _checkout_external_ref(self, verbosity, submodules): + """Checkout the reference from a remote repository + if is True, recursively initialize and update + the repo's submodules + """ + if self._tag: + ref = self._tag + elif self._branch: + ref = self._branch + else: + ref = self._hash + + remote_name = self._determine_remote_name() + if not remote_name: + remote_name = self._create_remote_name() + self._git_remote_add(remote_name, self._url) + self._git_fetch(remote_name) + + # NOTE(bja, 2018-03) we need to send separate ref and remote + # name to check_for_vaild_ref, but the combined name to + # checkout_ref! + self._check_for_valid_ref(ref, remote_name) + + if self._branch: + ref = '{0}/{1}'.format(remote_name, ref) + self._git_checkout_ref(ref, verbosity, submodules) + + def _sparse_checkout(self, repo_dir, verbosity): + """Use git read-tree to thin the working tree.""" + cwd = os.getcwd() + + cmd = ['cp', self._sparse, os.path.join(repo_dir, + '.git/info/sparse-checkout')] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + os.chdir(repo_dir) + self._git_sparse_checkout(verbosity) + + os.chdir(cwd) + + def _check_for_valid_ref(self, ref, remote_name=None): + """Try some basic sanity checks on the user supplied reference so we + can provide a more useful error message than calledprocess + error... + + """ + is_tag = self._ref_is_tag(ref) + is_branch = self._ref_is_branch(ref, remote_name) + is_hash = self._ref_is_hash(ref) + + is_valid = is_tag or is_branch or is_hash + if not is_valid: + msg = ('In repo "{0}": reference "{1}" does not appear to be a ' + 'valid tag, branch or hash! Please verify the reference ' + 'name (e.g. spelling), is available from: {2} '.format( + self._name, ref, self._url)) + fatal_error(msg) + + if is_tag: + is_unique_tag, msg = self._is_unique_tag(ref, remote_name) + if not is_unique_tag: + msg = ('In repo "{0}": tag "{1}" {2}'.format( + self._name, self._tag, msg)) + fatal_error(msg) + + return is_valid + + def _is_unique_tag(self, ref, remote_name): + """Verify that a reference is a valid tag and is unique (not a branch) + + Tags may be tag names, or SHA id's. It is also possible that a + branch and tag have the some name. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + + """ + is_tag = self._ref_is_tag(ref) + is_branch = self._ref_is_branch(ref, remote_name) + is_hash = self._ref_is_hash(ref) + + msg = '' + is_unique_tag = False + if is_tag and not is_branch: + # unique tag + msg = 'is ok' + is_unique_tag = True + elif is_tag and is_branch: + msg = ('is both a branch and a tag. git may checkout the branch ' + 'instead of the tag depending on your version of git.') + is_unique_tag = False + elif not is_tag and is_branch: + msg = ('is a branch, and not a tag. If you intended to checkout ' + 'a branch, please change the externals description to be ' + 'a branch. If you intended to checkout a tag, it does not ' + 'exist. Please check the name.') + is_unique_tag = False + else: # not is_tag and not is_branch: + if is_hash: + # probably a sha1 or HEAD, etc, we call it a tag + msg = 'is ok' + is_unique_tag = True + else: + # undetermined state. + msg = ('does not appear to be a valid tag, branch or hash! ' + 'Please check the name and repository.') + is_unique_tag = False + + return is_unique_tag, msg + + def _ref_is_tag(self, ref): + """Verify that a reference is a valid tag according to git. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + """ + is_tag = False + value = self._git_showref_tag(ref) + if value == 0: + is_tag = True + return is_tag + + def _ref_is_branch(self, ref, remote_name=None): + """Verify if a ref is any kind of branch (local, tracked remote, + untracked remote). + + """ + local_branch = False + remote_branch = False + if remote_name: + remote_branch = self._ref_is_remote_branch(ref, remote_name) + local_branch = self._ref_is_local_branch(ref) + + is_branch = False + if local_branch or remote_branch: + is_branch = True + return is_branch + + def _ref_is_local_branch(self, ref): + """Verify that a reference is a valid branch according to git. + + show-ref branch returns local branches that have been + previously checked out. It will not necessarily pick up + untracked remote branches. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + + """ + is_branch = False + value = self._git_showref_branch(ref) + if value == 0: + is_branch = True + return is_branch + + def _ref_is_remote_branch(self, ref, remote_name): + """Verify that a reference is a valid branch according to git. + + show-ref branch returns local branches that have been + previously checked out. It will not necessarily pick up + untracked remote branches. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + + """ + is_branch = False + value = self._git_lsremote_branch(ref, remote_name) + if value == 0: + is_branch = True + return is_branch + + def _ref_is_commit(self, ref): + """Verify that a reference is a valid commit according to git. + + This could be a tag, branch, sha1 id, HEAD and potentially others... + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + """ + is_commit = False + value, _ = self._git_revparse_commit(ref) + if value == 0: + is_commit = True + return is_commit + + def _ref_is_hash(self, ref): + """Verify that a reference is a valid hash according to git. + + Git doesn't seem to provide an exact way to determine if user + supplied reference is an actual hash. So we verify that the + ref is a valid commit and return the underlying commit + hash. Then check that the commit hash begins with the user + supplied string. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + + """ + is_hash = False + status, git_output = self._git_revparse_commit(ref) + if status == 0: + if git_output.strip().startswith(ref): + is_hash = True + return is_hash + + def _status_summary(self, stat, repo_dir_path): + """Determine the clean/dirty status of a git repository + + """ + cwd = os.getcwd() + os.chdir(repo_dir_path) + git_output = self._git_status_porcelain_v1z() + is_dirty = self._status_v1z_is_dirty(git_output) + if is_dirty: + stat.clean_state = ExternalStatus.DIRTY + else: + stat.clean_state = ExternalStatus.STATUS_OK + + # Now save the verbose status output incase the user wants to + # see it. + stat.status_output = self._git_status_verbose() + os.chdir(cwd) + + @staticmethod + def _status_v1z_is_dirty(git_output): + """Parse the git status output from --porcelain=v1 -z and determine if + the repo status is clean or dirty. Dirty means: + + * modified files + * missing files + * added files + * removed + * renamed + * unmerged + + Whether untracked files are considered depends on how the status + command was run (i.e., whether it was run with the '-u' option). + + NOTE: Based on the above definition, the porcelain status + should be an empty string to be considered 'clean'. Of course + this assumes we only get an empty string from an status + command on a clean checkout, and not some error + condition... Could alse use 'git diff --quiet'. + + """ + is_dirty = False + if git_output: + is_dirty = True + return is_dirty + + # ---------------------------------------------------------------- + # + # system call to git for information gathering + # + # ---------------------------------------------------------------- + @staticmethod + def _git_current_hash(): + """Return the full hash of the currently checked-out version. + + Returns a tuple, (hash_found, hash), where hash_found is a + logical specifying whether a hash was found for HEAD (False + could mean we're not in a git repository at all). (If hash_found + is False, then hash is ''.) + """ + status, git_output = GitRepository._git_revparse_commit("HEAD") + hash_found = not status + if not hash_found: + git_output = '' + return hash_found, git_output + + @staticmethod + def _git_current_branch(): + """Determines the name of the current branch. + + Returns a tuple, (branch_found, branch_name), where branch_found + is a logical specifying whether a branch name was found for + HEAD. (If branch_found is False, then branch_name is ''.) + """ + cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + branch_found = not status + if branch_found: + git_output = git_output.strip() + else: + git_output = '' + return branch_found, git_output + + @staticmethod + def _git_current_tag(): + """Determines the name tag corresponding to HEAD (if any). + + Returns a tuple, (tag_found, tag_name), where tag_found is a + logical specifying whether we found a tag name corresponding to + HEAD. (If tag_found is False, then tag_name is ''.) + """ + # git describe --exact-match --tags HEAD + cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + tag_found = not status + if tag_found: + git_output = git_output.strip() + else: + git_output = '' + return tag_found, git_output + + @staticmethod + def _git_showref_tag(ref): + """Run git show-ref check if the user supplied ref is a tag. + + could also use git rev-parse --quiet --verify tagname^{tag} + """ + cmd = ['git', 'show-ref', '--quiet', '--verify', + 'refs/tags/{0}'.format(ref), ] + status = execute_subprocess(cmd, status_to_caller=True) + return status + + @staticmethod + def _git_showref_branch(ref): + """Run git show-ref check if the user supplied ref is a local or + tracked remote branch. + + """ + cmd = ['git', 'show-ref', '--quiet', '--verify', + 'refs/heads/{0}'.format(ref), ] + status = execute_subprocess(cmd, status_to_caller=True) + return status + + @staticmethod + def _git_lsremote_branch(ref, remote_name): + """Run git ls-remote to check if the user supplied ref is a remote + branch that is not being tracked + + """ + cmd = ['git', 'ls-remote', '--exit-code', '--heads', + remote_name, ref, ] + status = execute_subprocess(cmd, status_to_caller=True) + return status + + @staticmethod + def _git_revparse_commit(ref): + """Run git rev-parse to detect if a reference is a SHA, HEAD or other + valid commit. + + """ + cmd = ['git', 'rev-parse', '--quiet', '--verify', + '{0}^{1}'.format(ref, '{commit}'), ] + status, git_output = execute_subprocess(cmd, status_to_caller=True, + output_to_caller=True) + git_output = git_output.strip() + return status, git_output + + @staticmethod + def _git_status_porcelain_v1z(): + """Run git status to obtain repository information. + + This is run with '--untracked=no' to ignore untracked files. + + The machine-portable format that is guaranteed not to change + between git versions or *user configuration*. + + """ + cmd = ['git', 'status', '--untracked-files=no', '--porcelain', '-z'] + git_output = execute_subprocess(cmd, output_to_caller=True) + return git_output + + @staticmethod + def _git_status_verbose(): + """Run the git status command to obtain repository information. + """ + cmd = ['git', 'status'] + git_output = execute_subprocess(cmd, output_to_caller=True) + return git_output + + @staticmethod + def _git_remote_verbose(): + """Run the git remote command to obtain repository information. + """ + cmd = ['git', 'remote', '--verbose'] + git_output = execute_subprocess(cmd, output_to_caller=True) + return git_output + + @staticmethod + def has_submodules(repo_dir_path=None): + """Return True iff the repository at (or the current + directory if is None) has a '.gitmodules' file + """ + if repo_dir_path is None: + fname = ExternalsDescription.GIT_SUBMODULES_FILENAME + else: + fname = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + + return os.path.exists(fname) + + # ---------------------------------------------------------------- + # + # system call to git for sideffects modifying the working tree + # + # ---------------------------------------------------------------- + @staticmethod + def _git_clone(url, repo_dir_name, verbosity): + """Run git clone for the side effect of creating a repository. + """ + cmd = ['git', 'clone', '--quiet'] + subcmd = None + + cmd.extend([url, repo_dir_name]) + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + if subcmd is not None: + os.chdir(repo_dir_name) + execute_subprocess(subcmd) + + @staticmethod + def _git_remote_add(name, url): + """Run the git remote command for the side effect of adding a remote + """ + cmd = ['git', 'remote', 'add', name, url] + execute_subprocess(cmd) + + @staticmethod + def _git_fetch(remote_name): + """Run the git fetch command for the side effect of updating the repo + """ + cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] + execute_subprocess(cmd) + + @staticmethod + def _git_checkout_ref(ref, verbosity, submodules): + """Run the git checkout command for the side effect of updating the repo + + Param: ref is a reference to a local or remote object in the + form 'origin/my_feature', or 'tag1'. + + """ + cmd = ['git', 'checkout', '--quiet', ref] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + if submodules: + GitRepository._git_update_submodules(verbosity) + + @staticmethod + def _git_sparse_checkout(verbosity): + """Configure repo via read-tree.""" + cmd = ['git', 'config', 'core.sparsecheckout', 'true'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + cmd = ['git', 'read-tree', '-mu', 'HEAD'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + + @staticmethod + def _git_update_submodules(verbosity): + """Run git submodule update for the side effect of updating this + repo's submodules. + """ + # First, verify that we have a .gitmodules file + if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): + cmd = ['git', 'submodule', 'update', '--init', '--recursive'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + + execute_subprocess(cmd) diff --git a/util/manage_externals/manic/repository_svn.py b/util/manage_externals/manic/repository_svn.py new file mode 100644 index 0000000000..2f0d4d848c --- /dev/null +++ b/util/manage_externals/manic/repository_svn.py @@ -0,0 +1,284 @@ +"""Class for interacting with svn repositories +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import os +import re +import xml.etree.ElementTree as ET + +from .global_constants import EMPTY_STR, VERBOSITY_VERBOSE +from .repository import Repository +from .externals_status import ExternalStatus +from .utils import fatal_error, indent_string, printlog +from .utils import execute_subprocess + + +class SvnRepository(Repository): + """ + Class to represent and operate on a repository description. + + For testing purpose, all system calls to svn should: + + * be isolated in separate functions with no application logic + * of the form: + - cmd = ['svn', ...] + - value = execute_subprocess(cmd, output_to_caller={T|F}, + status_to_caller={T|F}) + - return value + * be static methods (not rely on self) + * name as _svn_subcommand_args(user_args) + + This convention allows easy unit testing of the repository logic + by mocking the specific calls to return predefined results. + + """ + RE_URLLINE = re.compile(r'^URL:') + + def __init__(self, component_name, repo, ignore_ancestry=False): + """ + Parse repo (a XML element). + """ + Repository.__init__(self, component_name, repo) + self._ignore_ancestry = ignore_ancestry + if self._branch: + self._url = os.path.join(self._url, self._branch) + elif self._tag: + self._url = os.path.join(self._url, self._tag) + else: + msg = "DEV_ERROR in svn repository. Shouldn't be here!" + fatal_error(msg) + + # ---------------------------------------------------------------- + # + # Public API, defined by Repository + # + # ---------------------------------------------------------------- + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument + """Checkout or update the working copy + + If the repo destination directory exists, switch the sandbox to + match the externals description. + + If the repo destination directory does not exist, checkout the + correct branch or tag. + NB: is include as an argument for compatibility with + git functionality (repository_git.py) + + """ + repo_dir_path = os.path.join(base_dir_path, repo_dir_name) + if os.path.exists(repo_dir_path): + cwd = os.getcwd() + os.chdir(repo_dir_path) + self._svn_switch(self._url, self._ignore_ancestry, verbosity) + # svn switch can lead to a conflict state, but it gives a + # return code of 0. So now we need to make sure that we're + # in a clean (non-conflict) state. + self._abort_if_dirty(repo_dir_path, + "Expected clean state following switch") + os.chdir(cwd) + else: + self._svn_checkout(self._url, repo_dir_path, verbosity) + + def status(self, stat, repo_dir_path): + """ + Check and report the status of the repository + """ + self._check_sync(stat, repo_dir_path) + if os.path.exists(repo_dir_path): + self._status_summary(stat, repo_dir_path) + + # ---------------------------------------------------------------- + # + # Internal work functions + # + # ---------------------------------------------------------------- + def _check_sync(self, stat, repo_dir_path): + """Check to see if repository directory exists and is at the expected + url. Return: status object + + """ + if not os.path.exists(repo_dir_path): + # NOTE(bja, 2017-10) this state should have been handled by + # the source object and we never get here! + stat.sync_state = ExternalStatus.STATUS_ERROR + else: + svn_output = self._svn_info(repo_dir_path) + if not svn_output: + # directory exists, but info returned nothing. .svn + # directory removed or incomplete checkout? + stat.sync_state = ExternalStatus.UNKNOWN + else: + stat.sync_state, stat.current_version = \ + self._check_url(svn_output, self._url) + stat.expected_version = '/'.join(self._url.split('/')[3:]) + + def _abort_if_dirty(self, repo_dir_path, message): + """Check if the repo is in a dirty state; if so, abort with a + helpful message. + + """ + + stat = ExternalStatus() + self._status_summary(stat, repo_dir_path) + if stat.clean_state != ExternalStatus.STATUS_OK: + status = self._svn_status_verbose(repo_dir_path) + status = indent_string(status, 4) + errmsg = """In directory + {cwd} + +svn status now shows: +{status} + +ERROR: {message} + +One possible cause of this problem is that there may have been untracked +files in your working directory that had the same name as tracked files +in the new revision. + +To recover: Clean up the above directory (resolving conflicts, etc.), +then rerun checkout_externals. +""".format(cwd=repo_dir_path, message=message, status=status) + + fatal_error(errmsg) + + @staticmethod + def _check_url(svn_output, expected_url): + """Determine the svn url from svn info output and return whether it + matches the expected value. + + """ + url = None + for line in svn_output.splitlines(): + if SvnRepository.RE_URLLINE.match(line): + url = line.split(': ')[1].strip() + break + if not url: + status = ExternalStatus.UNKNOWN + elif url == expected_url: + status = ExternalStatus.STATUS_OK + else: + status = ExternalStatus.MODEL_MODIFIED + + if url: + current_version = '/'.join(url.split('/')[3:]) + else: + current_version = EMPTY_STR + + return status, current_version + + def _status_summary(self, stat, repo_dir_path): + """Report whether the svn repository is in-sync with the model + description and whether the sandbox is clean or dirty. + + """ + svn_output = self._svn_status_xml(repo_dir_path) + is_dirty = self.xml_status_is_dirty(svn_output) + if is_dirty: + stat.clean_state = ExternalStatus.DIRTY + else: + stat.clean_state = ExternalStatus.STATUS_OK + + # Now save the verbose status output incase the user wants to + # see it. + stat.status_output = self._svn_status_verbose(repo_dir_path) + + @staticmethod + def xml_status_is_dirty(svn_output): + """Parse svn status xml output and determine if the working copy is + clean or dirty. Dirty is defined as: + + * modified files + * added files + * deleted files + * missing files + + Unversioned files do not affect the clean/dirty status. + + 'external' is also an acceptable state + + """ + # pylint: disable=invalid-name + SVN_EXTERNAL = 'external' + SVN_UNVERSIONED = 'unversioned' + # pylint: enable=invalid-name + + is_dirty = False + try: + xml_status = ET.fromstring(svn_output) + except BaseException: + fatal_error( + "SVN returned invalid XML message {}".format(svn_output)) + xml_target = xml_status.find('./target') + entries = xml_target.findall('./entry') + for entry in entries: + status = entry.find('./wc-status') + item = status.get('item') + if item == SVN_EXTERNAL: + continue + if item == SVN_UNVERSIONED: + continue + else: + is_dirty = True + break + return is_dirty + + # ---------------------------------------------------------------- + # + # system call to svn for information gathering + # + # ---------------------------------------------------------------- + @staticmethod + def _svn_info(repo_dir_path): + """Return results of svn info command + """ + cmd = ['svn', 'info', repo_dir_path] + output = execute_subprocess(cmd, output_to_caller=True) + return output + + @staticmethod + def _svn_status_verbose(repo_dir_path): + """capture the full svn status output + """ + cmd = ['svn', 'status', repo_dir_path] + svn_output = execute_subprocess(cmd, output_to_caller=True) + return svn_output + + @staticmethod + def _svn_status_xml(repo_dir_path): + """ + Get status of the subversion sandbox in repo_dir + """ + cmd = ['svn', 'status', '--xml', repo_dir_path] + svn_output = execute_subprocess(cmd, output_to_caller=True) + return svn_output + + # ---------------------------------------------------------------- + # + # system call to svn for sideffects modifying the working tree + # + # ---------------------------------------------------------------- + @staticmethod + def _svn_checkout(url, repo_dir_path, verbosity): + """ + Checkout a subversion repository (repo_url) to checkout_dir. + """ + cmd = ['svn', 'checkout', '--quiet', url, repo_dir_path] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + + @staticmethod + def _svn_switch(url, ignore_ancestry, verbosity): + """ + Switch branches for in an svn sandbox + """ + cmd = ['svn', 'switch', '--quiet'] + if ignore_ancestry: + cmd.append('--ignore-ancestry') + cmd.append(url) + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) diff --git a/util/manage_externals/manic/sourcetree.py b/util/manage_externals/manic/sourcetree.py new file mode 100644 index 0000000000..3a63835c78 --- /dev/null +++ b/util/manage_externals/manic/sourcetree.py @@ -0,0 +1,351 @@ +""" + +FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! +""" + +import errno +import logging +import os + +from .externals_description import ExternalsDescription +from .externals_description import read_externals_description_file +from .externals_description import create_externals_description +from .repository_factory import create_repository +from .repository_git import GitRepository +from .externals_status import ExternalStatus +from .utils import fatal_error, printlog +from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR +from .global_constants import VERBOSITY_VERBOSE + +class _External(object): + """ + _External represents an external object inside a SourceTree + """ + + # pylint: disable=R0902 + + def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): + """Parse an external description file into a dictionary of externals. + + Input: + + root_dir : string - the root directory path where + 'local_path' is relative to. + + name : string - name of the ext_description object. may or may not + correspond to something in the path. + + ext_description : dict - source ExternalsDescription object + + svn_ignore_ancestry : bool - use --ignore-externals with svn switch + + """ + self._name = name + self._repo = None + self._externals = EMPTY_STR + self._externals_sourcetree = None + self._stat = ExternalStatus() + self._sparse = None + # Parse the sub-elements + + # _path : local path relative to the containing source tree + self._local_path = ext_description[ExternalsDescription.PATH] + # _repo_dir : full repository directory + repo_dir = os.path.join(root_dir, self._local_path) + self._repo_dir_path = os.path.abspath(repo_dir) + # _base_dir : base directory *containing* the repository + self._base_dir_path = os.path.dirname(self._repo_dir_path) + # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + self._repo_dir_name = os.path.basename(self._repo_dir_path) + assert(os.path.join(self._base_dir_path, self._repo_dir_name) + == self._repo_dir_path) + + self._required = ext_description[ExternalsDescription.REQUIRED] + self._externals = ext_description[ExternalsDescription.EXTERNALS] + # Treat a .gitmodules file as a backup externals config + if not self._externals: + if GitRepository.has_submodules(self._repo_dir_path): + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + + repo = create_repository( + name, ext_description[ExternalsDescription.REPO], + svn_ignore_ancestry=svn_ignore_ancestry) + if repo: + self._repo = repo + + if self._externals and (self._externals.lower() != 'none'): + self._create_externals_sourcetree() + + def get_name(self): + """ + Return the external object's name + """ + return self._name + + def get_local_path(self): + """ + Return the external object's path + """ + return self._local_path + + def status(self): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the external. + If the repo destination directory does not exist, checkout the correce + branch or tag. + If load_all is True, also load all of the the externals sub-externals. + """ + + self._stat.path = self.get_local_path() + if not self._required: + self._stat.source_type = ExternalStatus.OPTIONAL + elif self._local_path == LOCAL_PATH_INDICATOR: + # LOCAL_PATH_INDICATOR, '.' paths, are standalone + # component directories that are not managed by + # checkout_externals. + self._stat.source_type = ExternalStatus.STANDALONE + else: + # managed by checkout_externals + self._stat.source_type = ExternalStatus.MANAGED + + ext_stats = {} + + if not os.path.exists(self._repo_dir_path): + self._stat.sync_state = ExternalStatus.EMPTY + msg = ('status check: repository directory for "{0}" does not ' + 'exist.'.format(self._name)) + logging.info(msg) + self._stat.current_version = 'not checked out' + # NOTE(bja, 2018-01) directory doesn't exist, so we cannot + # use repo to determine the expected version. We just take + # a best-guess based on the assumption that only tag or + # branch should be set, but not both. + if not self._repo: + self._stat.expected_version = 'unknown' + else: + self._stat.expected_version = self._repo.tag() + self._repo.branch() + else: + if self._repo: + self._repo.status(self._stat, self._repo_dir_path) + + if self._externals and self._externals_sourcetree: + # we expect externals and they exist + cwd = os.getcwd() + # SourceTree expects to be called from the correct + # root directory. + os.chdir(self._repo_dir_path) + ext_stats = self._externals_sourcetree.status(self._local_path) + os.chdir(cwd) + + all_stats = {} + # don't add the root component because we don't manage it + # and can't provide useful info about it. + if self._local_path != LOCAL_PATH_INDICATOR: + # store the stats under tha local_path, not comp name so + # it will be sorted correctly + all_stats[self._stat.path] = self._stat + + if ext_stats: + all_stats.update(ext_stats) + + return all_stats + + def checkout(self, verbosity, load_all): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the external. + If the repo destination directory does not exist, checkout the correct + branch or tag. + If load_all is True, also load all of the the externals sub-externals. + """ + if load_all: + pass + # Make sure we are in correct location + + if not os.path.exists(self._repo_dir_path): + # repository directory doesn't exist. Need to check it + # out, and for that we need the base_dir_path to exist + try: + os.makedirs(self._base_dir_path) + except OSError as error: + if error.errno != errno.EEXIST: + msg = 'Could not create directory "{0}"'.format( + self._base_dir_path) + fatal_error(msg) + + if self._stat.source_type != ExternalStatus.STANDALONE: + if verbosity >= VERBOSITY_VERBOSE: + # NOTE(bja, 2018-01) probably do not want to pass + # verbosity in this case, because if (verbosity == + # VERBOSITY_DUMP), then the previous status output would + # also be dumped, adding noise to the output. + self._stat.log_status_message(VERBOSITY_VERBOSE) + + if self._repo: + if self._stat.sync_state == ExternalStatus.STATUS_OK: + # If we're already in sync, avoid showing verbose output + # from the checkout command, unless the verbosity level + # is 2 or more. + checkout_verbosity = verbosity - 1 + else: + checkout_verbosity = verbosity + + self._repo.checkout(self._base_dir_path, self._repo_dir_name, + checkout_verbosity, self.clone_recursive()) + + def checkout_externals(self, verbosity, load_all): + """Checkout the sub-externals for this object + """ + if self.load_externals(): + if self._externals_sourcetree: + # NOTE(bja, 2018-02): the subtree externals objects + # were created during initial status check. Updating + # the external may have changed which sub-externals + # are needed. We need to delete those objects and + # re-read the potentially modified externals + # description file. + self._externals_sourcetree = None + self._create_externals_sourcetree() + self._externals_sourcetree.checkout(verbosity, load_all) + + def load_externals(self): + 'Return True iff an externals file should be loaded' + load_ex = False + if os.path.exists(self._repo_dir_path): + if self._externals: + if self._externals.lower() != 'none': + load_ex = os.path.exists(os.path.join(self._repo_dir_path, + self._externals)) + + return load_ex + + def clone_recursive(self): + 'Return True iff any .gitmodules files should be processed' + # Try recursive unless there is an externals entry + recursive = not self._externals + + return recursive + + def _create_externals_sourcetree(self): + """ + """ + if not os.path.exists(self._repo_dir_path): + # NOTE(bja, 2017-10) repository has not been checked out + # yet, can't process the externals file. Assume we are + # checking status before code is checkoud out and this + # will be handled correctly later. + return + + cwd = os.getcwd() + os.chdir(self._repo_dir_path) + if self._externals.lower() == 'none': + msg = ('Internal: Attempt to create source tree for ' + 'externals = none in {}'.format(self._repo_dir_path)) + fatal_error(msg) + + if not os.path.exists(self._externals): + if GitRepository.has_submodules(): + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + + if not os.path.exists(self._externals): + # NOTE(bja, 2017-10) this check is redundent with the one + # in read_externals_description_file! + msg = ('External externals description file "{0}" ' + 'does not exist! In directory: {1}'.format( + self._externals, self._repo_dir_path)) + fatal_error(msg) + + externals_root = self._repo_dir_path + model_data = read_externals_description_file(externals_root, + self._externals) + externals = create_externals_description(model_data, + parent_repo=self._repo) + self._externals_sourcetree = SourceTree(externals_root, externals) + os.chdir(cwd) + +class SourceTree(object): + """ + SourceTree represents a group of managed externals + """ + + def __init__(self, root_dir, model, svn_ignore_ancestry=False): + """ + Build a SourceTree object from a model description + """ + self._root_dir = os.path.abspath(root_dir) + self._all_components = {} + self._required_compnames = [] + for comp in model: + src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) + self._all_components[comp] = src + if model[comp][ExternalsDescription.REQUIRED]: + self._required_compnames.append(comp) + + def status(self, relative_path_base=LOCAL_PATH_INDICATOR): + """Report the status components + + FIXME(bja, 2017-10) what do we do about situations where the + user checked out the optional components, but didn't add + optional for running status? What do we do where the user + didn't add optional to the checkout but did add it to the + status. -- For now, we run status on all components, and try + to do the right thing based on the results.... + + """ + load_comps = self._all_components.keys() + + summary = {} + for comp in load_comps: + printlog('{0}, '.format(comp), end='') + stat = self._all_components[comp].status() + for name in stat.keys(): + # check if we need to append the relative_path_base to + # the path so it will be sorted in the correct order. + if not stat[name].path.startswith(relative_path_base): + stat[name].path = os.path.join(relative_path_base, + stat[name].path) + # store under key = updated path, and delete the + # old key. + comp_stat = stat[name] + del stat[name] + stat[comp_stat.path] = comp_stat + summary.update(stat) + + return summary + + def checkout(self, verbosity, load_all, load_comp=None): + """ + Checkout or update indicated components into the the configured + subdirs. + + If load_all is True, recursively checkout all externals. + If load_all is False, load_comp is an optional set of components to load. + If load_all is True and load_comp is None, only load the required externals. + """ + if verbosity >= VERBOSITY_VERBOSE: + printlog('Checking out externals: ') + else: + printlog('Checking out externals: ', end='') + + if load_all: + load_comps = self._all_components.keys() + elif load_comp is not None: + load_comps = [load_comp] + else: + load_comps = self._required_compnames + + # checkout the primary externals + for comp in load_comps: + if verbosity < VERBOSITY_VERBOSE: + printlog('{0}, '.format(comp), end='') + else: + # verbose output handled by the _External object, just + # output a newline + printlog(EMPTY_STR) + self._all_components[comp].checkout(verbosity, load_all) + printlog('') + + # now give each external an opportunitity to checkout it's externals. + for comp in load_comps: + self._all_components[comp].checkout_externals(verbosity, load_all) diff --git a/util/manage_externals/manic/utils.py b/util/manage_externals/manic/utils.py new file mode 100644 index 0000000000..f57f43930c --- /dev/null +++ b/util/manage_externals/manic/utils.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python +""" +Common public utilities for manic package + +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import logging +import os +import subprocess +import sys +from threading import Timer + +from .global_constants import LOCAL_PATH_INDICATOR + +# --------------------------------------------------------------------- +# +# screen and logging output and functions to massage text for output +# +# --------------------------------------------------------------------- + + +def log_process_output(output): + """Log each line of process output at debug level so it can be + filtered if necessary. By default, output is a single string, and + logging.debug(output) will only put log info heading on the first + line. This makes it hard to filter with grep. + + """ + output = output.split('\n') + for line in output: + logging.debug(line) + + +def printlog(msg, **kwargs): + """Wrapper script around print to ensure that everything printed to + the screen also gets logged. + + """ + logging.info(msg) + if kwargs: + print(msg, **kwargs) + else: + print(msg) + sys.stdout.flush() + + +def last_n_lines(the_string, n_lines, truncation_message=None): + """Returns the last n lines of the given string + + Args: + the_string: str + n_lines: int + truncation_message: str, optional + + Returns a string containing the last n lines of the_string + + If truncation_message is provided, the returned string begins with + the given message if and only if the string is greater than n lines + to begin with. + """ + + lines = the_string.splitlines(True) + if len(lines) <= n_lines: + return_val = the_string + else: + lines_subset = lines[-n_lines:] + str_truncated = ''.join(lines_subset) + if truncation_message: + str_truncated = truncation_message + '\n' + str_truncated + return_val = str_truncated + + return return_val + + +def indent_string(the_string, indent_level): + """Indents the given string by a given number of spaces + + Args: + the_string: str + indent_level: int + + Returns a new string that is the same as the_string, except that + each line is indented by 'indent_level' spaces. + + In python3, this can be done with textwrap.indent. + """ + + lines = the_string.splitlines(True) + padding = ' ' * indent_level + lines_indented = [padding + line for line in lines] + return ''.join(lines_indented) + +# --------------------------------------------------------------------- +# +# error handling +# +# --------------------------------------------------------------------- + + +def fatal_error(message): + """ + Error output function + """ + logging.error(message) + raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) + + +# --------------------------------------------------------------------- +# +# Data conversion / manipulation +# +# --------------------------------------------------------------------- +def str_to_bool(bool_str): + """Convert a sting representation of as boolean into a true boolean. + + Conversion should be case insensitive. + """ + value = None + str_lower = bool_str.lower() + if str_lower in ('true', 't'): + value = True + elif str_lower in ('false', 'f'): + value = False + if value is None: + msg = ('ERROR: invalid boolean string value "{0}". ' + 'Must be "true" or "false"'.format(bool_str)) + fatal_error(msg) + return value + + +REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@'] + + +def is_remote_url(url): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + """ + remote_url = False + for prefix in REMOTE_PREFIXES: + if url.startswith(prefix): + remote_url = True + return remote_url + + +def split_remote_url(url): + """check if the user provided a local file path or a + remote. If remote, try to strip off protocol info. + + """ + remote_url = is_remote_url(url) + if not remote_url: + return url + + for prefix in REMOTE_PREFIXES: + url = url.replace(prefix, '') + + if '@' in url: + url = url.split('@')[1] + + if ':' in url: + url = url.split(':')[1] + + return url + + +def expand_local_url(url, field): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + Note: local paths of LOCAL_PATH_INDICATOR have special meaning and + represent local copy only, don't work with the remotes. + + """ + remote_url = is_remote_url(url) + if not remote_url: + if url.strip() == LOCAL_PATH_INDICATOR: + pass + else: + url = os.path.expandvars(url) + url = os.path.expanduser(url) + if not os.path.isabs(url): + msg = ('WARNING: Externals description for "{0}" contains a ' + 'url that is not remote and does not expand to an ' + 'absolute path. Version control operations may ' + 'fail.\n\nurl={1}'.format(field, url)) + printlog(msg) + else: + url = os.path.normpath(url) + return url + + +# --------------------------------------------------------------------- +# +# subprocess +# +# --------------------------------------------------------------------- + +# Give the user a helpful message if we detect that a command seems to +# be hanging. +_HANGING_SEC = 300 + + +def _hanging_msg(working_directory, command): + print(""" + +Command '{command}' +from directory {working_directory} +has taken {hanging_sec} seconds. It may be hanging. + +The command will continue to run, but you may want to abort +manage_externals with ^C and investigate. A possible cause of hangs is +when svn or git require authentication to access a private +repository. On some systems, svn and git requests for authentication +information will not be displayed to the user. In this case, the program +will appear to hang. Ensure you can run svn and git manually and access +all repositories without entering your authentication information. + +""".format(command=command, + working_directory=working_directory, + hanging_sec=_HANGING_SEC)) + + +def execute_subprocess(commands, status_to_caller=False, + output_to_caller=False): + """Wrapper around subprocess.check_output to handle common + exceptions. + + check_output runs a command with arguments and waits + for it to complete. + + check_output raises an exception on a nonzero return code. if + status_to_caller is true, execute_subprocess returns the subprocess + return code, otherwise execute_subprocess treats non-zero return + status as an error and raises an exception. + + """ + cwd = os.getcwd() + msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd) + logging.info(msg) + commands_str = ' '.join(commands) + logging.info(commands_str) + return_to_caller = status_to_caller or output_to_caller + status = -1 + output = '' + hanging_timer = Timer(_HANGING_SEC, _hanging_msg, + kwargs={"working_directory": cwd, + "command": commands_str}) + hanging_timer.start() + try: + output = subprocess.check_output(commands, stderr=subprocess.STDOUT, + universal_newlines=True) + log_process_output(output) + status = 0 + except OSError as error: + msg = failed_command_msg( + 'Command execution failed. Does the executable exist?', + commands) + logging.error(error) + fatal_error(msg) + except ValueError as error: + msg = failed_command_msg( + 'DEV_ERROR: Invalid arguments trying to run subprocess', + commands) + logging.error(error) + fatal_error(msg) + except subprocess.CalledProcessError as error: + # Only report the error if we are NOT returning to the + # caller. If we are returning to the caller, then it may be a + # simple status check. If returning, it is the callers + # responsibility determine if an error occurred and handle it + # appropriately. + if not return_to_caller: + msg_context = ('Process did not run successfully; ' + 'returned status {0}'.format(error.returncode)) + msg = failed_command_msg(msg_context, commands, + output=error.output) + logging.error(error) + logging.error(msg) + log_process_output(error.output) + fatal_error(msg) + status = error.returncode + finally: + hanging_timer.cancel() + + if status_to_caller and output_to_caller: + ret_value = (status, output) + elif status_to_caller: + ret_value = status + elif output_to_caller: + ret_value = output + else: + ret_value = None + + return ret_value + + +def failed_command_msg(msg_context, command, output=None): + """Template for consistent error messages from subprocess calls. + + If 'output' is given, it should provide the output from the failed + command + """ + + if output: + output_truncated = last_n_lines(output, 20, + truncation_message='[... Output truncated for brevity ...]') + errmsg = ('Failed with output:\n' + + indent_string(output_truncated, 4) + + '\nERROR: ') + else: + errmsg = '' + + command_str = ' '.join(command) + errmsg += """In directory + {cwd} +{context}: + {command} +""".format(cwd=os.getcwd(), context=msg_context, command=command_str) + + if output: + errmsg += 'See above for output from failed command.\n' + + return errmsg