From bfc7bb0b237d4cf4240551aa8b9c5d025724ac16 Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Mon, 27 Jan 2020 19:06:17 +0000 Subject: [PATCH 01/11] Issue #3 - initial add of manage_externals and needed Externals.cfg. Also added .gitignore file and removed scripts/files associated with no-longer-used prod_util and grid_util builds. --- .gitignore | 16 + Externals.cfg | 46 + modulefiles/module_nemsutil.hera | 10 - modulefiles/module_nemsutil.theia | 15 - modulefiles/module_nemsutil.wcoss | 13 - modulefiles/module_nemsutil.wcoss_cray | 17 - .../module_nemsutil.wcoss_cray_userlib | 19 - modulefiles/module_nemsutil.wcoss_dell_p3 | 12 - modulefiles/modulefile.grib_util.theia | 20 - modulefiles/modulefile.grib_util.wcoss | 32 - modulefiles/modulefile.grib_util.wcoss_cray | 22 - .../modulefile.grib_util.wcoss_cray_userlib | 22 - .../modulefile.grib_util.wcoss_dell_p3 | 15 - modulefiles/modulefile.prod_util.theia | 8 - modulefiles/modulefile.prod_util.wcoss_cray | 11 - .../modulefile.prod_util.wcoss_cray_userlib | 13 - .../modulefile.prod_util.wcoss_dell_p3 | 6 - modulefiles/modulefile.wgrib2.theia | 19 - modulefiles/modulefile.wgrib2.wcoss | 24 - modulefiles/modulefile.wgrib2.wcoss_cray | 13 - .../modulefile.wgrib2.wcoss_cray_userlib | 15 - modulefiles/modulefile.wgrib2.wcoss_dell_p3 | 11 - sorc/build_all.sh | 61 +- sorc/build_grib_util.sh | 88 -- sorc/build_prod_util.sh | 47 - sorc/fv3gfs_build.cfg | 2 - sorc/partial_build.sh | 4 +- util/manage_externals/checkout_externals | 36 + util/manage_externals/manic/__init__.py | 9 + util/manage_externals/manic/checkout.py | 424 +++++++++ .../manic/externals_description.py | 794 +++++++++++++++++ .../manic/externals_status.py | 164 ++++ .../manic/global_constants.py | 18 + util/manage_externals/manic/repository.py | 98 +++ .../manic/repository_factory.py | 29 + util/manage_externals/manic/repository_git.py | 819 ++++++++++++++++++ util/manage_externals/manic/repository_svn.py | 284 ++++++ util/manage_externals/manic/sourcetree.py | 351 ++++++++ util/manage_externals/manic/utils.py | 330 +++++++ 39 files changed, 3450 insertions(+), 487 deletions(-) create mode 100644 .gitignore create mode 100644 Externals.cfg delete mode 100644 modulefiles/module_nemsutil.hera delete mode 100644 modulefiles/module_nemsutil.theia delete mode 100644 modulefiles/module_nemsutil.wcoss delete mode 100644 modulefiles/module_nemsutil.wcoss_cray delete mode 100644 modulefiles/module_nemsutil.wcoss_cray_userlib delete mode 100644 modulefiles/module_nemsutil.wcoss_dell_p3 delete mode 100644 modulefiles/modulefile.grib_util.theia delete mode 100644 modulefiles/modulefile.grib_util.wcoss delete mode 100644 modulefiles/modulefile.grib_util.wcoss_cray delete mode 100644 modulefiles/modulefile.grib_util.wcoss_cray_userlib delete mode 100644 modulefiles/modulefile.grib_util.wcoss_dell_p3 delete mode 100644 modulefiles/modulefile.prod_util.theia delete mode 100644 modulefiles/modulefile.prod_util.wcoss_cray delete mode 100644 modulefiles/modulefile.prod_util.wcoss_cray_userlib delete mode 100644 modulefiles/modulefile.prod_util.wcoss_dell_p3 delete mode 100644 modulefiles/modulefile.wgrib2.theia delete mode 100644 modulefiles/modulefile.wgrib2.wcoss delete mode 100644 modulefiles/modulefile.wgrib2.wcoss_cray delete mode 100644 modulefiles/modulefile.wgrib2.wcoss_cray_userlib delete mode 100644 modulefiles/modulefile.wgrib2.wcoss_dell_p3 delete mode 100755 sorc/build_grib_util.sh delete mode 100755 sorc/build_prod_util.sh create mode 100755 util/manage_externals/checkout_externals create mode 100644 util/manage_externals/manic/__init__.py create mode 100755 util/manage_externals/manic/checkout.py create mode 100644 util/manage_externals/manic/externals_description.py create mode 100644 util/manage_externals/manic/externals_status.py create mode 100644 util/manage_externals/manic/global_constants.py create mode 100644 util/manage_externals/manic/repository.py create mode 100644 util/manage_externals/manic/repository_factory.py create mode 100644 util/manage_externals/manic/repository_git.py create mode 100644 util/manage_externals/manic/repository_svn.py create mode 100644 util/manage_externals/manic/sourcetree.py create mode 100644 util/manage_externals/manic/utils.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..d9a042c6ca --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# Ignore all compiled files +*.pyc +*.o +*.mod + +# Ignore exec folder +exec/ + +# Ignore sorc folders from externals +sorc/logs/ +sorc/fv3gfs.fd/ +sorc/gfs_post.fd/ +sorc/gsi.fd/ +sorc/ufs_utils.fd/ +sorc/gfs_wafs.fd/ +sorc/verif-global.fd/ diff --git a/Externals.cfg b/Externals.cfg new file mode 100644 index 0000000000..9fb968a378 --- /dev/null +++ b/Externals.cfg @@ -0,0 +1,46 @@ +# External sub-modules of global-workflow + +[NEMSfv3gfs] +tag = gfs.v16_PhysicsUpdate +local_path = sorc/fv3gfs.fd +repo_url = ssh://vlab.ncep.noaa.gov:29418/NEMSfv3gfs +protocol = git +required = True + +[GSI] +hash = cb8f69d82f38dcf85669b45aaf95dad068f0103c +local_path = sorc/gsi.fd +repo_url = ssh://vlab.ncep.noaa.gov:29418/ProdGSI +protocol = git +required = True + +[EMC_post] +hash = ba7e59b290c8149ff1c2fee98d01e99e4ef92ee6 +local_path = sorc/gfs_post.fd +repo_url = https://github.com/NOAA-EMC/EMC_post.git +protocol = git +required = True + +[UFS_UTILS] +tag = v1.1.0 +local_path = sorc/ufs_utils.fd +repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git +protocol = git +required = True + +[EMC_verif-global] +tag = verif_global_v1.2.2 +local_path = sorc/verif-global.fd +repo_url = ssh://vlab.ncep.noaa.gov:29418/EMC_verif-global +protocol = git +required = True + +[EMC_gfs_wafs] +tag = gfs_wafs.v5.0.11 +local_path = sorc/gfs_wafs.fd +repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git +protocol = git +required = False + +[externals_description] +schema_version = 1.0.0 diff --git a/modulefiles/module_nemsutil.hera b/modulefiles/module_nemsutil.hera deleted file mode 100644 index f1908fdf6e..0000000000 --- a/modulefiles/module_nemsutil.hera +++ /dev/null @@ -1,10 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles -module load w3nco/2.0.6 -module load bacio/2.0.3 -module load nemsio/2.2.3 - -export FCMP=ifort diff --git a/modulefiles/module_nemsutil.theia b/modulefiles/module_nemsutil.theia deleted file mode 100644 index 45685118eb..0000000000 --- a/modulefiles/module_nemsutil.theia +++ /dev/null @@ -1,15 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -# Loading Intel Compiler Suite -module load intel/14.0.2 -module load impi/5.1.2.150 - -# Loding nceplibs modules -module use -a $MOD_PATH -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load nemsio/v2.2.1 - -export FCMP=ifort diff --git a/modulefiles/module_nemsutil.wcoss b/modulefiles/module_nemsutil.wcoss deleted file mode 100644 index f421c1a88b..0000000000 --- a/modulefiles/module_nemsutil.wcoss +++ /dev/null @@ -1,13 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -# Loading Intel Compiler Suite -module load ics/14.0.1 - -# Loding nceplibs modules -module load w3nco/v2.0.6 -module load bacio/v2.0.1 -module load nemsio/v2.2.1 - -export FCMP=ifort diff --git a/modulefiles/module_nemsutil.wcoss_cray b/modulefiles/module_nemsutil.wcoss_cray deleted file mode 100644 index 371c8e0245..0000000000 --- a/modulefiles/module_nemsutil.wcoss_cray +++ /dev/null @@ -1,17 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -module purge -module load modules -module load PrgEnv-intel -module load cray-mpich -module load craype-sandybridge - -module load w3nco-intel/2.0.6 -module load bacio-intel/2.0.1 - -export NEMSIO_INC=/usrx/local/nceplibs/nemsio/nemsio_v2.2.3/incmod -export NEMSIO_LIB=/usrx/local/nceplibs/nemsio/nemsio_v2.2.3/libnemsio_v2.2.3.a - -export FCMP=ftn diff --git a/modulefiles/module_nemsutil.wcoss_cray_userlib b/modulefiles/module_nemsutil.wcoss_cray_userlib deleted file mode 100644 index 53fad475a5..0000000000 --- a/modulefiles/module_nemsutil.wcoss_cray_userlib +++ /dev/null @@ -1,19 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -# Load Intel environment -module purge -module load modules -module load PrgEnv-intel -module load cray-mpich -module load craype-sandybridge - -# Load NCEPLIBS modules -module unuse /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module use $MOD_PATH -module load w3nco/v2.0.6 -module load bacio/v2.0.2 -module load nemsio/v2.2.3 - -export FCMP=ftn diff --git a/modulefiles/module_nemsutil.wcoss_dell_p3 b/modulefiles/module_nemsutil.wcoss_dell_p3 deleted file mode 100644 index e93d581651..0000000000 --- a/modulefiles/module_nemsutil.wcoss_dell_p3 +++ /dev/null @@ -1,12 +0,0 @@ -#%Module##################################################### -## Module file for nemsutil -############################################################# - -module load ips/18.0.1.163 -module load impi/18.0.1 - -module load bacio/2.0.2 -module load w3nco/2.0.6 -module load nemsio/2.2.3 - -export FCMP=ifort diff --git a/modulefiles/modulefile.grib_util.theia b/modulefiles/modulefile.grib_util.theia deleted file mode 100644 index ecdd678049..0000000000 --- a/modulefiles/modulefile.grib_util.theia +++ /dev/null @@ -1,20 +0,0 @@ -#%Module###################################################################### -module use -a $MOD_PATH - -module load intel/14.0.2 - -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 - - -module load bacio/v2.0.2 -module load w3emc/v2.2.0 -module load w3nco/v2.0.6 -module load ip/v3.0.0 -module load sp/v2.0.2 -module load g2/v3.1.0 - -export FCMP=ifort -export CCMP=icc - diff --git a/modulefiles/modulefile.grib_util.wcoss b/modulefiles/modulefile.grib_util.wcoss deleted file mode 100644 index 0ae2e8f49e..0000000000 --- a/modulefiles/modulefile.grib_util.wcoss +++ /dev/null @@ -1,32 +0,0 @@ -#%Module###################################################################### -proc ModulesHelp { } { - puts stderr "Load modules for building GRIB utilities" -} -module-whatis "This module loads the modules and libraries for building\ - the GRIB utilities, including jasper, png, zlib, bacio, g2,\ - w3emc, w3nco, ip, sp, and iobuf." - -conflict build_grib_util - -# -# Loading required system modules -# - module load ics - module switch ics/15.0.6 - module load jasper/v1.900.1 - module load png/v1.2.44 - module load z/v1.2.6 - -# Loading Intel-Compiled NCEP Libraries - module load bacio/v2.0.1 - module load w3emc/v2.2.0 - module load w3nco/v2.0.6 - module load ip/v3.0.0 - module load sp/v2.0.2 - - # pre-implemented g2 v3.1.0 - module use /nwtest2/lib/modulefiles - module load g2/v3.1.0 - -setenv FCMP ifort -setenv CCMP icc diff --git a/modulefiles/modulefile.grib_util.wcoss_cray b/modulefiles/modulefile.grib_util.wcoss_cray deleted file mode 100644 index 191baa15df..0000000000 --- a/modulefiles/modulefile.grib_util.wcoss_cray +++ /dev/null @@ -1,22 +0,0 @@ -#%Module###################################################################### -module unload craype-haswell -module load craype-sandybridge -module unload PrgEnv-cray -module load PrgEnv-intel/5.2.56 -module switch intel/15.0.6.233 -module load iobuf/2.0.7 - -module load bacio-intel/2.0.1 -module load w3emc-intel/2.2.0 -module load w3nco-intel/2.0.6 -module load ip-intel/3.0.0 -module load sp-intel/2.0.2 -module load jasper-gnu-sandybridge/1.900.1 -module load png-intel-sandybridge/1.2.49 -module load zlib-intel-sandybridge/1.2.7 - -module use /gpfs/hps/nco/ops/nwtest/lib/modulefiles -module load g2-intel/3.1.0 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.grib_util.wcoss_cray_userlib b/modulefiles/modulefile.grib_util.wcoss_cray_userlib deleted file mode 100644 index 56ebe0c336..0000000000 --- a/modulefiles/modulefile.grib_util.wcoss_cray_userlib +++ /dev/null @@ -1,22 +0,0 @@ -#%Module###################################################################### -module unload craype-haswell -module load craype-sandybridge -module unload PrgEnv-cray -module load PrgEnv-intel/5.2.56 -module switch intel/15.0.6.233 -module load iobuf/2.0.7 - -module unuse /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module use -a $MOD_PATH -module load bacio/v2.0.2 -module load w3emc/v2.2.0 -module load w3nco/v2.0.6 -module load ip/v3.0.0 -module load sp/v2.0.2 -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 -module load g2/v3.1.0 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.grib_util.wcoss_dell_p3 b/modulefiles/modulefile.grib_util.wcoss_dell_p3 deleted file mode 100644 index bbebec9cbf..0000000000 --- a/modulefiles/modulefile.grib_util.wcoss_dell_p3 +++ /dev/null @@ -1,15 +0,0 @@ -#%Module###################################################################### - -module load bacio/2.0.2 -module load w3emc/2.3.0 -module load w3nco/2.0.6 -module load ip/3.0.1 -module load sp/2.0.2 - -module load jasper/1.900.1 -module load libpng/1.2.59 -module load zlib/1.2.11 -module load g2/3.1.0 - -export FCMP=ifort -export CCMP=icc diff --git a/modulefiles/modulefile.prod_util.theia b/modulefiles/modulefile.prod_util.theia deleted file mode 100644 index 3ad4d37bf6..0000000000 --- a/modulefiles/modulefile.prod_util.theia +++ /dev/null @@ -1,8 +0,0 @@ -#%Module###################################################################### -## -module load intel/16.1.150 -module use -a $MOD_PATH -module load w3nco/v2.0.6 - -export FCMP=ifort -export CCMP=icc diff --git a/modulefiles/modulefile.prod_util.wcoss_cray b/modulefiles/modulefile.prod_util.wcoss_cray deleted file mode 100644 index 51031c6bdd..0000000000 --- a/modulefiles/modulefile.prod_util.wcoss_cray +++ /dev/null @@ -1,11 +0,0 @@ -#%Module##################################################### -module purge -module load modules -module load PrgEnv-intel -module load cray-mpich -module load craype-sandybridge - -module load w3nco-intel/2.0.6 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.prod_util.wcoss_cray_userlib b/modulefiles/modulefile.prod_util.wcoss_cray_userlib deleted file mode 100644 index dd5209fcf0..0000000000 --- a/modulefiles/modulefile.prod_util.wcoss_cray_userlib +++ /dev/null @@ -1,13 +0,0 @@ -#%Module##################################################### -module purge -module load modules -module load PrgEnv-intel -module load cray-mpich -module load craype-sandybridge - -module unuse /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module use -a $MOD_PATH -module load w3nco/v2.0.6 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.prod_util.wcoss_dell_p3 b/modulefiles/modulefile.prod_util.wcoss_dell_p3 deleted file mode 100644 index 9186e13eb8..0000000000 --- a/modulefiles/modulefile.prod_util.wcoss_dell_p3 +++ /dev/null @@ -1,6 +0,0 @@ -#%Module##################################################### - -module load w3nco/2.0.6 - -export FCMP=ifort -export CCMP=icc diff --git a/modulefiles/modulefile.wgrib2.theia b/modulefiles/modulefile.wgrib2.theia deleted file mode 100644 index b1568e8cea..0000000000 --- a/modulefiles/modulefile.wgrib2.theia +++ /dev/null @@ -1,19 +0,0 @@ -#%Module###################################################################### - -module load intel/14.0.2 -module load netcdf/4.3.0 -module load hdf5/1.8.14 -module load szip/2.1 - -module use -a $MOD_PATH -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 -module load ip/v3.0.0 -module load sp/v2.0.2 -module load g2c/v1.5.0 - -export NETCDF_INCLUDE="-I${NETCDF}/include" - -export FCMP=ifort -export CCMP=icc diff --git a/modulefiles/modulefile.wgrib2.wcoss b/modulefiles/modulefile.wgrib2.wcoss deleted file mode 100644 index 0eea72e391..0000000000 --- a/modulefiles/modulefile.wgrib2.wcoss +++ /dev/null @@ -1,24 +0,0 @@ -#%Module###################################################################### -############################################################# -## Lin.Gan@noaa.gov -## EMC -## wgrib2 v2.0.5 -############################################################# -proc ModulesHelp { } { -puts stderr "Set environment veriables for wgrib2" -puts stderr "This module initializes the users environment" -puts stderr "to build the wgrib2 for WCOSS production.\n" -} -module-whatis "wgrib2" - -set ver v2.0.5 - -module load ics/15.0.6 -module load NetCDF/4.2/serial -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 -module load ip/v3.0.0 -module load sp/v2.0.2 -module load g2c/v1.5.0 - diff --git a/modulefiles/modulefile.wgrib2.wcoss_cray b/modulefiles/modulefile.wgrib2.wcoss_cray deleted file mode 100644 index 4d933141ad..0000000000 --- a/modulefiles/modulefile.wgrib2.wcoss_cray +++ /dev/null @@ -1,13 +0,0 @@ -#%Module###################################################################### -module load PrgEnv-gnu/5.2.56 -module load cray-netcdf/4.3.2 -module load craype/2.3.0 -module load craype-sandybridge -module load /gpfs/hps/nco/ops/nwtest/lib/modulefiles/g2c-gnu/1.5.0 - -module load jasper-gnu-sandybridge/1.900.1 -module load png-gnu-sandybridge/1.2.49 -module load zlib-gnu-sandybridge/1.2.7 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.wgrib2.wcoss_cray_userlib b/modulefiles/modulefile.wgrib2.wcoss_cray_userlib deleted file mode 100644 index 7b41a17f9f..0000000000 --- a/modulefiles/modulefile.wgrib2.wcoss_cray_userlib +++ /dev/null @@ -1,15 +0,0 @@ -#%Module###################################################################### -module load PrgEnv-gnu/5.2.56 -module load cray-netcdf/4.3.2 -module load craype/2.3.0 -module load craype-sandybridge - -module unuse /gpfs/hps/nco/ops/nwprod/lib/modulefiles -module use -a $MOD_PATH -module load jasper/v1.900.1 -module load png/v1.2.44 -module load z/v1.2.6 -module load g2c/v1.5.0 - -export FCMP=ftn -export CCMP=cc diff --git a/modulefiles/modulefile.wgrib2.wcoss_dell_p3 b/modulefiles/modulefile.wgrib2.wcoss_dell_p3 deleted file mode 100644 index 9a43e3ffd9..0000000000 --- a/modulefiles/modulefile.wgrib2.wcoss_dell_p3 +++ /dev/null @@ -1,11 +0,0 @@ -#%Module###################################################################### - -module load ips/18.0.1.163 - -module load g2c/1.5.0 -module load jasper/1.900.1 -module load libpng/1.2.59 -module load zlib/1.2.11 - -export FCMP=ifort -export CCMP=icc diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 1af1b2b16b..34bd2d6d4b 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -51,40 +51,60 @@ echo " .... Library build not currently supported .... " # build fv3 #------------------------------------ $Build_fv3gfs && { -echo " .... Building fv3 .... " -./build_fv3.sh > $logs_dir/build_fv3.log 2>&1 +if [ ! -d fv3gfs.fd ]; then + echo " **** SKIPPING: FV3GFS sorc folder (fv3gfs.fd) not present" +else + echo " .... Building fv3 .... " + ./build_fv3.sh > $logs_dir/build_fv3.log 2>&1 +fi } #------------------------------------ # build gsi #------------------------------------ $Build_gsi && { -echo " .... Building gsi .... " -./build_gsi.sh > $logs_dir/build_gsi.log 2>&1 +if [ ! -d gsi.fd ]; then + echo " **** SKIPPING: GSI sorc folder (gsi.fd) not present" +else + echo " .... Building gsi .... " + ./build_gsi.sh > $logs_dir/build_gsi.log 2>&1 +fi } #------------------------------------ # build ncep_post #------------------------------------ $Build_ncep_post && { -echo " .... Building ncep_post .... " -./build_ncep_post.sh > $logs_dir/build_ncep_post.log 2>&1 +if [ ! -d gfs_post.fd ]; then + echo " **** SKIPPING: EMC_post sorc folder (gfs_post.fd) not present" +else + echo " .... Building ncep_post .... " + ./build_ncep_post.sh > $logs_dir/build_ncep_post.log 2>&1 +fi } #------------------------------------ # build ufs_utils #------------------------------------ $Build_ufs_utils && { -echo " .... Building ufs_utils .... " -./build_ufs_utils.sh > $logs_dir/build_ufs_utils.log 2>&1 +if [ ! -d ufs_utils.fd ]; then + echo " **** SKIPPING: UFS_UTILS sorc folder (ufs_utils.fd) not present" +else + echo " .... Building ufs_utils .... " + ./build_ufs_utils.sh > $logs_dir/build_ufs_utils.log 2>&1 +fi } #------------------------------------ # build gfs_wafs #------------------------------------ $Build_gfs_wafs && { -echo " .... Building gfs_wafs .... " -./build_gfs_wafs.sh > $logs_dir/build_gfs_wafs .log 2>&1 +if [ ! -d gfs_wafs.fd ]; then + echo " **** SKIPPING: GFS WAFS sorc folder (gfs_wafs.fd) not present" +else + echo " .... Building gfs_wafs .... " + ./build_gfs_wafs.sh > $logs_dir/build_gfs_wafs .log 2>&1 +fi } #------------------------------------ @@ -178,24 +198,5 @@ if [ $target = wcoss -o $target = wcoss_cray -o $target = wcoss_dell_p3 ]; then } fi -#------------------------------------ -# build prod_util -#------------------------------------ -$Build_prod_util && { -echo " .... prod_util build not currently supported .... " -#echo " .... Building prod_util .... " -#./build_prod_util.sh > $logs_dir/build_prod_util.log 2>&1 -} - -#------------------------------------ -# build grib_util -#------------------------------------ -$Build_grib_util && { -echo " .... grib_util build not currently supported .... " -#echo " .... Building grib_util .... " -#./build_grib_util.sh > $logs_dir/build_grib_util.log 2>&1 -} - -echo;echo " .... Build system finished .... " - +echo;echo "----- Build system finished -----" exit 0 diff --git a/sorc/build_grib_util.sh b/sorc/build_grib_util.sh deleted file mode 100755 index 6569cc22c0..0000000000 --- a/sorc/build_grib_util.sh +++ /dev/null @@ -1,88 +0,0 @@ -#! /usr/bin/env bash -set -eux - -source ./machine-setup.sh > /dev/null 2>&1 -cwd=`pwd` - -USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} -if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles - source ../modulefiles/modulefile.grib_util.$target > /dev/null 2>&1 -else - export MOD_PATH=${cwd}/lib/modulefiles - if [ $target = wcoss_cray ]; then - source ../modulefiles/modulefile.grib_util.${target}_userlib > /dev/null 2>&1 - else - source ../modulefiles/modulefile.grib_util.$target > /dev/null 2>&1 - fi -fi - -# Move to util/sorc folder -cd ../util/sorc - -# Check final exec folder exists -if [ ! -d "../exec" ]; then - mkdir ../exec -fi - -for grib_util in cnvgrib copygb2 degrib2 grbindex tocgrib2 tocgrib \ - copygb grb2index grib2grib tocgrib2super -do - cd $grib_util.fd - make -f makefile_$target clean - make -f makefile_$target - make -f makefile_$target install - make -f makefile_$target clean - cd .. -done - -# -# compile wgrib -# -cd wgrib.cd - make -f makefile_$target clean - make -f makefile_$target - make -f makefile_$target install - make -f makefile_$target clean -cd .. - -# -# compile wgrib2 -# -cd $cwd -source ./machine-setup.sh > /dev/null 2>&1 - -if [ $target = wcoss_cray -a $USE_PREINST_LIBS != true ]; then - source ../modulefiles/modulefile.wgrib2.${target}_userlib > /dev/null 2>&1 -else - source ../modulefiles/modulefile.wgrib2.$target > /dev/null 2>&1 -fi - -# Move to util/sorc folder -cd ../util/sorc -cwd=`pwd` - -#---------------------------------------------------------------- -export CPPFLAGS="-ffast-math -O3 -DGFORTRAN" -cd $cwd/wgrib2.cd/gctpc/ -make -f makefile.gctpc clean -make -f makefile.gctpc -rm -f *.o -#---------------------------------------------------------------- -if [ $target = wcoss_cray ]; then - export FFLAGS=-O2 - cd $cwd/wgrib2.cd/iplib/ - make clean - make - rm -f *.o -fi -#---------------------------------------------------------------- -cd $cwd/wgrib2.cd -module list -make -f makefile_$target clean -make -f makefile_$target -make -f makefile_$target install -make -f makefile_$target clean -#---------------------------------------------------------------- - -exit diff --git a/sorc/build_prod_util.sh b/sorc/build_prod_util.sh deleted file mode 100755 index e4220f7c26..0000000000 --- a/sorc/build_prod_util.sh +++ /dev/null @@ -1,47 +0,0 @@ -#! /usr/bin/env bash -set -eux - -source ./machine-setup.sh > /dev/null 2>&1 -cwd=`pwd` - -USE_PREINST_LIBS=${USE_PREINST_LIBS:-"true"} -if [ $USE_PREINST_LIBS = true ]; then - export MOD_PATH=/scratch3/NCEPDEV/nwprod/lib/modulefiles - source ../modulefiles/modulefile.prod_util.$target > /dev/null 2>&1 -else - export MOD_PATH=${cwd}/lib/modulefiles - if [ $target = wcoss_cray ]; then - source ../modulefiles/modulefile.prod_util.${target}_userlib > /dev/null 2>&1 - else - source ../modulefiles/modulefile.prod_util.$target > /dev/null 2>&1 - fi -fi - -# Move to util/sorc folder -cd ../util/sorc - -# Check final exec folder exists -if [ ! -d "../exec" ]; then - mkdir ../exec -fi - -for prod_util in fsync_file -do - cd $prod_util.cd - make -f makefile clean - make -f makefile - make -f makefile install - make -f makefile clean - cd .. -done - -for prod_util in mdate ndate nhour -do - cd $prod_util.fd - make -f makefile clean - make -f makefile - make -f makefile install - make -f makefile clean - cd .. -done -exit diff --git a/sorc/fv3gfs_build.cfg b/sorc/fv3gfs_build.cfg index 8143eb0bd3..93fab78443 100644 --- a/sorc/fv3gfs_build.cfg +++ b/sorc/fv3gfs_build.cfg @@ -18,8 +18,6 @@ Building fv3nc2nemsio (fv3nc2nemsio) .................. yes Building regrid_nemsio (regrid_nemsio) ................ yes Building gfs_util (gfs_util) .......................... yes - Building prod_util (prod_util) ........................ no - Building grib_util (grib_util) ........................ no # -- END -- diff --git a/sorc/partial_build.sh b/sorc/partial_build.sh index 885548f052..da306969a0 100755 --- a/sorc/partial_build.sh +++ b/sorc/partial_build.sh @@ -14,9 +14,7 @@ "Build_gfs_bufrsnd" \ "Build_fv3nc2nemsio" \ "Build_regrid_nemsio" \ - "Build_gfs_util" \ - "Build_prod_util" \ - "Build_grib_util") + "Build_gfs_util") # # function parse_cfg: read config file and retrieve the values diff --git a/util/manage_externals/checkout_externals b/util/manage_externals/checkout_externals new file mode 100755 index 0000000000..a0698baef0 --- /dev/null +++ b/util/manage_externals/checkout_externals @@ -0,0 +1,36 @@ +#!/usr/bin/env python + +"""Main driver wrapper around the manic/checkout utility. + +Tool to assemble external respositories represented in an externals +description file. + +""" +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import sys +import traceback + +import manic + +if sys.hexversion < 0x02070000: + print(70 * '*') + print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) + print('It appears that you are running python {0}'.format( + '.'.join(str(x) for x in sys.version_info[0:3]))) + print(70 * '*') + sys.exit(1) + + +if __name__ == '__main__': + ARGS = manic.checkout.commandline_arguments() + try: + RET_STATUS, _ = manic.checkout.main(ARGS) + sys.exit(RET_STATUS) + except Exception as error: # pylint: disable=broad-except + manic.printlog(str(error)) + if ARGS.backtrace: + traceback.print_exc() + sys.exit(1) diff --git a/util/manage_externals/manic/__init__.py b/util/manage_externals/manic/__init__.py new file mode 100644 index 0000000000..11badedd3b --- /dev/null +++ b/util/manage_externals/manic/__init__.py @@ -0,0 +1,9 @@ +"""Public API for the manage_externals library +""" + +from manic import checkout +from manic.utils import printlog + +__all__ = [ + 'checkout', 'printlog', +] diff --git a/util/manage_externals/manic/checkout.py b/util/manage_externals/manic/checkout.py new file mode 100755 index 0000000000..edc5655954 --- /dev/null +++ b/util/manage_externals/manic/checkout.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python + +""" +Tool to assemble repositories represented in a model-description file. + +If loaded as a module (e.g., in a component's buildcpp), it can be used +to check the validity of existing subdirectories and load missing sources. +""" +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import argparse +import logging +import os +import os.path +import sys + +from manic.externals_description import create_externals_description +from manic.externals_description import read_externals_description_file +from manic.externals_status import check_safe_to_update_repos +from manic.sourcetree import SourceTree +from manic.utils import printlog, fatal_error +from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME + +if sys.hexversion < 0x02070000: + print(70 * '*') + print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) + print('It appears that you are running python {0}'.format( + VERSION_SEPERATOR.join(str(x) for x in sys.version_info[0:3]))) + print(70 * '*') + sys.exit(1) + + +# --------------------------------------------------------------------- +# +# User input +# +# --------------------------------------------------------------------- +def commandline_arguments(args=None): + """Process the command line arguments + + Params: args - optional args. Should only be used during systems + testing. + + Returns: processed command line arguments + """ + description = ''' + +%(prog)s manages checking out groups of externals from revision +control based on an externals description file. By default only the +required externals are checkout out. + +Running %(prog)s without the '--status' option will always attempt to +synchronize the working copy to exactly match the externals description. +''' + + epilog = ''' +``` +NOTE: %(prog)s *MUST* be run from the root of the source tree it +is managing. For example, if you cloned a repository with: + + $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev + +Then the root of the source tree is /path/to/some-project-dev. If you +obtained a sub-project via a checkout of another project: + + $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev + +and you need to checkout the sub-project externals, then the root of the +source tree remains /path/to/some-project-dev. Do *NOT* run %(prog)s +from within /path/to/some-project-dev/sub-project + +The root of the source tree will be referred to as `${SRC_ROOT}` below. + + +# Supported workflows + + * Checkout all required components from the default externals + description file: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s + + * To update all required components to the current values in the + externals description file, re-run %(prog)s: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s + + If there are *any* modifications to *any* working copy according + to the git or svn 'status' command, %(prog)s + will not update any external repositories. Modifications + include: modified files, added files, removed files, or missing + files. + + To avoid this safety check, edit the externals description file + and comment out the modified external block. + + * Checkout all required components from a user specified externals + description file: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s --externals my-externals.cfg + + * Status summary of the repositories managed by %(prog)s: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s --status + + ./cime + s ./components/cism + ./components/mosart + e-o ./components/rtm + M ./src/fates + e-o ./tools/PTCLM + + + where: + * column one indicates the status of the repository in relation + to the externals description file. + * column two indicates whether the working copy has modified files. + * column three shows how the repository is managed, optional or required + + Column one will be one of these values: + * s : out-of-sync : repository is checked out at a different commit + compared with the externals description + * e : empty : directory does not exist - %(prog)s has not been run + * ? : unknown : directory exists but .git or .svn directories are missing + + Column two will be one of these values: + * M : Modified : modified, added, deleted or missing files + * : blank / space : clean + * - : dash : no meaningful state, for empty repositories + + Column three will be one of these values: + * o : optional : optionally repository + * : blank / space : required repository + + * Detailed git or svn status of the repositories managed by %(prog)s: + + $ cd ${SRC_ROOT} + $ ./manage_externals/%(prog)s --status --verbose + +# Externals description file + + The externals description contains a list of the external + repositories that are used and their version control locations. The + file format is the standard ini/cfg configuration file format. Each + external is defined by a section containing the component name in + square brackets: + + * name (string) : component name, e.g. [cime], [cism], etc. + + Each section has the following keyword-value pairs: + + * required (boolean) : whether the component is a required checkout, + 'true' or 'false'. + + * local_path (string) : component path *relative* to where + %(prog)s is called. + + * protoctol (string) : version control protocol that is used to + manage the component. Valid values are 'git', 'svn', + 'externals_only'. + + Switching an external between different protocols is not + supported, e.g. from svn to git. To switch protocols, you need to + manually move the old working copy to a new location. + + Note: 'externals_only' will only process the external's own + external description file without trying to manage a repository + for the component. This is used for retrieving externals for + standalone components like cam and ctsm which also serve as + sub-components within a larger project. If the source root of the + externals_only component is the same as the main source root, then + the local path must be set to '.', the unix current working + directory, e. g. 'local_path = .' + + * repo_url (string) : URL for the repository location, examples: + * https://svn-ccsm-models.cgd.ucar.edu/glc + * git@github.com:esmci/cime.git + * /path/to/local/repository + * . + + NOTE: To operate on only the local clone and and ignore remote + repositories, set the url to '.' (the unix current path), + i.e. 'repo_url = .' . This can be used to checkout a local branch + instead of the upstream branch. + + If a repo url is determined to be a local path (not a network url) + then user expansion, e.g. ~/, and environment variable expansion, + e.g. $HOME or $REPO_ROOT, will be performed. + + Relative paths are difficult to get correct, especially for mixed + use repos. It is advised that local paths expand to absolute paths. + If relative paths are used, they should be relative to one level + above local_path. If local path is 'src/foo', the the relative url + should be relative to 'src'. + + * tag (string) : tag to checkout + + * hash (string) : the git hash to checkout. Only applies to git + repositories. + + * branch (string) : branch to checkout from the specified + repository. Specifying a branch on a remote repository means that + %(prog)s will checkout the version of the branch in the remote, + not the the version in the local repository (if it exists). + + Note: one and only one of tag, branch hash must be supplied. + + * externals (string) : used to make manage_externals aware of + sub-externals required by an external. This is a relative path to + the external's root directory. For example, if LIBX is often used + as a sub-external, it might have an externals file (for its + externals) called Externals_LIBX.cfg. To use libx as a standalone + checkout, it would have another file, Externals.cfg with the + following entry: + + [ libx ] + local_path = . + protocol = externals_only + externals = Externals_LIBX.cfg + required = True + + Now, %(prog)s will process Externals.cfg and also process + Externals_LIBX.cfg as if it was a sub-external. + + * from_submodule (True / False) : used to pull the repo_url, local_path, + and hash properties for this external from the .gitmodules file in + this repository. Note that the section name (the entry in square + brackets) must match the name in the .gitmodules file. + If from_submodule is True, the protocol must be git and no repo_url, + local_path, hash, branch, or tag entries are allowed. + Default: False + + * sparse (string) : used to control a sparse checkout. This optional + entry should point to a filename (path relative to local_path) that + contains instructions on which repository paths to include (or + exclude) from the working tree. + See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree + Default: sparse checkout is disabled + + * Lines beginning with '#' or ';' are comments and will be ignored. + +# Obtaining this tool, reporting issues, etc. + + The master repository for manage_externals is + https://github.com/ESMCI/manage_externals. Any issues with this tool + should be reported there. + +# Troubleshooting + +Operations performed by manage_externals utilities are explicit and +data driven. %(prog)s will always attempt to make the working copy +*exactly* match what is in the externals file when modifying the +working copy of a repository. + +If %(prog)s is not doing what you expected, double check the contents +of the externals description file or examine the output of +./manage_externals/%(prog)s --status + +''' + + parser = argparse.ArgumentParser( + description=description, epilog=epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + + # + # user options + # + parser.add_argument("components", nargs="*", + help="Specific component(s) to checkout. By default, " + "all required externals are checked out.") + + parser.add_argument('-e', '--externals', nargs='?', + default='Externals.cfg', + help='The externals description filename. ' + 'Default: %(default)s.') + + parser.add_argument('-o', '--optional', action='store_true', default=False, + help='By default only the required externals ' + 'are checked out. This flag will also checkout the ' + 'optional externals.') + + parser.add_argument('-S', '--status', action='store_true', default=False, + help='Output the status of the repositories managed by ' + '%(prog)s. By default only summary information ' + 'is provided. Use the verbose option to see details.') + + parser.add_argument('-v', '--verbose', action='count', default=0, + help='Output additional information to ' + 'the screen and log file. This flag can be ' + 'used up to two times, increasing the ' + 'verbosity level each time.') + + parser.add_argument('--svn-ignore-ancestry', action='store_true', default=False, + help='By default, subversion will abort if a component is ' + 'already checked out and there is no common ancestry with ' + 'the new URL. This flag passes the "--ignore-ancestry" flag ' + 'to the svn switch call. (This is not recommended unless ' + 'you are sure about what you are doing.)') + + # + # developer options + # + parser.add_argument('--backtrace', action='store_true', + help='DEVELOPER: show exception backtraces as extra ' + 'debugging output') + + parser.add_argument('-d', '--debug', action='store_true', default=False, + help='DEVELOPER: output additional debugging ' + 'information to the screen and log file.') + + logging_group = parser.add_mutually_exclusive_group() + + logging_group.add_argument('--logging', dest='do_logging', + action='store_true', + help='DEVELOPER: enable logging.') + logging_group.add_argument('--no-logging', dest='do_logging', + action='store_false', default=False, + help='DEVELOPER: disable logging ' + '(this is the default)') + + if args: + options = parser.parse_args(args) + else: + options = parser.parse_args() + return options + + +# --------------------------------------------------------------------- +# +# main +# +# --------------------------------------------------------------------- +def main(args): + """ + Function to call when module is called from the command line. + Parse externals file and load required repositories or all repositories if + the --all option is passed. + + Returns a tuple (overall_status, tree_status). overall_status is 0 + on success, non-zero on failure. tree_status gives the full status + *before* executing the checkout command - i.e., the status that it + used to determine if it's safe to proceed with the checkout. + """ + if args.do_logging: + logging.basicConfig(filename=LOG_FILE_NAME, + format='%(levelname)s : %(asctime)s : %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + level=logging.DEBUG) + + program_name = os.path.basename(sys.argv[0]) + logging.info('Beginning of %s', program_name) + + load_all = False + if args.optional: + load_all = True + + root_dir = os.path.abspath(os.getcwd()) + external_data = read_externals_description_file(root_dir, args.externals) + external = create_externals_description( + external_data, components=args.components) + + for comp in args.components: + if comp not in external.keys(): + fatal_error( + "No component {} found in {}".format( + comp, args.externals)) + + source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) + printlog('Checking status of externals: ', end='') + tree_status = source_tree.status() + printlog('') + + if args.status: + # user requested status-only + for comp in sorted(tree_status.keys()): + tree_status[comp].log_status_message(args.verbose) + else: + # checkout / update the external repositories. + safe_to_update = check_safe_to_update_repos(tree_status) + if not safe_to_update: + # print status + for comp in sorted(tree_status.keys()): + tree_status[comp].log_status_message(args.verbose) + # exit gracefully + msg = """The external repositories labeled with 'M' above are not in a clean state. + +The following are two options for how to proceed: + +(1) Go into each external that is not in a clean state and issue either + an 'svn status' or a 'git status' command. Either revert or commit + your changes so that all externals are in a clean state. (Note, + though, that it is okay to have untracked files in your working + directory.) Then rerun {program_name}. + +(2) Alternatively, you do not have to rely on {program_name}. Instead, you + can manually update out-of-sync externals (labeled with 's' above) + as described in the configuration file {config_file}. + + +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before re-running the +checkout_externals tool. +""".format(program_name=program_name, config_file=args.externals) + + printlog('-' * 70) + printlog(msg) + printlog('-' * 70) + else: + if not args.components: + source_tree.checkout(args.verbose, load_all) + for comp in args.components: + source_tree.checkout(args.verbose, load_all, load_comp=comp) + printlog('') + + logging.info('%s completed without exceptions.', program_name) + # NOTE(bja, 2017-11) tree status is used by the systems tests + return 0, tree_status diff --git a/util/manage_externals/manic/externals_description.py b/util/manage_externals/manic/externals_description.py new file mode 100644 index 0000000000..b0c4f736a7 --- /dev/null +++ b/util/manage_externals/manic/externals_description.py @@ -0,0 +1,794 @@ +#!/usr/bin/env python + +"""Model description + +Model description is the representation of the various externals +included in the model. It processes in input data structure, and +converts it into a standard interface that is used by the rest of the +system. + +To maintain backward compatibility, externals description files should +follow semantic versioning rules, http://semver.org/ + + + +""" +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import logging +import os +import os.path +import re + +# ConfigParser in python2 was renamed to configparser in python3. +# In python2, ConfigParser returns byte strings, str, instead of unicode. +# We need unicode to be compatible with xml and json parser and python3. +try: + # python2 + from ConfigParser import SafeConfigParser as config_parser + from ConfigParser import MissingSectionHeaderError + from ConfigParser import NoSectionError, NoOptionError + + USE_PYTHON2 = True + + def config_string_cleaner(text): + """convert strings into unicode + """ + return text.decode('utf-8') +except ImportError: + # python3 + from configparser import ConfigParser as config_parser + from configparser import MissingSectionHeaderError + from configparser import NoSectionError, NoOptionError + + USE_PYTHON2 = False + + def config_string_cleaner(text): + """Python3 already uses unicode strings, so just return the string + without modification. + + """ + return text + +from .utils import printlog, fatal_error, str_to_bool, expand_local_url +from .utils import execute_subprocess +from .global_constants import EMPTY_STR, PPRINTER, VERSION_SEPERATOR + +# +# Globals +# +DESCRIPTION_SECTION = 'externals_description' +VERSION_ITEM = 'schema_version' + + +def read_externals_description_file(root_dir, file_name): + """Read a file containing an externals description and + create its internal representation. + + """ + root_dir = os.path.abspath(root_dir) + msg = 'In directory : {0}'.format(root_dir) + logging.info(msg) + printlog('Processing externals description file : {0}'.format(file_name)) + + file_path = os.path.join(root_dir, file_name) + if not os.path.exists(file_name): + if file_name.lower() == "none": + msg = ('INTERNAL ERROR: Attempt to read externals file ' + 'from {0} when not configured'.format(file_path)) + else: + msg = ('ERROR: Model description file, "{0}", does not ' + 'exist at path:\n {1}\nDid you run from the root of ' + 'the source tree?'.format(file_name, file_path)) + + fatal_error(msg) + + externals_description = None + if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: + externals_description = read_gitmodules_file(root_dir, file_name) + else: + try: + config = config_parser() + config.read(file_path) + externals_description = config + except MissingSectionHeaderError: + # not a cfg file + pass + + if externals_description is None: + msg = 'Unknown file format!' + fatal_error(msg) + + return externals_description + +class LstripReader(object): + "LstripReader formats .gitmodules files to be acceptable for configparser" + def __init__(self, filename): + with open(filename, 'r') as infile: + lines = infile.readlines() + self._lines = list() + self._num_lines = len(lines) + self._index = 0 + for line in lines: + self._lines.append(line.lstrip()) + + def readlines(self): + """Return all the lines from this object's file""" + return self._lines + + def readline(self, size=-1): + """Format and return the next line or raise StopIteration""" + try: + line = self.next() + except StopIteration: + line = '' + + if (size > 0) and (len(line) < size): + return line[0:size] + + return line + + def __iter__(self): + """Begin an iteration""" + self._index = 0 + return self + + def next(self): + """Return the next line or raise StopIteration""" + if self._index >= self._num_lines: + raise StopIteration + + self._index = self._index + 1 + return self._lines[self._index - 1] + + def __next__(self): + return self.next() + +def git_submodule_status(repo_dir): + """Run the git submodule status command to obtain submodule hashes. + """ + # This function is here instead of GitRepository to avoid a dependency loop + cwd = os.getcwd() + os.chdir(repo_dir) + cmd = ['git', 'submodule', 'status'] + git_output = execute_subprocess(cmd, output_to_caller=True) + submodules = {} + submods = git_output.split('\n') + for submod in submods: + if submod: + status = submod[0] + items = submod[1:].split(' ') + if len(items) > 2: + tag = items[2] + else: + tag = None + + submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} + + os.chdir(cwd) + return submodules + +def parse_submodules_desc_section(section_items, file_path): + """Find the path and url for this submodule description""" + path = None + url = None + for item in section_items: + name = item[0].strip().lower() + if name == 'path': + path = item[1].strip() + elif name == 'url': + url = item[1].strip() + else: + msg = 'WARNING: Ignoring unknown {} property, in {}' + msg = msg.format(item[0], file_path) # fool pylint + logging.warning(msg) + + return path, url + +def read_gitmodules_file(root_dir, file_name): + # pylint: disable=deprecated-method + # Disabling this check because the method is only used for python2 + """Read a .gitmodules file and convert it to be compatible with an + externals description. + """ + root_dir = os.path.abspath(root_dir) + msg = 'In directory : {0}'.format(root_dir) + logging.info(msg) + printlog('Processing submodules description file : {0}'.format(file_name)) + + file_path = os.path.join(root_dir, file_name) + if not os.path.exists(file_name): + msg = ('ERROR: submodules description file, "{0}", does not ' + 'exist at path:\n {1}'.format(file_name, file_path)) + fatal_error(msg) + + submodules_description = None + externals_description = None + try: + config = config_parser() + if USE_PYTHON2: + config.readfp(LstripReader(file_path), filename=file_name) + else: + config.read_file(LstripReader(file_path), source=file_name) + + submodules_description = config + except MissingSectionHeaderError: + # not a cfg file + pass + + if submodules_description is None: + msg = 'Unknown file format!' + fatal_error(msg) + else: + # Convert the submodules description to an externals description + externals_description = config_parser() + # We need to grab all the commit hashes for this repo + submods = git_submodule_status(root_dir) + for section in submodules_description.sections(): + if section[0:9] == 'submodule': + sec_name = section[9:].strip(' "') + externals_description.add_section(sec_name) + section_items = submodules_description.items(section) + path, url = parse_submodules_desc_section(section_items, + file_path) + + if path is None: + msg = 'Submodule {} missing path'.format(sec_name) + fatal_error(msg) + + if url is None: + msg = 'Submodule {} missing url'.format(sec_name) + fatal_error(msg) + + externals_description.set(sec_name, + ExternalsDescription.PATH, path) + externals_description.set(sec_name, + ExternalsDescription.PROTOCOL, 'git') + externals_description.set(sec_name, + ExternalsDescription.REPO_URL, url) + externals_description.set(sec_name, + ExternalsDescription.REQUIRED, 'True') + git_hash = submods[sec_name]['hash'] + externals_description.set(sec_name, + ExternalsDescription.HASH, git_hash) + + # Required items + externals_description.add_section(DESCRIPTION_SECTION) + externals_description.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0') + + return externals_description + +def create_externals_description( + model_data, model_format='cfg', components=None, parent_repo=None): + """Create the a externals description object from the provided data + """ + externals_description = None + if model_format == 'dict': + externals_description = ExternalsDescriptionDict( + model_data, components=components) + elif model_format == 'cfg': + major, _, _ = get_cfg_schema_version(model_data) + if major == 1: + externals_description = ExternalsDescriptionConfigV1( + model_data, components=components, parent_repo=parent_repo) + else: + msg = ('Externals description file has unsupported schema ' + 'version "{0}".'.format(major)) + fatal_error(msg) + else: + msg = 'Unknown model data format "{0}"'.format(model_format) + fatal_error(msg) + return externals_description + + +def get_cfg_schema_version(model_cfg): + """Extract the major, minor, patch version of the config file schema + + Params: + model_cfg - config parser object containing the externas description data + + Returns: + major = integer major version + minor = integer minor version + patch = integer patch version + """ + semver_str = '' + try: + semver_str = model_cfg.get(DESCRIPTION_SECTION, VERSION_ITEM) + except (NoSectionError, NoOptionError): + msg = ('externals description file must have the required ' + 'section: "{0}" and item "{1}"'.format(DESCRIPTION_SECTION, + VERSION_ITEM)) + fatal_error(msg) + + # NOTE(bja, 2017-11) Assume we don't care about the + # build/pre-release metadata for now! + version_list = re.split(r'[-+]', semver_str) + version_str = version_list[0] + version = version_str.split(VERSION_SEPERATOR) + try: + major = int(version[0].strip()) + minor = int(version[1].strip()) + patch = int(version[2].strip()) + except ValueError: + msg = ('Config file schema version must have integer digits for ' + 'major, minor and patch versions. ' + 'Received "{0}"'.format(version_str)) + fatal_error(msg) + return major, minor, patch + + +class ExternalsDescription(dict): + """Base externals description class that is independent of the user input + format. Different input formats can all be converted to this + representation to provide a consistent represtentation for the + rest of the objects in the system. + + NOTE(bja, 2018-03): do NOT define _schema_major etc at the class + level in the base class. The nested/recursive nature of externals + means different schema versions may be present in a single run! + + All inheriting classes must overwrite: + self._schema_major and self._input_major + self._schema_minor and self._input_minor + self._schema_patch and self._input_patch + + where _schema_x is the supported schema, _input_x is the user + input value. + + """ + # keywords defining the interface into the externals description data + EXTERNALS = 'externals' + BRANCH = 'branch' + SUBMODULE = 'from_submodule' + HASH = 'hash' + NAME = 'name' + PATH = 'local_path' + PROTOCOL = 'protocol' + REPO = 'repo' + REPO_URL = 'repo_url' + REQUIRED = 'required' + TAG = 'tag' + SPARSE = 'sparse' + + PROTOCOL_EXTERNALS_ONLY = 'externals_only' + PROTOCOL_GIT = 'git' + PROTOCOL_SVN = 'svn' + GIT_SUBMODULES_FILENAME = '.gitmodules' + KNOWN_PRROTOCOLS = [PROTOCOL_GIT, PROTOCOL_SVN, PROTOCOL_EXTERNALS_ONLY] + + # v1 xml keywords + _V1_TREE_PATH = 'TREE_PATH' + _V1_ROOT = 'ROOT' + _V1_TAG = 'TAG' + _V1_BRANCH = 'BRANCH' + _V1_REQ_SOURCE = 'REQ_SOURCE' + + _source_schema = {REQUIRED: True, + PATH: 'string', + EXTERNALS: 'string', + SUBMODULE : True, + REPO: {PROTOCOL: 'string', + REPO_URL: 'string', + TAG: 'string', + BRANCH: 'string', + HASH: 'string', + SPARSE: 'string', + } + } + + def __init__(self, parent_repo=None): + """Convert the xml into a standardized dict that can be used to + construct the source objects + + """ + dict.__init__(self) + + self._schema_major = None + self._schema_minor = None + self._schema_patch = None + self._input_major = None + self._input_minor = None + self._input_patch = None + self._parent_repo = parent_repo + + def _verify_schema_version(self): + """Use semantic versioning rules to verify we can process this schema. + + """ + known = '{0}.{1}.{2}'.format(self._schema_major, + self._schema_minor, + self._schema_patch) + received = '{0}.{1}.{2}'.format(self._input_major, + self._input_minor, + self._input_patch) + + if self._input_major != self._schema_major: + # should never get here, the factory should handle this correctly! + msg = ('DEV_ERROR: version "{0}" parser received ' + 'version "{1}" input.'.format(known, received)) + fatal_error(msg) + + if self._input_minor > self._schema_minor: + msg = ('Incompatible schema version:\n' + ' User supplied schema version "{0}" is too new."\n' + ' Can only process version "{1}" files and ' + 'older.'.format(received, known)) + fatal_error(msg) + + if self._input_patch > self._schema_patch: + # NOTE(bja, 2018-03) ignoring for now... Not clear what + # conditions the test is needed. + pass + + def _check_user_input(self): + """Run a series of checks to attempt to validate the user input and + detect errors as soon as possible. + + NOTE(bja, 2018-03) These checks are called *after* the file is + read. That means the schema check can not occur here. + + Note: the order is important. check_optional will create + optional with null data. run check_data first to ensure + required data was provided correctly by the user. + + """ + self._check_data() + self._check_optional() + self._validate() + + def _check_data(self): + # pylint: disable=too-many-branches,too-many-statements + """Check user supplied data is valid where possible. + """ + for ext_name in self.keys(): + if (self[ext_name][self.REPO][self.PROTOCOL] + not in self.KNOWN_PRROTOCOLS): + msg = 'Unknown repository protocol "{0}" in "{1}".'.format( + self[ext_name][self.REPO][self.PROTOCOL], ext_name) + fatal_error(msg) + + if (self[ext_name][self.REPO][self.PROTOCOL] == + self.PROTOCOL_SVN): + if self.HASH in self[ext_name][self.REPO]: + msg = ('In repo description for "{0}". svn repositories ' + 'may not include the "hash" keyword.'.format( + ext_name)) + fatal_error(msg) + + if ((self[ext_name][self.REPO][self.PROTOCOL] != self.PROTOCOL_GIT) + and (self.SUBMODULE in self[ext_name])): + msg = ('self.SUBMODULE is only supported with {0} protocol, ' + '"{1}" is defined as an {2} repository') + fatal_error(msg.format(self.PROTOCOL_GIT, ext_name, + self[ext_name][self.REPO][self.PROTOCOL])) + + if (self[ext_name][self.REPO][self.PROTOCOL] != + self.PROTOCOL_EXTERNALS_ONLY): + ref_count = 0 + found_refs = '' + if self.TAG in self[ext_name][self.REPO]: + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.TAG, self[ext_name][self.REPO][self.TAG], + found_refs) + if self.BRANCH in self[ext_name][self.REPO]: + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.BRANCH, self[ext_name][self.REPO][self.BRANCH], + found_refs) + if self.HASH in self[ext_name][self.REPO]: + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.HASH, self[ext_name][self.REPO][self.HASH], + found_refs) + if (self.SUBMODULE in self[ext_name] and + self[ext_name][self.SUBMODULE]): + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.SUBMODULE, + self[ext_name][self.SUBMODULE], found_refs) + + if ref_count > 1: + msg = 'Model description is over specified! ' + if self.SUBMODULE in self[ext_name]: + msg += ('from_submodule is not compatible with ' + '"tag", "branch", or "hash" ') + else: + msg += (' Only one of "tag", "branch", or "hash" ' + 'may be specified ') + + msg += 'for repo description of "{0}".'.format(ext_name) + msg = '{0}\nFound: {1}'.format(msg, found_refs) + fatal_error(msg) + elif ref_count < 1: + msg = ('Model description is under specified! One of ' + '"tag", "branch", or "hash" must be specified for ' + 'repo description of "{0}"'.format(ext_name)) + fatal_error(msg) + + if (self.REPO_URL not in self[ext_name][self.REPO] and + (self.SUBMODULE not in self[ext_name] or + not self[ext_name][self.SUBMODULE])): + msg = ('Model description is under specified! Must have ' + '"repo_url" in repo ' + 'description for "{0}"'.format(ext_name)) + fatal_error(msg) + + if (self.SUBMODULE in self[ext_name] and + self[ext_name][self.SUBMODULE]): + if self.REPO_URL in self[ext_name][self.REPO]: + msg = ('Model description is over specified! ' + 'from_submodule keyword is not compatible ' + 'with {0} keyword for'.format(self.REPO_URL)) + msg = '{0} repo description of "{1}"'.format(msg, + ext_name) + fatal_error(msg) + + if self.PATH in self[ext_name]: + msg = ('Model description is over specified! ' + 'from_submodule keyword is not compatible with ' + '{0} keyword for'.format(self.PATH)) + msg = '{0} repo description of "{1}"'.format(msg, + ext_name) + fatal_error(msg) + + if self.REPO_URL in self[ext_name][self.REPO]: + url = expand_local_url( + self[ext_name][self.REPO][self.REPO_URL], ext_name) + self[ext_name][self.REPO][self.REPO_URL] = url + + def _check_optional(self): + # pylint: disable=too-many-branches + """Some fields like externals, repo:tag repo:branch are + (conditionally) optional. We don't want the user to be + required to enter them in every externals description file, but + still want to validate the input. Check conditions and add + default values if appropriate. + + """ + submod_desc = None # Only load submodules info once + for field in self: + # truely optional + if self.EXTERNALS not in self[field]: + self[field][self.EXTERNALS] = EMPTY_STR + + # git and svn repos must tags and branches for validation purposes. + if self.TAG not in self[field][self.REPO]: + self[field][self.REPO][self.TAG] = EMPTY_STR + if self.BRANCH not in self[field][self.REPO]: + self[field][self.REPO][self.BRANCH] = EMPTY_STR + if self.HASH not in self[field][self.REPO]: + self[field][self.REPO][self.HASH] = EMPTY_STR + if self.REPO_URL not in self[field][self.REPO]: + self[field][self.REPO][self.REPO_URL] = EMPTY_STR + if self.SPARSE not in self[field][self.REPO]: + self[field][self.REPO][self.SPARSE] = EMPTY_STR + + # from_submodule has a complex relationship with other fields + if self.SUBMODULE in self[field]: + # User wants to use submodule information, is it available? + if self._parent_repo is None: + # No parent == no submodule information + PPRINTER.pprint(self[field]) + msg = 'No parent submodule for "{0}"'.format(field) + fatal_error(msg) + elif self._parent_repo.protocol() != self.PROTOCOL_GIT: + PPRINTER.pprint(self[field]) + msg = 'Parent protocol, "{0}", does not support submodules' + fatal_error(msg.format(self._parent_repo.protocol())) + else: + args = self._repo_config_from_submodule(field, submod_desc) + repo_url, repo_path, ref_hash, submod_desc = args + + if repo_url is None: + msg = ('Cannot checkout "{0}" as a submodule, ' + 'repo not found in {1} file') + fatal_error(msg.format(field, + self.GIT_SUBMODULES_FILENAME)) + # Fill in submodule fields + self[field][self.REPO][self.REPO_URL] = repo_url + self[field][self.REPO][self.HASH] = ref_hash + self[field][self.PATH] = repo_path + + if self[field][self.SUBMODULE]: + # We should get everything from the parent submodule + # configuration. + pass + # No else (from _submodule = False is the default) + else: + # Add the default value (not using submodule information) + self[field][self.SUBMODULE] = False + + def _repo_config_from_submodule(self, field, submod_desc): + """Find the external config information for a repository from + its submodule configuration information. + """ + if submod_desc is None: + repo_path = os.getcwd() # Is this always correct? + submod_file = self._parent_repo.submodules_file(repo_path=repo_path) + if submod_file is None: + msg = ('Cannot checkout "{0}" from submodule information\n' + ' Parent repo, "{1}" does not have submodules') + fatal_error(msg.format(field, self._parent_repo.name())) + + submod_file = read_gitmodules_file(repo_path, submod_file) + submod_desc = create_externals_description(submod_file) + + # Can we find our external? + repo_url = None + repo_path = None + ref_hash = None + for ext_field in submod_desc: + if field == ext_field: + ext = submod_desc[ext_field] + repo_url = ext[self.REPO][self.REPO_URL] + repo_path = ext[self.PATH] + ref_hash = ext[self.REPO][self.HASH] + break + + return repo_url, repo_path, ref_hash, submod_desc + + def _validate(self): + """Validate that the parsed externals description contains all necessary + fields. + + """ + def print_compare_difference(data_a, data_b, loc_a, loc_b): + """Look through the data structures and print the differences. + + """ + for item in data_a: + if item in data_b: + if not isinstance(data_b[item], type(data_a[item])): + printlog(" {item}: {loc} = {val} ({val_type})".format( + item=item, loc=loc_a, val=data_a[item], + val_type=type(data_a[item]))) + printlog(" {item} {loc} = {val} ({val_type})".format( + item=' ' * len(item), loc=loc_b, val=data_b[item], + val_type=type(data_b[item]))) + else: + printlog(" {item}: {loc} = {val} ({val_type})".format( + item=item, loc=loc_a, val=data_a[item], + val_type=type(data_a[item]))) + printlog(" {item} {loc} missing".format( + item=' ' * len(item), loc=loc_b)) + + def validate_data_struct(schema, data): + """Compare a data structure against a schema and validate all required + fields are present. + + """ + is_valid = False + in_ref = True + valid = True + if isinstance(schema, dict) and isinstance(data, dict): + # Both are dicts, recursively verify that all fields + # in schema are present in the data. + for key in schema: + in_ref = in_ref and (key in data) + if in_ref: + valid = valid and ( + validate_data_struct(schema[key], data[key])) + + is_valid = in_ref and valid + else: + # non-recursive structure. verify data and schema have + # the same type. + is_valid = isinstance(data, type(schema)) + + if not is_valid: + printlog(" Unmatched schema and input:") + if isinstance(schema, dict): + print_compare_difference(schema, data, 'schema', 'input') + print_compare_difference(data, schema, 'input', 'schema') + else: + printlog(" schema = {0} ({1})".format( + schema, type(schema))) + printlog(" input = {0} ({1})".format(data, type(data))) + + return is_valid + + for field in self: + valid = validate_data_struct(self._source_schema, self[field]) + if not valid: + PPRINTER.pprint(self._source_schema) + PPRINTER.pprint(self[field]) + msg = 'ERROR: source for "{0}" did not validate'.format(field) + fatal_error(msg) + + +class ExternalsDescriptionDict(ExternalsDescription): + """Create a externals description object from a dictionary using the API + representations. Primarily used to simplify creating model + description files for unit testing. + + """ + + def __init__(self, model_data, components=None): + """Parse a native dictionary into a externals description. + """ + ExternalsDescription.__init__(self) + self._schema_major = 1 + self._schema_minor = 0 + self._schema_patch = 0 + self._input_major = 1 + self._input_minor = 0 + self._input_patch = 0 + self._verify_schema_version() + if components: + for key in model_data.items(): + if key not in components: + del model_data[key] + + self.update(model_data) + self._check_user_input() + + +class ExternalsDescriptionConfigV1(ExternalsDescription): + """Create a externals description object from a config_parser object, + schema version 1. + + """ + + def __init__(self, model_data, components=None, parent_repo=None): + """Convert the config data into a standardized dict that can be used to + construct the source objects + + """ + ExternalsDescription.__init__(self, parent_repo=parent_repo) + self._schema_major = 1 + self._schema_minor = 1 + self._schema_patch = 0 + self._input_major, self._input_minor, self._input_patch = \ + get_cfg_schema_version(model_data) + self._verify_schema_version() + self._remove_metadata(model_data) + self._parse_cfg(model_data, components=components) + self._check_user_input() + + @staticmethod + def _remove_metadata(model_data): + """Remove the metadata section from the model configuration file so + that it is simpler to look through the file and construct the + externals description. + + """ + model_data.remove_section(DESCRIPTION_SECTION) + + def _parse_cfg(self, cfg_data, components=None): + """Parse a config_parser object into a externals description. + """ + def list_to_dict(input_list, convert_to_lower_case=True): + """Convert a list of key-value pairs into a dictionary. + """ + output_dict = {} + for item in input_list: + key = config_string_cleaner(item[0].strip()) + value = config_string_cleaner(item[1].strip()) + if convert_to_lower_case: + key = key.lower() + output_dict[key] = value + return output_dict + + for section in cfg_data.sections(): + name = config_string_cleaner(section.lower().strip()) + if components and name not in components: + continue + self[name] = {} + self[name].update(list_to_dict(cfg_data.items(section))) + self[name][self.REPO] = {} + loop_keys = self[name].copy().keys() + for item in loop_keys: + if item in self._source_schema: + if isinstance(self._source_schema[item], bool): + self[name][item] = str_to_bool(self[name][item]) + elif item in self._source_schema[self.REPO]: + self[name][self.REPO][item] = self[name][item] + del self[name][item] + else: + msg = ('Invalid input: "{sect}" contains unknown ' + 'item "{item}".'.format(sect=name, item=item)) + fatal_error(msg) diff --git a/util/manage_externals/manic/externals_status.py b/util/manage_externals/manic/externals_status.py new file mode 100644 index 0000000000..d3d238f289 --- /dev/null +++ b/util/manage_externals/manic/externals_status.py @@ -0,0 +1,164 @@ +"""ExternalStatus + +Class to store status and state information about repositories and +create a string representation. + +""" +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +from .global_constants import EMPTY_STR +from .utils import printlog, indent_string +from .global_constants import VERBOSITY_VERBOSE, VERBOSITY_DUMP + + +class ExternalStatus(object): + """Class to represent the status of a given source repository or tree. + + Individual repositories determine their own status in the + Repository objects. This object is just resposible for storing the + information and passing it up to a higher level for reporting or + global decisions. + + There are two states of concern: + + * If the repository is in-sync with the externals description file. + + * If the repostiory working copy is clean and there are no pending + transactions (e.g. add, remove, rename, untracked files). + + """ + DEFAULT = '-' + UNKNOWN = '?' + EMPTY = 'e' + MODEL_MODIFIED = 's' # a.k.a. out-of-sync + DIRTY = 'M' + + STATUS_OK = ' ' + STATUS_ERROR = '!' + + # source types + OPTIONAL = 'o' + STANDALONE = 's' + MANAGED = ' ' + + def __init__(self): + self.sync_state = self.DEFAULT + self.clean_state = self.DEFAULT + self.source_type = self.DEFAULT + self.path = EMPTY_STR + self.current_version = EMPTY_STR + self.expected_version = EMPTY_STR + self.status_output = EMPTY_STR + + def log_status_message(self, verbosity): + """Write status message to the screen and log file + """ + self._default_status_message() + if verbosity >= VERBOSITY_VERBOSE: + self._verbose_status_message() + if verbosity >= VERBOSITY_DUMP: + self._dump_status_message() + + def _default_status_message(self): + """Return the default terse status message string + """ + msg = '{sync}{clean}{src_type} {path}'.format( + sync=self.sync_state, clean=self.clean_state, + src_type=self.source_type, path=self.path) + printlog(msg) + + def _verbose_status_message(self): + """Return the verbose status message string + """ + clean_str = self.DEFAULT + if self.clean_state == self.STATUS_OK: + clean_str = 'clean sandbox' + elif self.clean_state == self.DIRTY: + clean_str = 'modified sandbox' + + sync_str = 'on {0}'.format(self.current_version) + if self.sync_state != self.STATUS_OK: + sync_str = '{current} --> {expected}'.format( + current=self.current_version, expected=self.expected_version) + msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) + printlog(msg) + + def _dump_status_message(self): + """Return the dump status message string + """ + msg = indent_string(self.status_output, 12) + printlog(msg) + + def safe_to_update(self): + """Report if it is safe to update a repository. Safe is defined as: + + * If a repository is empty, it is safe to update. + + * If a repository exists and has a clean working copy state + with no pending transactions. + + """ + safe_to_update = False + repo_exists = self.exists() + if not repo_exists: + safe_to_update = True + else: + # If the repo exists, it must be in ok or modified + # sync_state. Any other sync_state at this point + # represents a logic error that should have been handled + # before now! + sync_safe = ((self.sync_state == ExternalStatus.STATUS_OK) or + (self.sync_state == ExternalStatus.MODEL_MODIFIED)) + if sync_safe: + # The clean_state must be STATUS_OK to update. Otherwise we + # are dirty or there was a missed error previously. + if self.clean_state == ExternalStatus.STATUS_OK: + safe_to_update = True + return safe_to_update + + def exists(self): + """Determine if the repo exists. This is indicated by: + + * sync_state is not EMPTY + + * if the sync_state is empty, then the valid states for + clean_state are default, empty or unknown. Anything else + and there was probably an internal logic error. + + NOTE(bja, 2017-10) For the moment we are considering a + sync_state of default or unknown to require user intervention, + but we may want to relax this convention. This is probably a + result of a network error or internal logic error but more + testing is needed. + + """ + is_empty = (self.sync_state == ExternalStatus.EMPTY) + clean_valid = ((self.clean_state == ExternalStatus.DEFAULT) or + (self.clean_state == ExternalStatus.EMPTY) or + (self.clean_state == ExternalStatus.UNKNOWN)) + + if is_empty and clean_valid: + exists = False + else: + exists = True + return exists + + +def check_safe_to_update_repos(tree_status): + """Check if *ALL* repositories are in a safe state to update. We don't + want to do a partial update of the repositories then die, leaving + the model in an inconsistent state. + + Note: if there is an update to do, the repositories will by + definiation be out of synce with the externals description, so we + can't use that as criteria for updating. + + """ + safe_to_update = True + for comp in tree_status: + stat = tree_status[comp] + safe_to_update &= stat.safe_to_update() + + return safe_to_update diff --git a/util/manage_externals/manic/global_constants.py b/util/manage_externals/manic/global_constants.py new file mode 100644 index 0000000000..0e91cffc90 --- /dev/null +++ b/util/manage_externals/manic/global_constants.py @@ -0,0 +1,18 @@ +"""Globals shared across modules +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import pprint + +EMPTY_STR = '' +LOCAL_PATH_INDICATOR = '.' +VERSION_SEPERATOR = '.' +LOG_FILE_NAME = 'manage_externals.log' +PPRINTER = pprint.PrettyPrinter(indent=4) + +VERBOSITY_DEFAULT = 0 +VERBOSITY_VERBOSE = 1 +VERBOSITY_DUMP = 2 diff --git a/util/manage_externals/manic/repository.py b/util/manage_externals/manic/repository.py new file mode 100644 index 0000000000..ea4230fb7b --- /dev/null +++ b/util/manage_externals/manic/repository.py @@ -0,0 +1,98 @@ +"""Base class representation of a repository +""" + +from .externals_description import ExternalsDescription +from .utils import fatal_error +from .global_constants import EMPTY_STR + + +class Repository(object): + """ + Class to represent and operate on a repository description. + """ + + def __init__(self, component_name, repo): + """ + Parse repo externals description + """ + self._name = component_name + self._protocol = repo[ExternalsDescription.PROTOCOL] + self._tag = repo[ExternalsDescription.TAG] + self._branch = repo[ExternalsDescription.BRANCH] + self._hash = repo[ExternalsDescription.HASH] + self._url = repo[ExternalsDescription.REPO_URL] + self._sparse = repo[ExternalsDescription.SPARSE] + + if self._url is EMPTY_STR: + fatal_error('repo must have a URL') + + if ((self._tag is EMPTY_STR) and (self._branch is EMPTY_STR) and + (self._hash is EMPTY_STR)): + fatal_error('{0} repo must have a branch, tag or hash element') + + ref_count = 0 + if self._tag is not EMPTY_STR: + ref_count += 1 + if self._branch is not EMPTY_STR: + ref_count += 1 + if self._hash is not EMPTY_STR: + ref_count += 1 + if ref_count != 1: + fatal_error('repo {0} must have exactly one of ' + 'tag, branch or hash.'.format(self._name)) + + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the source. + If the repo destination directory does not exist, checkout the correce + branch or tag. + NB: is include as an argument for compatibility with + git functionality (repository_git.py) + """ + msg = ('DEV_ERROR: checkout method must be implemented in all ' + 'repository classes! {0}'.format(self.__class__.__name__)) + fatal_error(msg) + + def status(self, stat, repo_dir_path): # pylint: disable=unused-argument + """Report the status of the repo + + """ + msg = ('DEV_ERROR: status method must be implemented in all ' + 'repository classes! {0}'.format(self.__class__.__name__)) + fatal_error(msg) + + def submodules_file(self, repo_path=None): + # pylint: disable=no-self-use,unused-argument + """Stub for use by non-git VC systems""" + return None + + def url(self): + """Public access of repo url. + """ + return self._url + + def tag(self): + """Public access of repo tag + """ + return self._tag + + def branch(self): + """Public access of repo branch. + """ + return self._branch + + def hash(self): + """Public access of repo hash. + """ + return self._hash + + def name(self): + """Public access of repo name. + """ + return self._name + + def protocol(self): + """Public access of repo protocol. + """ + return self._protocol diff --git a/util/manage_externals/manic/repository_factory.py b/util/manage_externals/manic/repository_factory.py new file mode 100644 index 0000000000..80a92a9d8a --- /dev/null +++ b/util/manage_externals/manic/repository_factory.py @@ -0,0 +1,29 @@ +"""Factory for creating and initializing the appropriate repository class +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +from .repository_git import GitRepository +from .repository_svn import SvnRepository +from .externals_description import ExternalsDescription +from .utils import fatal_error + + +def create_repository(component_name, repo_info, svn_ignore_ancestry=False): + """Determine what type of repository we have, i.e. git or svn, and + create the appropriate object. + + """ + protocol = repo_info[ExternalsDescription.PROTOCOL].lower() + if protocol == 'git': + repo = GitRepository(component_name, repo_info) + elif protocol == 'svn': + repo = SvnRepository(component_name, repo_info, ignore_ancestry=svn_ignore_ancestry) + elif protocol == 'externals_only': + repo = None + else: + msg = 'Unknown repo protocol "{0}"'.format(protocol) + fatal_error(msg) + return repo diff --git a/util/manage_externals/manic/repository_git.py b/util/manage_externals/manic/repository_git.py new file mode 100644 index 0000000000..f986051001 --- /dev/null +++ b/util/manage_externals/manic/repository_git.py @@ -0,0 +1,819 @@ +"""Class for interacting with git repositories +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import copy +import os + +from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR +from .global_constants import VERBOSITY_VERBOSE +from .repository import Repository +from .externals_status import ExternalStatus +from .externals_description import ExternalsDescription, git_submodule_status +from .utils import expand_local_url, split_remote_url, is_remote_url +from .utils import fatal_error, printlog +from .utils import execute_subprocess + + +class GitRepository(Repository): + """Class to represent and operate on a repository description. + + For testing purpose, all system calls to git should: + + * be isolated in separate functions with no application logic + * of the form: + - cmd = ['git', ...] + - value = execute_subprocess(cmd, output_to_caller={T|F}, + status_to_caller={T|F}) + - return value + * be static methods (not rely on self) + * name as _git_subcommand_args(user_args) + + This convention allows easy unit testing of the repository logic + by mocking the specific calls to return predefined results. + + """ + + def __init__(self, component_name, repo): + """ + Parse repo (a XML element). + """ + Repository.__init__(self, component_name, repo) + self._gitmodules = None + self._submods = None + + # ---------------------------------------------------------------- + # + # Public API, defined by Repository + # + # ---------------------------------------------------------------- + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the source. + If the repo destination directory does not exist, checkout the correct + branch or tag. + """ + repo_dir_path = os.path.join(base_dir_path, repo_dir_name) + repo_dir_exists = os.path.exists(repo_dir_path) + if (repo_dir_exists and not os.listdir( + repo_dir_path)) or not repo_dir_exists: + self._clone_repo(base_dir_path, repo_dir_name, verbosity) + self._checkout_ref(repo_dir_path, verbosity, recursive) + gmpath = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + if os.path.exists(gmpath): + self._gitmodules = gmpath + self._submods = git_submodule_status(repo_dir_path) + else: + self._gitmodules = None + self._submods = None + + def status(self, stat, repo_dir_path): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the source. + If the repo destination directory does not exist, checkout the correct + branch or tag. + """ + self._check_sync(stat, repo_dir_path) + if os.path.exists(repo_dir_path): + self._status_summary(stat, repo_dir_path) + + def submodules_file(self, repo_path=None): + if repo_path is not None: + gmpath = os.path.join(repo_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + if os.path.exists(gmpath): + self._gitmodules = gmpath + self._submods = git_submodule_status(repo_path) + + return self._gitmodules + + # ---------------------------------------------------------------- + # + # Internal work functions + # + # ---------------------------------------------------------------- + def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): + """Prepare to execute the clone by managing directory location + """ + cwd = os.getcwd() + os.chdir(base_dir_path) + self._git_clone(self._url, repo_dir_name, verbosity) + os.chdir(cwd) + + def _current_ref(self): + """Determine the *name* associated with HEAD. + + If we're on a branch, then returns the branch name; otherwise, + if we're on a tag, then returns the tag name; otherwise, returns + the current hash. Returns an empty string if no reference can be + determined (e.g., if we're not actually in a git repository). + """ + ref_found = False + + # If we're on a branch, then use that as the current ref + branch_found, branch_name = self._git_current_branch() + if branch_found: + current_ref = branch_name + ref_found = True + + if not ref_found: + # Otherwise, if we're exactly at a tag, use that as the + # current ref + tag_found, tag_name = self._git_current_tag() + if tag_found: + current_ref = tag_name + ref_found = True + + if not ref_found: + # Otherwise, use current hash as the current ref + hash_found, hash_name = self._git_current_hash() + if hash_found: + current_ref = hash_name + ref_found = True + + if not ref_found: + # If we still can't find a ref, return empty string. This + # can happen if we're not actually in a git repo + current_ref = '' + + return current_ref + + def _check_sync(self, stat, repo_dir_path): + """Determine whether a git repository is in-sync with the model + description. + + Because repos can have multiple remotes, the only criteria is + whether the branch or tag is the same. + + """ + if not os.path.exists(repo_dir_path): + # NOTE(bja, 2017-10) condition should have been determined + # by _Source() object and should never be here! + stat.sync_state = ExternalStatus.STATUS_ERROR + else: + git_dir = os.path.join(repo_dir_path, '.git') + if not os.path.exists(git_dir): + # NOTE(bja, 2017-10) directory exists, but no git repo + # info.... Can't test with subprocess git command + # because git will move up directory tree until it + # finds the parent repo git dir! + stat.sync_state = ExternalStatus.UNKNOWN + else: + self._check_sync_logic(stat, repo_dir_path) + + def _check_sync_logic(self, stat, repo_dir_path): + """Compare the underlying hashes of the currently checkout ref and the + expected ref. + + Output: sets the sync_state as well as the current and + expected ref in the input status object. + + """ + def compare_refs(current_ref, expected_ref): + """Compare the current and expected ref. + + """ + if current_ref == expected_ref: + status = ExternalStatus.STATUS_OK + else: + status = ExternalStatus.MODEL_MODIFIED + return status + + cwd = os.getcwd() + os.chdir(repo_dir_path) + + # get the full hash of the current commit + _, current_ref = self._git_current_hash() + + if self._branch: + if self._url == LOCAL_PATH_INDICATOR: + expected_ref = self._branch + else: + remote_name = self._determine_remote_name() + if not remote_name: + # git doesn't know about this remote. by definition + # this is a modified state. + expected_ref = "unknown_remote/{0}".format(self._branch) + else: + expected_ref = "{0}/{1}".format(remote_name, self._branch) + elif self._hash: + expected_ref = self._hash + elif self._tag: + expected_ref = self._tag + else: + msg = 'In repo "{0}": none of branch, hash or tag are set'.format( + self._name) + fatal_error(msg) + + # record the *names* of the current and expected branches + stat.current_version = self._current_ref() + stat.expected_version = copy.deepcopy(expected_ref) + + if current_ref == EMPTY_STR: + stat.sync_state = ExternalStatus.UNKNOWN + else: + # get the underlying hash of the expected ref + revparse_status, expected_ref_hash = self._git_revparse_commit( + expected_ref) + if revparse_status: + # We failed to get the hash associated with + # expected_ref. Maybe we should assign this to some special + # status, but for now we're just calling this out-of-sync to + # remain consistent with how this worked before. + stat.sync_state = ExternalStatus.MODEL_MODIFIED + else: + # compare the underlying hashes + stat.sync_state = compare_refs(current_ref, expected_ref_hash) + + os.chdir(cwd) + + def _determine_remote_name(self): + """Return the remote name. + + Note that this is for the *future* repo url and branch, not + the current working copy! + + """ + git_output = self._git_remote_verbose() + git_output = git_output.splitlines() + remote_name = '' + for line in git_output: + data = line.strip() + if not data: + continue + data = data.split() + name = data[0].strip() + url = data[1].strip() + if self._url == url: + remote_name = name + break + return remote_name + + def _create_remote_name(self): + """The url specified in the externals description file was not known + to git. We need to add it, which means adding a unique and + safe name.... + + The assigned name needs to be safe for git to use, e.g. can't + look like a path 'foo/bar' and work with both remote and local paths. + + Remote paths include but are not limited to: git, ssh, https, + github, gitlab, bitbucket, custom server, etc. + + Local paths can be relative or absolute. They may contain + shell variables, e.g. ${REPO_ROOT}/repo_name, or username + expansion, i.e. ~/ or ~someuser/. + + Relative paths must be at least one layer of redirection, i.e. + container/../ext_repo, but may be many layers deep, e.g. + container/../../../../../ext_repo + + NOTE(bja, 2017-11) + + The base name below may not be unique, for example if the + user has local paths like: + + /path/to/my/repos/nice_repo + /path/to/other/repos/nice_repo + + But the current implementation should cover most common + use cases for remotes and still provide usable names. + + """ + url = copy.deepcopy(self._url) + if is_remote_url(url): + url = split_remote_url(url) + else: + url = expand_local_url(url, self._name) + url = url.split('/') + repo_name = url[-1] + base_name = url[-2] + # repo name should nominally already be something that git can + # deal with. We need to remove other possibly troublesome + # punctuation, e.g. /, $, from the base name. + unsafe_characters = '!@#$%^&*()[]{}\\/,;~' + for unsafe in unsafe_characters: + base_name = base_name.replace(unsafe, '') + remote_name = "{0}_{1}".format(base_name, repo_name) + return remote_name + + def _checkout_ref(self, repo_dir, verbosity, submodules): + """Checkout the user supplied reference + if is True, recursively initialize and update + the repo's submodules + """ + # import pdb; pdb.set_trace() + cwd = os.getcwd() + os.chdir(repo_dir) + if self._url.strip() == LOCAL_PATH_INDICATOR: + self._checkout_local_ref(verbosity, submodules) + else: + self._checkout_external_ref(verbosity, submodules) + + if self._sparse: + self._sparse_checkout(repo_dir, verbosity) + os.chdir(cwd) + + + def _checkout_local_ref(self, verbosity, submodules): + """Checkout the reference considering the local repo only. Do not + fetch any additional remotes or specify the remote when + checkout out the ref. + if is True, recursively initialize and update + the repo's submodules + """ + if self._tag: + ref = self._tag + elif self._branch: + ref = self._branch + else: + ref = self._hash + + self._check_for_valid_ref(ref) + self._git_checkout_ref(ref, verbosity, submodules) + + def _checkout_external_ref(self, verbosity, submodules): + """Checkout the reference from a remote repository + if is True, recursively initialize and update + the repo's submodules + """ + if self._tag: + ref = self._tag + elif self._branch: + ref = self._branch + else: + ref = self._hash + + remote_name = self._determine_remote_name() + if not remote_name: + remote_name = self._create_remote_name() + self._git_remote_add(remote_name, self._url) + self._git_fetch(remote_name) + + # NOTE(bja, 2018-03) we need to send separate ref and remote + # name to check_for_vaild_ref, but the combined name to + # checkout_ref! + self._check_for_valid_ref(ref, remote_name) + + if self._branch: + ref = '{0}/{1}'.format(remote_name, ref) + self._git_checkout_ref(ref, verbosity, submodules) + + def _sparse_checkout(self, repo_dir, verbosity): + """Use git read-tree to thin the working tree.""" + cwd = os.getcwd() + + cmd = ['cp', self._sparse, os.path.join(repo_dir, + '.git/info/sparse-checkout')] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + os.chdir(repo_dir) + self._git_sparse_checkout(verbosity) + + os.chdir(cwd) + + def _check_for_valid_ref(self, ref, remote_name=None): + """Try some basic sanity checks on the user supplied reference so we + can provide a more useful error message than calledprocess + error... + + """ + is_tag = self._ref_is_tag(ref) + is_branch = self._ref_is_branch(ref, remote_name) + is_hash = self._ref_is_hash(ref) + + is_valid = is_tag or is_branch or is_hash + if not is_valid: + msg = ('In repo "{0}": reference "{1}" does not appear to be a ' + 'valid tag, branch or hash! Please verify the reference ' + 'name (e.g. spelling), is available from: {2} '.format( + self._name, ref, self._url)) + fatal_error(msg) + + if is_tag: + is_unique_tag, msg = self._is_unique_tag(ref, remote_name) + if not is_unique_tag: + msg = ('In repo "{0}": tag "{1}" {2}'.format( + self._name, self._tag, msg)) + fatal_error(msg) + + return is_valid + + def _is_unique_tag(self, ref, remote_name): + """Verify that a reference is a valid tag and is unique (not a branch) + + Tags may be tag names, or SHA id's. It is also possible that a + branch and tag have the some name. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + + """ + is_tag = self._ref_is_tag(ref) + is_branch = self._ref_is_branch(ref, remote_name) + is_hash = self._ref_is_hash(ref) + + msg = '' + is_unique_tag = False + if is_tag and not is_branch: + # unique tag + msg = 'is ok' + is_unique_tag = True + elif is_tag and is_branch: + msg = ('is both a branch and a tag. git may checkout the branch ' + 'instead of the tag depending on your version of git.') + is_unique_tag = False + elif not is_tag and is_branch: + msg = ('is a branch, and not a tag. If you intended to checkout ' + 'a branch, please change the externals description to be ' + 'a branch. If you intended to checkout a tag, it does not ' + 'exist. Please check the name.') + is_unique_tag = False + else: # not is_tag and not is_branch: + if is_hash: + # probably a sha1 or HEAD, etc, we call it a tag + msg = 'is ok' + is_unique_tag = True + else: + # undetermined state. + msg = ('does not appear to be a valid tag, branch or hash! ' + 'Please check the name and repository.') + is_unique_tag = False + + return is_unique_tag, msg + + def _ref_is_tag(self, ref): + """Verify that a reference is a valid tag according to git. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + """ + is_tag = False + value = self._git_showref_tag(ref) + if value == 0: + is_tag = True + return is_tag + + def _ref_is_branch(self, ref, remote_name=None): + """Verify if a ref is any kind of branch (local, tracked remote, + untracked remote). + + """ + local_branch = False + remote_branch = False + if remote_name: + remote_branch = self._ref_is_remote_branch(ref, remote_name) + local_branch = self._ref_is_local_branch(ref) + + is_branch = False + if local_branch or remote_branch: + is_branch = True + return is_branch + + def _ref_is_local_branch(self, ref): + """Verify that a reference is a valid branch according to git. + + show-ref branch returns local branches that have been + previously checked out. It will not necessarily pick up + untracked remote branches. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + + """ + is_branch = False + value = self._git_showref_branch(ref) + if value == 0: + is_branch = True + return is_branch + + def _ref_is_remote_branch(self, ref, remote_name): + """Verify that a reference is a valid branch according to git. + + show-ref branch returns local branches that have been + previously checked out. It will not necessarily pick up + untracked remote branches. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + + """ + is_branch = False + value = self._git_lsremote_branch(ref, remote_name) + if value == 0: + is_branch = True + return is_branch + + def _ref_is_commit(self, ref): + """Verify that a reference is a valid commit according to git. + + This could be a tag, branch, sha1 id, HEAD and potentially others... + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + """ + is_commit = False + value, _ = self._git_revparse_commit(ref) + if value == 0: + is_commit = True + return is_commit + + def _ref_is_hash(self, ref): + """Verify that a reference is a valid hash according to git. + + Git doesn't seem to provide an exact way to determine if user + supplied reference is an actual hash. So we verify that the + ref is a valid commit and return the underlying commit + hash. Then check that the commit hash begins with the user + supplied string. + + Note: values returned by git_showref_* and git_revparse are + shell return codes, which are zero for success, non-zero for + error! + + """ + is_hash = False + status, git_output = self._git_revparse_commit(ref) + if status == 0: + if git_output.strip().startswith(ref): + is_hash = True + return is_hash + + def _status_summary(self, stat, repo_dir_path): + """Determine the clean/dirty status of a git repository + + """ + cwd = os.getcwd() + os.chdir(repo_dir_path) + git_output = self._git_status_porcelain_v1z() + is_dirty = self._status_v1z_is_dirty(git_output) + if is_dirty: + stat.clean_state = ExternalStatus.DIRTY + else: + stat.clean_state = ExternalStatus.STATUS_OK + + # Now save the verbose status output incase the user wants to + # see it. + stat.status_output = self._git_status_verbose() + os.chdir(cwd) + + @staticmethod + def _status_v1z_is_dirty(git_output): + """Parse the git status output from --porcelain=v1 -z and determine if + the repo status is clean or dirty. Dirty means: + + * modified files + * missing files + * added files + * removed + * renamed + * unmerged + + Whether untracked files are considered depends on how the status + command was run (i.e., whether it was run with the '-u' option). + + NOTE: Based on the above definition, the porcelain status + should be an empty string to be considered 'clean'. Of course + this assumes we only get an empty string from an status + command on a clean checkout, and not some error + condition... Could alse use 'git diff --quiet'. + + """ + is_dirty = False + if git_output: + is_dirty = True + return is_dirty + + # ---------------------------------------------------------------- + # + # system call to git for information gathering + # + # ---------------------------------------------------------------- + @staticmethod + def _git_current_hash(): + """Return the full hash of the currently checked-out version. + + Returns a tuple, (hash_found, hash), where hash_found is a + logical specifying whether a hash was found for HEAD (False + could mean we're not in a git repository at all). (If hash_found + is False, then hash is ''.) + """ + status, git_output = GitRepository._git_revparse_commit("HEAD") + hash_found = not status + if not hash_found: + git_output = '' + return hash_found, git_output + + @staticmethod + def _git_current_branch(): + """Determines the name of the current branch. + + Returns a tuple, (branch_found, branch_name), where branch_found + is a logical specifying whether a branch name was found for + HEAD. (If branch_found is False, then branch_name is ''.) + """ + cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + branch_found = not status + if branch_found: + git_output = git_output.strip() + else: + git_output = '' + return branch_found, git_output + + @staticmethod + def _git_current_tag(): + """Determines the name tag corresponding to HEAD (if any). + + Returns a tuple, (tag_found, tag_name), where tag_found is a + logical specifying whether we found a tag name corresponding to + HEAD. (If tag_found is False, then tag_name is ''.) + """ + # git describe --exact-match --tags HEAD + cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + tag_found = not status + if tag_found: + git_output = git_output.strip() + else: + git_output = '' + return tag_found, git_output + + @staticmethod + def _git_showref_tag(ref): + """Run git show-ref check if the user supplied ref is a tag. + + could also use git rev-parse --quiet --verify tagname^{tag} + """ + cmd = ['git', 'show-ref', '--quiet', '--verify', + 'refs/tags/{0}'.format(ref), ] + status = execute_subprocess(cmd, status_to_caller=True) + return status + + @staticmethod + def _git_showref_branch(ref): + """Run git show-ref check if the user supplied ref is a local or + tracked remote branch. + + """ + cmd = ['git', 'show-ref', '--quiet', '--verify', + 'refs/heads/{0}'.format(ref), ] + status = execute_subprocess(cmd, status_to_caller=True) + return status + + @staticmethod + def _git_lsremote_branch(ref, remote_name): + """Run git ls-remote to check if the user supplied ref is a remote + branch that is not being tracked + + """ + cmd = ['git', 'ls-remote', '--exit-code', '--heads', + remote_name, ref, ] + status = execute_subprocess(cmd, status_to_caller=True) + return status + + @staticmethod + def _git_revparse_commit(ref): + """Run git rev-parse to detect if a reference is a SHA, HEAD or other + valid commit. + + """ + cmd = ['git', 'rev-parse', '--quiet', '--verify', + '{0}^{1}'.format(ref, '{commit}'), ] + status, git_output = execute_subprocess(cmd, status_to_caller=True, + output_to_caller=True) + git_output = git_output.strip() + return status, git_output + + @staticmethod + def _git_status_porcelain_v1z(): + """Run git status to obtain repository information. + + This is run with '--untracked=no' to ignore untracked files. + + The machine-portable format that is guaranteed not to change + between git versions or *user configuration*. + + """ + cmd = ['git', 'status', '--untracked-files=no', '--porcelain', '-z'] + git_output = execute_subprocess(cmd, output_to_caller=True) + return git_output + + @staticmethod + def _git_status_verbose(): + """Run the git status command to obtain repository information. + """ + cmd = ['git', 'status'] + git_output = execute_subprocess(cmd, output_to_caller=True) + return git_output + + @staticmethod + def _git_remote_verbose(): + """Run the git remote command to obtain repository information. + """ + cmd = ['git', 'remote', '--verbose'] + git_output = execute_subprocess(cmd, output_to_caller=True) + return git_output + + @staticmethod + def has_submodules(repo_dir_path=None): + """Return True iff the repository at (or the current + directory if is None) has a '.gitmodules' file + """ + if repo_dir_path is None: + fname = ExternalsDescription.GIT_SUBMODULES_FILENAME + else: + fname = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + + return os.path.exists(fname) + + # ---------------------------------------------------------------- + # + # system call to git for sideffects modifying the working tree + # + # ---------------------------------------------------------------- + @staticmethod + def _git_clone(url, repo_dir_name, verbosity): + """Run git clone for the side effect of creating a repository. + """ + cmd = ['git', 'clone', '--quiet'] + subcmd = None + + cmd.extend([url, repo_dir_name]) + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + if subcmd is not None: + os.chdir(repo_dir_name) + execute_subprocess(subcmd) + + @staticmethod + def _git_remote_add(name, url): + """Run the git remote command for the side effect of adding a remote + """ + cmd = ['git', 'remote', 'add', name, url] + execute_subprocess(cmd) + + @staticmethod + def _git_fetch(remote_name): + """Run the git fetch command for the side effect of updating the repo + """ + cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] + execute_subprocess(cmd) + + @staticmethod + def _git_checkout_ref(ref, verbosity, submodules): + """Run the git checkout command for the side effect of updating the repo + + Param: ref is a reference to a local or remote object in the + form 'origin/my_feature', or 'tag1'. + + """ + cmd = ['git', 'checkout', '--quiet', ref] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + if submodules: + GitRepository._git_update_submodules(verbosity) + + @staticmethod + def _git_sparse_checkout(verbosity): + """Configure repo via read-tree.""" + cmd = ['git', 'config', 'core.sparsecheckout', 'true'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + cmd = ['git', 'read-tree', '-mu', 'HEAD'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + + @staticmethod + def _git_update_submodules(verbosity): + """Run git submodule update for the side effect of updating this + repo's submodules. + """ + # First, verify that we have a .gitmodules file + if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): + cmd = ['git', 'submodule', 'update', '--init', '--recursive'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + + execute_subprocess(cmd) diff --git a/util/manage_externals/manic/repository_svn.py b/util/manage_externals/manic/repository_svn.py new file mode 100644 index 0000000000..2f0d4d848c --- /dev/null +++ b/util/manage_externals/manic/repository_svn.py @@ -0,0 +1,284 @@ +"""Class for interacting with svn repositories +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import os +import re +import xml.etree.ElementTree as ET + +from .global_constants import EMPTY_STR, VERBOSITY_VERBOSE +from .repository import Repository +from .externals_status import ExternalStatus +from .utils import fatal_error, indent_string, printlog +from .utils import execute_subprocess + + +class SvnRepository(Repository): + """ + Class to represent and operate on a repository description. + + For testing purpose, all system calls to svn should: + + * be isolated in separate functions with no application logic + * of the form: + - cmd = ['svn', ...] + - value = execute_subprocess(cmd, output_to_caller={T|F}, + status_to_caller={T|F}) + - return value + * be static methods (not rely on self) + * name as _svn_subcommand_args(user_args) + + This convention allows easy unit testing of the repository logic + by mocking the specific calls to return predefined results. + + """ + RE_URLLINE = re.compile(r'^URL:') + + def __init__(self, component_name, repo, ignore_ancestry=False): + """ + Parse repo (a XML element). + """ + Repository.__init__(self, component_name, repo) + self._ignore_ancestry = ignore_ancestry + if self._branch: + self._url = os.path.join(self._url, self._branch) + elif self._tag: + self._url = os.path.join(self._url, self._tag) + else: + msg = "DEV_ERROR in svn repository. Shouldn't be here!" + fatal_error(msg) + + # ---------------------------------------------------------------- + # + # Public API, defined by Repository + # + # ---------------------------------------------------------------- + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument + """Checkout or update the working copy + + If the repo destination directory exists, switch the sandbox to + match the externals description. + + If the repo destination directory does not exist, checkout the + correct branch or tag. + NB: is include as an argument for compatibility with + git functionality (repository_git.py) + + """ + repo_dir_path = os.path.join(base_dir_path, repo_dir_name) + if os.path.exists(repo_dir_path): + cwd = os.getcwd() + os.chdir(repo_dir_path) + self._svn_switch(self._url, self._ignore_ancestry, verbosity) + # svn switch can lead to a conflict state, but it gives a + # return code of 0. So now we need to make sure that we're + # in a clean (non-conflict) state. + self._abort_if_dirty(repo_dir_path, + "Expected clean state following switch") + os.chdir(cwd) + else: + self._svn_checkout(self._url, repo_dir_path, verbosity) + + def status(self, stat, repo_dir_path): + """ + Check and report the status of the repository + """ + self._check_sync(stat, repo_dir_path) + if os.path.exists(repo_dir_path): + self._status_summary(stat, repo_dir_path) + + # ---------------------------------------------------------------- + # + # Internal work functions + # + # ---------------------------------------------------------------- + def _check_sync(self, stat, repo_dir_path): + """Check to see if repository directory exists and is at the expected + url. Return: status object + + """ + if not os.path.exists(repo_dir_path): + # NOTE(bja, 2017-10) this state should have been handled by + # the source object and we never get here! + stat.sync_state = ExternalStatus.STATUS_ERROR + else: + svn_output = self._svn_info(repo_dir_path) + if not svn_output: + # directory exists, but info returned nothing. .svn + # directory removed or incomplete checkout? + stat.sync_state = ExternalStatus.UNKNOWN + else: + stat.sync_state, stat.current_version = \ + self._check_url(svn_output, self._url) + stat.expected_version = '/'.join(self._url.split('/')[3:]) + + def _abort_if_dirty(self, repo_dir_path, message): + """Check if the repo is in a dirty state; if so, abort with a + helpful message. + + """ + + stat = ExternalStatus() + self._status_summary(stat, repo_dir_path) + if stat.clean_state != ExternalStatus.STATUS_OK: + status = self._svn_status_verbose(repo_dir_path) + status = indent_string(status, 4) + errmsg = """In directory + {cwd} + +svn status now shows: +{status} + +ERROR: {message} + +One possible cause of this problem is that there may have been untracked +files in your working directory that had the same name as tracked files +in the new revision. + +To recover: Clean up the above directory (resolving conflicts, etc.), +then rerun checkout_externals. +""".format(cwd=repo_dir_path, message=message, status=status) + + fatal_error(errmsg) + + @staticmethod + def _check_url(svn_output, expected_url): + """Determine the svn url from svn info output and return whether it + matches the expected value. + + """ + url = None + for line in svn_output.splitlines(): + if SvnRepository.RE_URLLINE.match(line): + url = line.split(': ')[1].strip() + break + if not url: + status = ExternalStatus.UNKNOWN + elif url == expected_url: + status = ExternalStatus.STATUS_OK + else: + status = ExternalStatus.MODEL_MODIFIED + + if url: + current_version = '/'.join(url.split('/')[3:]) + else: + current_version = EMPTY_STR + + return status, current_version + + def _status_summary(self, stat, repo_dir_path): + """Report whether the svn repository is in-sync with the model + description and whether the sandbox is clean or dirty. + + """ + svn_output = self._svn_status_xml(repo_dir_path) + is_dirty = self.xml_status_is_dirty(svn_output) + if is_dirty: + stat.clean_state = ExternalStatus.DIRTY + else: + stat.clean_state = ExternalStatus.STATUS_OK + + # Now save the verbose status output incase the user wants to + # see it. + stat.status_output = self._svn_status_verbose(repo_dir_path) + + @staticmethod + def xml_status_is_dirty(svn_output): + """Parse svn status xml output and determine if the working copy is + clean or dirty. Dirty is defined as: + + * modified files + * added files + * deleted files + * missing files + + Unversioned files do not affect the clean/dirty status. + + 'external' is also an acceptable state + + """ + # pylint: disable=invalid-name + SVN_EXTERNAL = 'external' + SVN_UNVERSIONED = 'unversioned' + # pylint: enable=invalid-name + + is_dirty = False + try: + xml_status = ET.fromstring(svn_output) + except BaseException: + fatal_error( + "SVN returned invalid XML message {}".format(svn_output)) + xml_target = xml_status.find('./target') + entries = xml_target.findall('./entry') + for entry in entries: + status = entry.find('./wc-status') + item = status.get('item') + if item == SVN_EXTERNAL: + continue + if item == SVN_UNVERSIONED: + continue + else: + is_dirty = True + break + return is_dirty + + # ---------------------------------------------------------------- + # + # system call to svn for information gathering + # + # ---------------------------------------------------------------- + @staticmethod + def _svn_info(repo_dir_path): + """Return results of svn info command + """ + cmd = ['svn', 'info', repo_dir_path] + output = execute_subprocess(cmd, output_to_caller=True) + return output + + @staticmethod + def _svn_status_verbose(repo_dir_path): + """capture the full svn status output + """ + cmd = ['svn', 'status', repo_dir_path] + svn_output = execute_subprocess(cmd, output_to_caller=True) + return svn_output + + @staticmethod + def _svn_status_xml(repo_dir_path): + """ + Get status of the subversion sandbox in repo_dir + """ + cmd = ['svn', 'status', '--xml', repo_dir_path] + svn_output = execute_subprocess(cmd, output_to_caller=True) + return svn_output + + # ---------------------------------------------------------------- + # + # system call to svn for sideffects modifying the working tree + # + # ---------------------------------------------------------------- + @staticmethod + def _svn_checkout(url, repo_dir_path, verbosity): + """ + Checkout a subversion repository (repo_url) to checkout_dir. + """ + cmd = ['svn', 'checkout', '--quiet', url, repo_dir_path] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + + @staticmethod + def _svn_switch(url, ignore_ancestry, verbosity): + """ + Switch branches for in an svn sandbox + """ + cmd = ['svn', 'switch', '--quiet'] + if ignore_ancestry: + cmd.append('--ignore-ancestry') + cmd.append(url) + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) diff --git a/util/manage_externals/manic/sourcetree.py b/util/manage_externals/manic/sourcetree.py new file mode 100644 index 0000000000..3a63835c78 --- /dev/null +++ b/util/manage_externals/manic/sourcetree.py @@ -0,0 +1,351 @@ +""" + +FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! +""" + +import errno +import logging +import os + +from .externals_description import ExternalsDescription +from .externals_description import read_externals_description_file +from .externals_description import create_externals_description +from .repository_factory import create_repository +from .repository_git import GitRepository +from .externals_status import ExternalStatus +from .utils import fatal_error, printlog +from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR +from .global_constants import VERBOSITY_VERBOSE + +class _External(object): + """ + _External represents an external object inside a SourceTree + """ + + # pylint: disable=R0902 + + def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): + """Parse an external description file into a dictionary of externals. + + Input: + + root_dir : string - the root directory path where + 'local_path' is relative to. + + name : string - name of the ext_description object. may or may not + correspond to something in the path. + + ext_description : dict - source ExternalsDescription object + + svn_ignore_ancestry : bool - use --ignore-externals with svn switch + + """ + self._name = name + self._repo = None + self._externals = EMPTY_STR + self._externals_sourcetree = None + self._stat = ExternalStatus() + self._sparse = None + # Parse the sub-elements + + # _path : local path relative to the containing source tree + self._local_path = ext_description[ExternalsDescription.PATH] + # _repo_dir : full repository directory + repo_dir = os.path.join(root_dir, self._local_path) + self._repo_dir_path = os.path.abspath(repo_dir) + # _base_dir : base directory *containing* the repository + self._base_dir_path = os.path.dirname(self._repo_dir_path) + # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + self._repo_dir_name = os.path.basename(self._repo_dir_path) + assert(os.path.join(self._base_dir_path, self._repo_dir_name) + == self._repo_dir_path) + + self._required = ext_description[ExternalsDescription.REQUIRED] + self._externals = ext_description[ExternalsDescription.EXTERNALS] + # Treat a .gitmodules file as a backup externals config + if not self._externals: + if GitRepository.has_submodules(self._repo_dir_path): + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + + repo = create_repository( + name, ext_description[ExternalsDescription.REPO], + svn_ignore_ancestry=svn_ignore_ancestry) + if repo: + self._repo = repo + + if self._externals and (self._externals.lower() != 'none'): + self._create_externals_sourcetree() + + def get_name(self): + """ + Return the external object's name + """ + return self._name + + def get_local_path(self): + """ + Return the external object's path + """ + return self._local_path + + def status(self): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the external. + If the repo destination directory does not exist, checkout the correce + branch or tag. + If load_all is True, also load all of the the externals sub-externals. + """ + + self._stat.path = self.get_local_path() + if not self._required: + self._stat.source_type = ExternalStatus.OPTIONAL + elif self._local_path == LOCAL_PATH_INDICATOR: + # LOCAL_PATH_INDICATOR, '.' paths, are standalone + # component directories that are not managed by + # checkout_externals. + self._stat.source_type = ExternalStatus.STANDALONE + else: + # managed by checkout_externals + self._stat.source_type = ExternalStatus.MANAGED + + ext_stats = {} + + if not os.path.exists(self._repo_dir_path): + self._stat.sync_state = ExternalStatus.EMPTY + msg = ('status check: repository directory for "{0}" does not ' + 'exist.'.format(self._name)) + logging.info(msg) + self._stat.current_version = 'not checked out' + # NOTE(bja, 2018-01) directory doesn't exist, so we cannot + # use repo to determine the expected version. We just take + # a best-guess based on the assumption that only tag or + # branch should be set, but not both. + if not self._repo: + self._stat.expected_version = 'unknown' + else: + self._stat.expected_version = self._repo.tag() + self._repo.branch() + else: + if self._repo: + self._repo.status(self._stat, self._repo_dir_path) + + if self._externals and self._externals_sourcetree: + # we expect externals and they exist + cwd = os.getcwd() + # SourceTree expects to be called from the correct + # root directory. + os.chdir(self._repo_dir_path) + ext_stats = self._externals_sourcetree.status(self._local_path) + os.chdir(cwd) + + all_stats = {} + # don't add the root component because we don't manage it + # and can't provide useful info about it. + if self._local_path != LOCAL_PATH_INDICATOR: + # store the stats under tha local_path, not comp name so + # it will be sorted correctly + all_stats[self._stat.path] = self._stat + + if ext_stats: + all_stats.update(ext_stats) + + return all_stats + + def checkout(self, verbosity, load_all): + """ + If the repo destination directory exists, ensure it is correct (from + correct URL, correct branch or tag), and possibly update the external. + If the repo destination directory does not exist, checkout the correct + branch or tag. + If load_all is True, also load all of the the externals sub-externals. + """ + if load_all: + pass + # Make sure we are in correct location + + if not os.path.exists(self._repo_dir_path): + # repository directory doesn't exist. Need to check it + # out, and for that we need the base_dir_path to exist + try: + os.makedirs(self._base_dir_path) + except OSError as error: + if error.errno != errno.EEXIST: + msg = 'Could not create directory "{0}"'.format( + self._base_dir_path) + fatal_error(msg) + + if self._stat.source_type != ExternalStatus.STANDALONE: + if verbosity >= VERBOSITY_VERBOSE: + # NOTE(bja, 2018-01) probably do not want to pass + # verbosity in this case, because if (verbosity == + # VERBOSITY_DUMP), then the previous status output would + # also be dumped, adding noise to the output. + self._stat.log_status_message(VERBOSITY_VERBOSE) + + if self._repo: + if self._stat.sync_state == ExternalStatus.STATUS_OK: + # If we're already in sync, avoid showing verbose output + # from the checkout command, unless the verbosity level + # is 2 or more. + checkout_verbosity = verbosity - 1 + else: + checkout_verbosity = verbosity + + self._repo.checkout(self._base_dir_path, self._repo_dir_name, + checkout_verbosity, self.clone_recursive()) + + def checkout_externals(self, verbosity, load_all): + """Checkout the sub-externals for this object + """ + if self.load_externals(): + if self._externals_sourcetree: + # NOTE(bja, 2018-02): the subtree externals objects + # were created during initial status check. Updating + # the external may have changed which sub-externals + # are needed. We need to delete those objects and + # re-read the potentially modified externals + # description file. + self._externals_sourcetree = None + self._create_externals_sourcetree() + self._externals_sourcetree.checkout(verbosity, load_all) + + def load_externals(self): + 'Return True iff an externals file should be loaded' + load_ex = False + if os.path.exists(self._repo_dir_path): + if self._externals: + if self._externals.lower() != 'none': + load_ex = os.path.exists(os.path.join(self._repo_dir_path, + self._externals)) + + return load_ex + + def clone_recursive(self): + 'Return True iff any .gitmodules files should be processed' + # Try recursive unless there is an externals entry + recursive = not self._externals + + return recursive + + def _create_externals_sourcetree(self): + """ + """ + if not os.path.exists(self._repo_dir_path): + # NOTE(bja, 2017-10) repository has not been checked out + # yet, can't process the externals file. Assume we are + # checking status before code is checkoud out and this + # will be handled correctly later. + return + + cwd = os.getcwd() + os.chdir(self._repo_dir_path) + if self._externals.lower() == 'none': + msg = ('Internal: Attempt to create source tree for ' + 'externals = none in {}'.format(self._repo_dir_path)) + fatal_error(msg) + + if not os.path.exists(self._externals): + if GitRepository.has_submodules(): + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + + if not os.path.exists(self._externals): + # NOTE(bja, 2017-10) this check is redundent with the one + # in read_externals_description_file! + msg = ('External externals description file "{0}" ' + 'does not exist! In directory: {1}'.format( + self._externals, self._repo_dir_path)) + fatal_error(msg) + + externals_root = self._repo_dir_path + model_data = read_externals_description_file(externals_root, + self._externals) + externals = create_externals_description(model_data, + parent_repo=self._repo) + self._externals_sourcetree = SourceTree(externals_root, externals) + os.chdir(cwd) + +class SourceTree(object): + """ + SourceTree represents a group of managed externals + """ + + def __init__(self, root_dir, model, svn_ignore_ancestry=False): + """ + Build a SourceTree object from a model description + """ + self._root_dir = os.path.abspath(root_dir) + self._all_components = {} + self._required_compnames = [] + for comp in model: + src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) + self._all_components[comp] = src + if model[comp][ExternalsDescription.REQUIRED]: + self._required_compnames.append(comp) + + def status(self, relative_path_base=LOCAL_PATH_INDICATOR): + """Report the status components + + FIXME(bja, 2017-10) what do we do about situations where the + user checked out the optional components, but didn't add + optional for running status? What do we do where the user + didn't add optional to the checkout but did add it to the + status. -- For now, we run status on all components, and try + to do the right thing based on the results.... + + """ + load_comps = self._all_components.keys() + + summary = {} + for comp in load_comps: + printlog('{0}, '.format(comp), end='') + stat = self._all_components[comp].status() + for name in stat.keys(): + # check if we need to append the relative_path_base to + # the path so it will be sorted in the correct order. + if not stat[name].path.startswith(relative_path_base): + stat[name].path = os.path.join(relative_path_base, + stat[name].path) + # store under key = updated path, and delete the + # old key. + comp_stat = stat[name] + del stat[name] + stat[comp_stat.path] = comp_stat + summary.update(stat) + + return summary + + def checkout(self, verbosity, load_all, load_comp=None): + """ + Checkout or update indicated components into the the configured + subdirs. + + If load_all is True, recursively checkout all externals. + If load_all is False, load_comp is an optional set of components to load. + If load_all is True and load_comp is None, only load the required externals. + """ + if verbosity >= VERBOSITY_VERBOSE: + printlog('Checking out externals: ') + else: + printlog('Checking out externals: ', end='') + + if load_all: + load_comps = self._all_components.keys() + elif load_comp is not None: + load_comps = [load_comp] + else: + load_comps = self._required_compnames + + # checkout the primary externals + for comp in load_comps: + if verbosity < VERBOSITY_VERBOSE: + printlog('{0}, '.format(comp), end='') + else: + # verbose output handled by the _External object, just + # output a newline + printlog(EMPTY_STR) + self._all_components[comp].checkout(verbosity, load_all) + printlog('') + + # now give each external an opportunitity to checkout it's externals. + for comp in load_comps: + self._all_components[comp].checkout_externals(verbosity, load_all) diff --git a/util/manage_externals/manic/utils.py b/util/manage_externals/manic/utils.py new file mode 100644 index 0000000000..f57f43930c --- /dev/null +++ b/util/manage_externals/manic/utils.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python +""" +Common public utilities for manic package + +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import logging +import os +import subprocess +import sys +from threading import Timer + +from .global_constants import LOCAL_PATH_INDICATOR + +# --------------------------------------------------------------------- +# +# screen and logging output and functions to massage text for output +# +# --------------------------------------------------------------------- + + +def log_process_output(output): + """Log each line of process output at debug level so it can be + filtered if necessary. By default, output is a single string, and + logging.debug(output) will only put log info heading on the first + line. This makes it hard to filter with grep. + + """ + output = output.split('\n') + for line in output: + logging.debug(line) + + +def printlog(msg, **kwargs): + """Wrapper script around print to ensure that everything printed to + the screen also gets logged. + + """ + logging.info(msg) + if kwargs: + print(msg, **kwargs) + else: + print(msg) + sys.stdout.flush() + + +def last_n_lines(the_string, n_lines, truncation_message=None): + """Returns the last n lines of the given string + + Args: + the_string: str + n_lines: int + truncation_message: str, optional + + Returns a string containing the last n lines of the_string + + If truncation_message is provided, the returned string begins with + the given message if and only if the string is greater than n lines + to begin with. + """ + + lines = the_string.splitlines(True) + if len(lines) <= n_lines: + return_val = the_string + else: + lines_subset = lines[-n_lines:] + str_truncated = ''.join(lines_subset) + if truncation_message: + str_truncated = truncation_message + '\n' + str_truncated + return_val = str_truncated + + return return_val + + +def indent_string(the_string, indent_level): + """Indents the given string by a given number of spaces + + Args: + the_string: str + indent_level: int + + Returns a new string that is the same as the_string, except that + each line is indented by 'indent_level' spaces. + + In python3, this can be done with textwrap.indent. + """ + + lines = the_string.splitlines(True) + padding = ' ' * indent_level + lines_indented = [padding + line for line in lines] + return ''.join(lines_indented) + +# --------------------------------------------------------------------- +# +# error handling +# +# --------------------------------------------------------------------- + + +def fatal_error(message): + """ + Error output function + """ + logging.error(message) + raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) + + +# --------------------------------------------------------------------- +# +# Data conversion / manipulation +# +# --------------------------------------------------------------------- +def str_to_bool(bool_str): + """Convert a sting representation of as boolean into a true boolean. + + Conversion should be case insensitive. + """ + value = None + str_lower = bool_str.lower() + if str_lower in ('true', 't'): + value = True + elif str_lower in ('false', 'f'): + value = False + if value is None: + msg = ('ERROR: invalid boolean string value "{0}". ' + 'Must be "true" or "false"'.format(bool_str)) + fatal_error(msg) + return value + + +REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@'] + + +def is_remote_url(url): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + """ + remote_url = False + for prefix in REMOTE_PREFIXES: + if url.startswith(prefix): + remote_url = True + return remote_url + + +def split_remote_url(url): + """check if the user provided a local file path or a + remote. If remote, try to strip off protocol info. + + """ + remote_url = is_remote_url(url) + if not remote_url: + return url + + for prefix in REMOTE_PREFIXES: + url = url.replace(prefix, '') + + if '@' in url: + url = url.split('@')[1] + + if ':' in url: + url = url.split(':')[1] + + return url + + +def expand_local_url(url, field): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + Note: local paths of LOCAL_PATH_INDICATOR have special meaning and + represent local copy only, don't work with the remotes. + + """ + remote_url = is_remote_url(url) + if not remote_url: + if url.strip() == LOCAL_PATH_INDICATOR: + pass + else: + url = os.path.expandvars(url) + url = os.path.expanduser(url) + if not os.path.isabs(url): + msg = ('WARNING: Externals description for "{0}" contains a ' + 'url that is not remote and does not expand to an ' + 'absolute path. Version control operations may ' + 'fail.\n\nurl={1}'.format(field, url)) + printlog(msg) + else: + url = os.path.normpath(url) + return url + + +# --------------------------------------------------------------------- +# +# subprocess +# +# --------------------------------------------------------------------- + +# Give the user a helpful message if we detect that a command seems to +# be hanging. +_HANGING_SEC = 300 + + +def _hanging_msg(working_directory, command): + print(""" + +Command '{command}' +from directory {working_directory} +has taken {hanging_sec} seconds. It may be hanging. + +The command will continue to run, but you may want to abort +manage_externals with ^C and investigate. A possible cause of hangs is +when svn or git require authentication to access a private +repository. On some systems, svn and git requests for authentication +information will not be displayed to the user. In this case, the program +will appear to hang. Ensure you can run svn and git manually and access +all repositories without entering your authentication information. + +""".format(command=command, + working_directory=working_directory, + hanging_sec=_HANGING_SEC)) + + +def execute_subprocess(commands, status_to_caller=False, + output_to_caller=False): + """Wrapper around subprocess.check_output to handle common + exceptions. + + check_output runs a command with arguments and waits + for it to complete. + + check_output raises an exception on a nonzero return code. if + status_to_caller is true, execute_subprocess returns the subprocess + return code, otherwise execute_subprocess treats non-zero return + status as an error and raises an exception. + + """ + cwd = os.getcwd() + msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd) + logging.info(msg) + commands_str = ' '.join(commands) + logging.info(commands_str) + return_to_caller = status_to_caller or output_to_caller + status = -1 + output = '' + hanging_timer = Timer(_HANGING_SEC, _hanging_msg, + kwargs={"working_directory": cwd, + "command": commands_str}) + hanging_timer.start() + try: + output = subprocess.check_output(commands, stderr=subprocess.STDOUT, + universal_newlines=True) + log_process_output(output) + status = 0 + except OSError as error: + msg = failed_command_msg( + 'Command execution failed. Does the executable exist?', + commands) + logging.error(error) + fatal_error(msg) + except ValueError as error: + msg = failed_command_msg( + 'DEV_ERROR: Invalid arguments trying to run subprocess', + commands) + logging.error(error) + fatal_error(msg) + except subprocess.CalledProcessError as error: + # Only report the error if we are NOT returning to the + # caller. If we are returning to the caller, then it may be a + # simple status check. If returning, it is the callers + # responsibility determine if an error occurred and handle it + # appropriately. + if not return_to_caller: + msg_context = ('Process did not run successfully; ' + 'returned status {0}'.format(error.returncode)) + msg = failed_command_msg(msg_context, commands, + output=error.output) + logging.error(error) + logging.error(msg) + log_process_output(error.output) + fatal_error(msg) + status = error.returncode + finally: + hanging_timer.cancel() + + if status_to_caller and output_to_caller: + ret_value = (status, output) + elif status_to_caller: + ret_value = status + elif output_to_caller: + ret_value = output + else: + ret_value = None + + return ret_value + + +def failed_command_msg(msg_context, command, output=None): + """Template for consistent error messages from subprocess calls. + + If 'output' is given, it should provide the output from the failed + command + """ + + if output: + output_truncated = last_n_lines(output, 20, + truncation_message='[... Output truncated for brevity ...]') + errmsg = ('Failed with output:\n' + + indent_string(output_truncated, 4) + + '\nERROR: ') + else: + errmsg = '' + + command_str = ' '.join(command) + errmsg += """In directory + {cwd} +{context}: + {command} +""".format(cwd=os.getcwd(), context=msg_context, command=command_str) + + if output: + errmsg += 'See above for output from failed command.\n' + + return errmsg From cf0086311daaf62ee33df010946d0a0ddc5bc400 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Tue, 28 Jan 2020 15:23:14 +0000 Subject: [PATCH 02/11] Issue #3 - remove copy of manage_externals under util and add README.md file --- README.md | 7 + util/manage_externals/checkout_externals | 36 - util/manage_externals/manic/__init__.py | 9 - util/manage_externals/manic/checkout.py | 424 --------- .../manic/externals_description.py | 794 ----------------- .../manic/externals_status.py | 164 ---- .../manic/global_constants.py | 18 - util/manage_externals/manic/repository.py | 98 --- .../manic/repository_factory.py | 29 - util/manage_externals/manic/repository_git.py | 819 ------------------ util/manage_externals/manic/repository_svn.py | 284 ------ util/manage_externals/manic/sourcetree.py | 351 -------- util/manage_externals/manic/utils.py | 330 ------- 13 files changed, 7 insertions(+), 3356 deletions(-) create mode 100644 README.md delete mode 100755 util/manage_externals/checkout_externals delete mode 100644 util/manage_externals/manic/__init__.py delete mode 100755 util/manage_externals/manic/checkout.py delete mode 100644 util/manage_externals/manic/externals_description.py delete mode 100644 util/manage_externals/manic/externals_status.py delete mode 100644 util/manage_externals/manic/global_constants.py delete mode 100644 util/manage_externals/manic/repository.py delete mode 100644 util/manage_externals/manic/repository_factory.py delete mode 100644 util/manage_externals/manic/repository_git.py delete mode 100644 util/manage_externals/manic/repository_svn.py delete mode 100644 util/manage_externals/manic/sourcetree.py delete mode 100644 util/manage_externals/manic/utils.py diff --git a/README.md b/README.md new file mode 100644 index 0000000000..6c3f627e53 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# global-workflow +Global Superstructure/Workflow currently supporting the Finite-Volume on a Cubed-Sphere Global Forecast System (FV3GFS) + +The global-workflow depends on the following prerequisities to be available on the system: + +* manage_externals - A utility from ESMCI to checkout external dependencies. Manage_externals can be obtained at the following address and should in the users PATH: https://github.com/ESMCI/manage_externals + diff --git a/util/manage_externals/checkout_externals b/util/manage_externals/checkout_externals deleted file mode 100755 index a0698baef0..0000000000 --- a/util/manage_externals/checkout_externals +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python - -"""Main driver wrapper around the manic/checkout utility. - -Tool to assemble external respositories represented in an externals -description file. - -""" -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import sys -import traceback - -import manic - -if sys.hexversion < 0x02070000: - print(70 * '*') - print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) - print('It appears that you are running python {0}'.format( - '.'.join(str(x) for x in sys.version_info[0:3]))) - print(70 * '*') - sys.exit(1) - - -if __name__ == '__main__': - ARGS = manic.checkout.commandline_arguments() - try: - RET_STATUS, _ = manic.checkout.main(ARGS) - sys.exit(RET_STATUS) - except Exception as error: # pylint: disable=broad-except - manic.printlog(str(error)) - if ARGS.backtrace: - traceback.print_exc() - sys.exit(1) diff --git a/util/manage_externals/manic/__init__.py b/util/manage_externals/manic/__init__.py deleted file mode 100644 index 11badedd3b..0000000000 --- a/util/manage_externals/manic/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Public API for the manage_externals library -""" - -from manic import checkout -from manic.utils import printlog - -__all__ = [ - 'checkout', 'printlog', -] diff --git a/util/manage_externals/manic/checkout.py b/util/manage_externals/manic/checkout.py deleted file mode 100755 index edc5655954..0000000000 --- a/util/manage_externals/manic/checkout.py +++ /dev/null @@ -1,424 +0,0 @@ -#!/usr/bin/env python - -""" -Tool to assemble repositories represented in a model-description file. - -If loaded as a module (e.g., in a component's buildcpp), it can be used -to check the validity of existing subdirectories and load missing sources. -""" -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import argparse -import logging -import os -import os.path -import sys - -from manic.externals_description import create_externals_description -from manic.externals_description import read_externals_description_file -from manic.externals_status import check_safe_to_update_repos -from manic.sourcetree import SourceTree -from manic.utils import printlog, fatal_error -from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME - -if sys.hexversion < 0x02070000: - print(70 * '*') - print('ERROR: {0} requires python >= 2.7.x. '.format(sys.argv[0])) - print('It appears that you are running python {0}'.format( - VERSION_SEPERATOR.join(str(x) for x in sys.version_info[0:3]))) - print(70 * '*') - sys.exit(1) - - -# --------------------------------------------------------------------- -# -# User input -# -# --------------------------------------------------------------------- -def commandline_arguments(args=None): - """Process the command line arguments - - Params: args - optional args. Should only be used during systems - testing. - - Returns: processed command line arguments - """ - description = ''' - -%(prog)s manages checking out groups of externals from revision -control based on an externals description file. By default only the -required externals are checkout out. - -Running %(prog)s without the '--status' option will always attempt to -synchronize the working copy to exactly match the externals description. -''' - - epilog = ''' -``` -NOTE: %(prog)s *MUST* be run from the root of the source tree it -is managing. For example, if you cloned a repository with: - - $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev - -Then the root of the source tree is /path/to/some-project-dev. If you -obtained a sub-project via a checkout of another project: - - $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev - -and you need to checkout the sub-project externals, then the root of the -source tree remains /path/to/some-project-dev. Do *NOT* run %(prog)s -from within /path/to/some-project-dev/sub-project - -The root of the source tree will be referred to as `${SRC_ROOT}` below. - - -# Supported workflows - - * Checkout all required components from the default externals - description file: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s - - * To update all required components to the current values in the - externals description file, re-run %(prog)s: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s - - If there are *any* modifications to *any* working copy according - to the git or svn 'status' command, %(prog)s - will not update any external repositories. Modifications - include: modified files, added files, removed files, or missing - files. - - To avoid this safety check, edit the externals description file - and comment out the modified external block. - - * Checkout all required components from a user specified externals - description file: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s --externals my-externals.cfg - - * Status summary of the repositories managed by %(prog)s: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s --status - - ./cime - s ./components/cism - ./components/mosart - e-o ./components/rtm - M ./src/fates - e-o ./tools/PTCLM - - - where: - * column one indicates the status of the repository in relation - to the externals description file. - * column two indicates whether the working copy has modified files. - * column three shows how the repository is managed, optional or required - - Column one will be one of these values: - * s : out-of-sync : repository is checked out at a different commit - compared with the externals description - * e : empty : directory does not exist - %(prog)s has not been run - * ? : unknown : directory exists but .git or .svn directories are missing - - Column two will be one of these values: - * M : Modified : modified, added, deleted or missing files - * : blank / space : clean - * - : dash : no meaningful state, for empty repositories - - Column three will be one of these values: - * o : optional : optionally repository - * : blank / space : required repository - - * Detailed git or svn status of the repositories managed by %(prog)s: - - $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s --status --verbose - -# Externals description file - - The externals description contains a list of the external - repositories that are used and their version control locations. The - file format is the standard ini/cfg configuration file format. Each - external is defined by a section containing the component name in - square brackets: - - * name (string) : component name, e.g. [cime], [cism], etc. - - Each section has the following keyword-value pairs: - - * required (boolean) : whether the component is a required checkout, - 'true' or 'false'. - - * local_path (string) : component path *relative* to where - %(prog)s is called. - - * protoctol (string) : version control protocol that is used to - manage the component. Valid values are 'git', 'svn', - 'externals_only'. - - Switching an external between different protocols is not - supported, e.g. from svn to git. To switch protocols, you need to - manually move the old working copy to a new location. - - Note: 'externals_only' will only process the external's own - external description file without trying to manage a repository - for the component. This is used for retrieving externals for - standalone components like cam and ctsm which also serve as - sub-components within a larger project. If the source root of the - externals_only component is the same as the main source root, then - the local path must be set to '.', the unix current working - directory, e. g. 'local_path = .' - - * repo_url (string) : URL for the repository location, examples: - * https://svn-ccsm-models.cgd.ucar.edu/glc - * git@github.com:esmci/cime.git - * /path/to/local/repository - * . - - NOTE: To operate on only the local clone and and ignore remote - repositories, set the url to '.' (the unix current path), - i.e. 'repo_url = .' . This can be used to checkout a local branch - instead of the upstream branch. - - If a repo url is determined to be a local path (not a network url) - then user expansion, e.g. ~/, and environment variable expansion, - e.g. $HOME or $REPO_ROOT, will be performed. - - Relative paths are difficult to get correct, especially for mixed - use repos. It is advised that local paths expand to absolute paths. - If relative paths are used, they should be relative to one level - above local_path. If local path is 'src/foo', the the relative url - should be relative to 'src'. - - * tag (string) : tag to checkout - - * hash (string) : the git hash to checkout. Only applies to git - repositories. - - * branch (string) : branch to checkout from the specified - repository. Specifying a branch on a remote repository means that - %(prog)s will checkout the version of the branch in the remote, - not the the version in the local repository (if it exists). - - Note: one and only one of tag, branch hash must be supplied. - - * externals (string) : used to make manage_externals aware of - sub-externals required by an external. This is a relative path to - the external's root directory. For example, if LIBX is often used - as a sub-external, it might have an externals file (for its - externals) called Externals_LIBX.cfg. To use libx as a standalone - checkout, it would have another file, Externals.cfg with the - following entry: - - [ libx ] - local_path = . - protocol = externals_only - externals = Externals_LIBX.cfg - required = True - - Now, %(prog)s will process Externals.cfg and also process - Externals_LIBX.cfg as if it was a sub-external. - - * from_submodule (True / False) : used to pull the repo_url, local_path, - and hash properties for this external from the .gitmodules file in - this repository. Note that the section name (the entry in square - brackets) must match the name in the .gitmodules file. - If from_submodule is True, the protocol must be git and no repo_url, - local_path, hash, branch, or tag entries are allowed. - Default: False - - * sparse (string) : used to control a sparse checkout. This optional - entry should point to a filename (path relative to local_path) that - contains instructions on which repository paths to include (or - exclude) from the working tree. - See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree - Default: sparse checkout is disabled - - * Lines beginning with '#' or ';' are comments and will be ignored. - -# Obtaining this tool, reporting issues, etc. - - The master repository for manage_externals is - https://github.com/ESMCI/manage_externals. Any issues with this tool - should be reported there. - -# Troubleshooting - -Operations performed by manage_externals utilities are explicit and -data driven. %(prog)s will always attempt to make the working copy -*exactly* match what is in the externals file when modifying the -working copy of a repository. - -If %(prog)s is not doing what you expected, double check the contents -of the externals description file or examine the output of -./manage_externals/%(prog)s --status - -''' - - parser = argparse.ArgumentParser( - description=description, epilog=epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) - - # - # user options - # - parser.add_argument("components", nargs="*", - help="Specific component(s) to checkout. By default, " - "all required externals are checked out.") - - parser.add_argument('-e', '--externals', nargs='?', - default='Externals.cfg', - help='The externals description filename. ' - 'Default: %(default)s.') - - parser.add_argument('-o', '--optional', action='store_true', default=False, - help='By default only the required externals ' - 'are checked out. This flag will also checkout the ' - 'optional externals.') - - parser.add_argument('-S', '--status', action='store_true', default=False, - help='Output the status of the repositories managed by ' - '%(prog)s. By default only summary information ' - 'is provided. Use the verbose option to see details.') - - parser.add_argument('-v', '--verbose', action='count', default=0, - help='Output additional information to ' - 'the screen and log file. This flag can be ' - 'used up to two times, increasing the ' - 'verbosity level each time.') - - parser.add_argument('--svn-ignore-ancestry', action='store_true', default=False, - help='By default, subversion will abort if a component is ' - 'already checked out and there is no common ancestry with ' - 'the new URL. This flag passes the "--ignore-ancestry" flag ' - 'to the svn switch call. (This is not recommended unless ' - 'you are sure about what you are doing.)') - - # - # developer options - # - parser.add_argument('--backtrace', action='store_true', - help='DEVELOPER: show exception backtraces as extra ' - 'debugging output') - - parser.add_argument('-d', '--debug', action='store_true', default=False, - help='DEVELOPER: output additional debugging ' - 'information to the screen and log file.') - - logging_group = parser.add_mutually_exclusive_group() - - logging_group.add_argument('--logging', dest='do_logging', - action='store_true', - help='DEVELOPER: enable logging.') - logging_group.add_argument('--no-logging', dest='do_logging', - action='store_false', default=False, - help='DEVELOPER: disable logging ' - '(this is the default)') - - if args: - options = parser.parse_args(args) - else: - options = parser.parse_args() - return options - - -# --------------------------------------------------------------------- -# -# main -# -# --------------------------------------------------------------------- -def main(args): - """ - Function to call when module is called from the command line. - Parse externals file and load required repositories or all repositories if - the --all option is passed. - - Returns a tuple (overall_status, tree_status). overall_status is 0 - on success, non-zero on failure. tree_status gives the full status - *before* executing the checkout command - i.e., the status that it - used to determine if it's safe to proceed with the checkout. - """ - if args.do_logging: - logging.basicConfig(filename=LOG_FILE_NAME, - format='%(levelname)s : %(asctime)s : %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - level=logging.DEBUG) - - program_name = os.path.basename(sys.argv[0]) - logging.info('Beginning of %s', program_name) - - load_all = False - if args.optional: - load_all = True - - root_dir = os.path.abspath(os.getcwd()) - external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description( - external_data, components=args.components) - - for comp in args.components: - if comp not in external.keys(): - fatal_error( - "No component {} found in {}".format( - comp, args.externals)) - - source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) - printlog('Checking status of externals: ', end='') - tree_status = source_tree.status() - printlog('') - - if args.status: - # user requested status-only - for comp in sorted(tree_status.keys()): - tree_status[comp].log_status_message(args.verbose) - else: - # checkout / update the external repositories. - safe_to_update = check_safe_to_update_repos(tree_status) - if not safe_to_update: - # print status - for comp in sorted(tree_status.keys()): - tree_status[comp].log_status_message(args.verbose) - # exit gracefully - msg = """The external repositories labeled with 'M' above are not in a clean state. - -The following are two options for how to proceed: - -(1) Go into each external that is not in a clean state and issue either - an 'svn status' or a 'git status' command. Either revert or commit - your changes so that all externals are in a clean state. (Note, - though, that it is okay to have untracked files in your working - directory.) Then rerun {program_name}. - -(2) Alternatively, you do not have to rely on {program_name}. Instead, you - can manually update out-of-sync externals (labeled with 's' above) - as described in the configuration file {config_file}. - - -The external repositories labeled with '?' above are not under version -control using the expected protocol. If you are sure you want to switch -protocols, and you don't have any work you need to save from this -directory, then run "rm -rf [directory]" before re-running the -checkout_externals tool. -""".format(program_name=program_name, config_file=args.externals) - - printlog('-' * 70) - printlog(msg) - printlog('-' * 70) - else: - if not args.components: - source_tree.checkout(args.verbose, load_all) - for comp in args.components: - source_tree.checkout(args.verbose, load_all, load_comp=comp) - printlog('') - - logging.info('%s completed without exceptions.', program_name) - # NOTE(bja, 2017-11) tree status is used by the systems tests - return 0, tree_status diff --git a/util/manage_externals/manic/externals_description.py b/util/manage_externals/manic/externals_description.py deleted file mode 100644 index b0c4f736a7..0000000000 --- a/util/manage_externals/manic/externals_description.py +++ /dev/null @@ -1,794 +0,0 @@ -#!/usr/bin/env python - -"""Model description - -Model description is the representation of the various externals -included in the model. It processes in input data structure, and -converts it into a standard interface that is used by the rest of the -system. - -To maintain backward compatibility, externals description files should -follow semantic versioning rules, http://semver.org/ - - - -""" -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import logging -import os -import os.path -import re - -# ConfigParser in python2 was renamed to configparser in python3. -# In python2, ConfigParser returns byte strings, str, instead of unicode. -# We need unicode to be compatible with xml and json parser and python3. -try: - # python2 - from ConfigParser import SafeConfigParser as config_parser - from ConfigParser import MissingSectionHeaderError - from ConfigParser import NoSectionError, NoOptionError - - USE_PYTHON2 = True - - def config_string_cleaner(text): - """convert strings into unicode - """ - return text.decode('utf-8') -except ImportError: - # python3 - from configparser import ConfigParser as config_parser - from configparser import MissingSectionHeaderError - from configparser import NoSectionError, NoOptionError - - USE_PYTHON2 = False - - def config_string_cleaner(text): - """Python3 already uses unicode strings, so just return the string - without modification. - - """ - return text - -from .utils import printlog, fatal_error, str_to_bool, expand_local_url -from .utils import execute_subprocess -from .global_constants import EMPTY_STR, PPRINTER, VERSION_SEPERATOR - -# -# Globals -# -DESCRIPTION_SECTION = 'externals_description' -VERSION_ITEM = 'schema_version' - - -def read_externals_description_file(root_dir, file_name): - """Read a file containing an externals description and - create its internal representation. - - """ - root_dir = os.path.abspath(root_dir) - msg = 'In directory : {0}'.format(root_dir) - logging.info(msg) - printlog('Processing externals description file : {0}'.format(file_name)) - - file_path = os.path.join(root_dir, file_name) - if not os.path.exists(file_name): - if file_name.lower() == "none": - msg = ('INTERNAL ERROR: Attempt to read externals file ' - 'from {0} when not configured'.format(file_path)) - else: - msg = ('ERROR: Model description file, "{0}", does not ' - 'exist at path:\n {1}\nDid you run from the root of ' - 'the source tree?'.format(file_name, file_path)) - - fatal_error(msg) - - externals_description = None - if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: - externals_description = read_gitmodules_file(root_dir, file_name) - else: - try: - config = config_parser() - config.read(file_path) - externals_description = config - except MissingSectionHeaderError: - # not a cfg file - pass - - if externals_description is None: - msg = 'Unknown file format!' - fatal_error(msg) - - return externals_description - -class LstripReader(object): - "LstripReader formats .gitmodules files to be acceptable for configparser" - def __init__(self, filename): - with open(filename, 'r') as infile: - lines = infile.readlines() - self._lines = list() - self._num_lines = len(lines) - self._index = 0 - for line in lines: - self._lines.append(line.lstrip()) - - def readlines(self): - """Return all the lines from this object's file""" - return self._lines - - def readline(self, size=-1): - """Format and return the next line or raise StopIteration""" - try: - line = self.next() - except StopIteration: - line = '' - - if (size > 0) and (len(line) < size): - return line[0:size] - - return line - - def __iter__(self): - """Begin an iteration""" - self._index = 0 - return self - - def next(self): - """Return the next line or raise StopIteration""" - if self._index >= self._num_lines: - raise StopIteration - - self._index = self._index + 1 - return self._lines[self._index - 1] - - def __next__(self): - return self.next() - -def git_submodule_status(repo_dir): - """Run the git submodule status command to obtain submodule hashes. - """ - # This function is here instead of GitRepository to avoid a dependency loop - cwd = os.getcwd() - os.chdir(repo_dir) - cmd = ['git', 'submodule', 'status'] - git_output = execute_subprocess(cmd, output_to_caller=True) - submodules = {} - submods = git_output.split('\n') - for submod in submods: - if submod: - status = submod[0] - items = submod[1:].split(' ') - if len(items) > 2: - tag = items[2] - else: - tag = None - - submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} - - os.chdir(cwd) - return submodules - -def parse_submodules_desc_section(section_items, file_path): - """Find the path and url for this submodule description""" - path = None - url = None - for item in section_items: - name = item[0].strip().lower() - if name == 'path': - path = item[1].strip() - elif name == 'url': - url = item[1].strip() - else: - msg = 'WARNING: Ignoring unknown {} property, in {}' - msg = msg.format(item[0], file_path) # fool pylint - logging.warning(msg) - - return path, url - -def read_gitmodules_file(root_dir, file_name): - # pylint: disable=deprecated-method - # Disabling this check because the method is only used for python2 - """Read a .gitmodules file and convert it to be compatible with an - externals description. - """ - root_dir = os.path.abspath(root_dir) - msg = 'In directory : {0}'.format(root_dir) - logging.info(msg) - printlog('Processing submodules description file : {0}'.format(file_name)) - - file_path = os.path.join(root_dir, file_name) - if not os.path.exists(file_name): - msg = ('ERROR: submodules description file, "{0}", does not ' - 'exist at path:\n {1}'.format(file_name, file_path)) - fatal_error(msg) - - submodules_description = None - externals_description = None - try: - config = config_parser() - if USE_PYTHON2: - config.readfp(LstripReader(file_path), filename=file_name) - else: - config.read_file(LstripReader(file_path), source=file_name) - - submodules_description = config - except MissingSectionHeaderError: - # not a cfg file - pass - - if submodules_description is None: - msg = 'Unknown file format!' - fatal_error(msg) - else: - # Convert the submodules description to an externals description - externals_description = config_parser() - # We need to grab all the commit hashes for this repo - submods = git_submodule_status(root_dir) - for section in submodules_description.sections(): - if section[0:9] == 'submodule': - sec_name = section[9:].strip(' "') - externals_description.add_section(sec_name) - section_items = submodules_description.items(section) - path, url = parse_submodules_desc_section(section_items, - file_path) - - if path is None: - msg = 'Submodule {} missing path'.format(sec_name) - fatal_error(msg) - - if url is None: - msg = 'Submodule {} missing url'.format(sec_name) - fatal_error(msg) - - externals_description.set(sec_name, - ExternalsDescription.PATH, path) - externals_description.set(sec_name, - ExternalsDescription.PROTOCOL, 'git') - externals_description.set(sec_name, - ExternalsDescription.REPO_URL, url) - externals_description.set(sec_name, - ExternalsDescription.REQUIRED, 'True') - git_hash = submods[sec_name]['hash'] - externals_description.set(sec_name, - ExternalsDescription.HASH, git_hash) - - # Required items - externals_description.add_section(DESCRIPTION_SECTION) - externals_description.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0') - - return externals_description - -def create_externals_description( - model_data, model_format='cfg', components=None, parent_repo=None): - """Create the a externals description object from the provided data - """ - externals_description = None - if model_format == 'dict': - externals_description = ExternalsDescriptionDict( - model_data, components=components) - elif model_format == 'cfg': - major, _, _ = get_cfg_schema_version(model_data) - if major == 1: - externals_description = ExternalsDescriptionConfigV1( - model_data, components=components, parent_repo=parent_repo) - else: - msg = ('Externals description file has unsupported schema ' - 'version "{0}".'.format(major)) - fatal_error(msg) - else: - msg = 'Unknown model data format "{0}"'.format(model_format) - fatal_error(msg) - return externals_description - - -def get_cfg_schema_version(model_cfg): - """Extract the major, minor, patch version of the config file schema - - Params: - model_cfg - config parser object containing the externas description data - - Returns: - major = integer major version - minor = integer minor version - patch = integer patch version - """ - semver_str = '' - try: - semver_str = model_cfg.get(DESCRIPTION_SECTION, VERSION_ITEM) - except (NoSectionError, NoOptionError): - msg = ('externals description file must have the required ' - 'section: "{0}" and item "{1}"'.format(DESCRIPTION_SECTION, - VERSION_ITEM)) - fatal_error(msg) - - # NOTE(bja, 2017-11) Assume we don't care about the - # build/pre-release metadata for now! - version_list = re.split(r'[-+]', semver_str) - version_str = version_list[0] - version = version_str.split(VERSION_SEPERATOR) - try: - major = int(version[0].strip()) - minor = int(version[1].strip()) - patch = int(version[2].strip()) - except ValueError: - msg = ('Config file schema version must have integer digits for ' - 'major, minor and patch versions. ' - 'Received "{0}"'.format(version_str)) - fatal_error(msg) - return major, minor, patch - - -class ExternalsDescription(dict): - """Base externals description class that is independent of the user input - format. Different input formats can all be converted to this - representation to provide a consistent represtentation for the - rest of the objects in the system. - - NOTE(bja, 2018-03): do NOT define _schema_major etc at the class - level in the base class. The nested/recursive nature of externals - means different schema versions may be present in a single run! - - All inheriting classes must overwrite: - self._schema_major and self._input_major - self._schema_minor and self._input_minor - self._schema_patch and self._input_patch - - where _schema_x is the supported schema, _input_x is the user - input value. - - """ - # keywords defining the interface into the externals description data - EXTERNALS = 'externals' - BRANCH = 'branch' - SUBMODULE = 'from_submodule' - HASH = 'hash' - NAME = 'name' - PATH = 'local_path' - PROTOCOL = 'protocol' - REPO = 'repo' - REPO_URL = 'repo_url' - REQUIRED = 'required' - TAG = 'tag' - SPARSE = 'sparse' - - PROTOCOL_EXTERNALS_ONLY = 'externals_only' - PROTOCOL_GIT = 'git' - PROTOCOL_SVN = 'svn' - GIT_SUBMODULES_FILENAME = '.gitmodules' - KNOWN_PRROTOCOLS = [PROTOCOL_GIT, PROTOCOL_SVN, PROTOCOL_EXTERNALS_ONLY] - - # v1 xml keywords - _V1_TREE_PATH = 'TREE_PATH' - _V1_ROOT = 'ROOT' - _V1_TAG = 'TAG' - _V1_BRANCH = 'BRANCH' - _V1_REQ_SOURCE = 'REQ_SOURCE' - - _source_schema = {REQUIRED: True, - PATH: 'string', - EXTERNALS: 'string', - SUBMODULE : True, - REPO: {PROTOCOL: 'string', - REPO_URL: 'string', - TAG: 'string', - BRANCH: 'string', - HASH: 'string', - SPARSE: 'string', - } - } - - def __init__(self, parent_repo=None): - """Convert the xml into a standardized dict that can be used to - construct the source objects - - """ - dict.__init__(self) - - self._schema_major = None - self._schema_minor = None - self._schema_patch = None - self._input_major = None - self._input_minor = None - self._input_patch = None - self._parent_repo = parent_repo - - def _verify_schema_version(self): - """Use semantic versioning rules to verify we can process this schema. - - """ - known = '{0}.{1}.{2}'.format(self._schema_major, - self._schema_minor, - self._schema_patch) - received = '{0}.{1}.{2}'.format(self._input_major, - self._input_minor, - self._input_patch) - - if self._input_major != self._schema_major: - # should never get here, the factory should handle this correctly! - msg = ('DEV_ERROR: version "{0}" parser received ' - 'version "{1}" input.'.format(known, received)) - fatal_error(msg) - - if self._input_minor > self._schema_minor: - msg = ('Incompatible schema version:\n' - ' User supplied schema version "{0}" is too new."\n' - ' Can only process version "{1}" files and ' - 'older.'.format(received, known)) - fatal_error(msg) - - if self._input_patch > self._schema_patch: - # NOTE(bja, 2018-03) ignoring for now... Not clear what - # conditions the test is needed. - pass - - def _check_user_input(self): - """Run a series of checks to attempt to validate the user input and - detect errors as soon as possible. - - NOTE(bja, 2018-03) These checks are called *after* the file is - read. That means the schema check can not occur here. - - Note: the order is important. check_optional will create - optional with null data. run check_data first to ensure - required data was provided correctly by the user. - - """ - self._check_data() - self._check_optional() - self._validate() - - def _check_data(self): - # pylint: disable=too-many-branches,too-many-statements - """Check user supplied data is valid where possible. - """ - for ext_name in self.keys(): - if (self[ext_name][self.REPO][self.PROTOCOL] - not in self.KNOWN_PRROTOCOLS): - msg = 'Unknown repository protocol "{0}" in "{1}".'.format( - self[ext_name][self.REPO][self.PROTOCOL], ext_name) - fatal_error(msg) - - if (self[ext_name][self.REPO][self.PROTOCOL] == - self.PROTOCOL_SVN): - if self.HASH in self[ext_name][self.REPO]: - msg = ('In repo description for "{0}". svn repositories ' - 'may not include the "hash" keyword.'.format( - ext_name)) - fatal_error(msg) - - if ((self[ext_name][self.REPO][self.PROTOCOL] != self.PROTOCOL_GIT) - and (self.SUBMODULE in self[ext_name])): - msg = ('self.SUBMODULE is only supported with {0} protocol, ' - '"{1}" is defined as an {2} repository') - fatal_error(msg.format(self.PROTOCOL_GIT, ext_name, - self[ext_name][self.REPO][self.PROTOCOL])) - - if (self[ext_name][self.REPO][self.PROTOCOL] != - self.PROTOCOL_EXTERNALS_ONLY): - ref_count = 0 - found_refs = '' - if self.TAG in self[ext_name][self.REPO]: - ref_count += 1 - found_refs = '"{0} = {1}", {2}'.format( - self.TAG, self[ext_name][self.REPO][self.TAG], - found_refs) - if self.BRANCH in self[ext_name][self.REPO]: - ref_count += 1 - found_refs = '"{0} = {1}", {2}'.format( - self.BRANCH, self[ext_name][self.REPO][self.BRANCH], - found_refs) - if self.HASH in self[ext_name][self.REPO]: - ref_count += 1 - found_refs = '"{0} = {1}", {2}'.format( - self.HASH, self[ext_name][self.REPO][self.HASH], - found_refs) - if (self.SUBMODULE in self[ext_name] and - self[ext_name][self.SUBMODULE]): - ref_count += 1 - found_refs = '"{0} = {1}", {2}'.format( - self.SUBMODULE, - self[ext_name][self.SUBMODULE], found_refs) - - if ref_count > 1: - msg = 'Model description is over specified! ' - if self.SUBMODULE in self[ext_name]: - msg += ('from_submodule is not compatible with ' - '"tag", "branch", or "hash" ') - else: - msg += (' Only one of "tag", "branch", or "hash" ' - 'may be specified ') - - msg += 'for repo description of "{0}".'.format(ext_name) - msg = '{0}\nFound: {1}'.format(msg, found_refs) - fatal_error(msg) - elif ref_count < 1: - msg = ('Model description is under specified! One of ' - '"tag", "branch", or "hash" must be specified for ' - 'repo description of "{0}"'.format(ext_name)) - fatal_error(msg) - - if (self.REPO_URL not in self[ext_name][self.REPO] and - (self.SUBMODULE not in self[ext_name] or - not self[ext_name][self.SUBMODULE])): - msg = ('Model description is under specified! Must have ' - '"repo_url" in repo ' - 'description for "{0}"'.format(ext_name)) - fatal_error(msg) - - if (self.SUBMODULE in self[ext_name] and - self[ext_name][self.SUBMODULE]): - if self.REPO_URL in self[ext_name][self.REPO]: - msg = ('Model description is over specified! ' - 'from_submodule keyword is not compatible ' - 'with {0} keyword for'.format(self.REPO_URL)) - msg = '{0} repo description of "{1}"'.format(msg, - ext_name) - fatal_error(msg) - - if self.PATH in self[ext_name]: - msg = ('Model description is over specified! ' - 'from_submodule keyword is not compatible with ' - '{0} keyword for'.format(self.PATH)) - msg = '{0} repo description of "{1}"'.format(msg, - ext_name) - fatal_error(msg) - - if self.REPO_URL in self[ext_name][self.REPO]: - url = expand_local_url( - self[ext_name][self.REPO][self.REPO_URL], ext_name) - self[ext_name][self.REPO][self.REPO_URL] = url - - def _check_optional(self): - # pylint: disable=too-many-branches - """Some fields like externals, repo:tag repo:branch are - (conditionally) optional. We don't want the user to be - required to enter them in every externals description file, but - still want to validate the input. Check conditions and add - default values if appropriate. - - """ - submod_desc = None # Only load submodules info once - for field in self: - # truely optional - if self.EXTERNALS not in self[field]: - self[field][self.EXTERNALS] = EMPTY_STR - - # git and svn repos must tags and branches for validation purposes. - if self.TAG not in self[field][self.REPO]: - self[field][self.REPO][self.TAG] = EMPTY_STR - if self.BRANCH not in self[field][self.REPO]: - self[field][self.REPO][self.BRANCH] = EMPTY_STR - if self.HASH not in self[field][self.REPO]: - self[field][self.REPO][self.HASH] = EMPTY_STR - if self.REPO_URL not in self[field][self.REPO]: - self[field][self.REPO][self.REPO_URL] = EMPTY_STR - if self.SPARSE not in self[field][self.REPO]: - self[field][self.REPO][self.SPARSE] = EMPTY_STR - - # from_submodule has a complex relationship with other fields - if self.SUBMODULE in self[field]: - # User wants to use submodule information, is it available? - if self._parent_repo is None: - # No parent == no submodule information - PPRINTER.pprint(self[field]) - msg = 'No parent submodule for "{0}"'.format(field) - fatal_error(msg) - elif self._parent_repo.protocol() != self.PROTOCOL_GIT: - PPRINTER.pprint(self[field]) - msg = 'Parent protocol, "{0}", does not support submodules' - fatal_error(msg.format(self._parent_repo.protocol())) - else: - args = self._repo_config_from_submodule(field, submod_desc) - repo_url, repo_path, ref_hash, submod_desc = args - - if repo_url is None: - msg = ('Cannot checkout "{0}" as a submodule, ' - 'repo not found in {1} file') - fatal_error(msg.format(field, - self.GIT_SUBMODULES_FILENAME)) - # Fill in submodule fields - self[field][self.REPO][self.REPO_URL] = repo_url - self[field][self.REPO][self.HASH] = ref_hash - self[field][self.PATH] = repo_path - - if self[field][self.SUBMODULE]: - # We should get everything from the parent submodule - # configuration. - pass - # No else (from _submodule = False is the default) - else: - # Add the default value (not using submodule information) - self[field][self.SUBMODULE] = False - - def _repo_config_from_submodule(self, field, submod_desc): - """Find the external config information for a repository from - its submodule configuration information. - """ - if submod_desc is None: - repo_path = os.getcwd() # Is this always correct? - submod_file = self._parent_repo.submodules_file(repo_path=repo_path) - if submod_file is None: - msg = ('Cannot checkout "{0}" from submodule information\n' - ' Parent repo, "{1}" does not have submodules') - fatal_error(msg.format(field, self._parent_repo.name())) - - submod_file = read_gitmodules_file(repo_path, submod_file) - submod_desc = create_externals_description(submod_file) - - # Can we find our external? - repo_url = None - repo_path = None - ref_hash = None - for ext_field in submod_desc: - if field == ext_field: - ext = submod_desc[ext_field] - repo_url = ext[self.REPO][self.REPO_URL] - repo_path = ext[self.PATH] - ref_hash = ext[self.REPO][self.HASH] - break - - return repo_url, repo_path, ref_hash, submod_desc - - def _validate(self): - """Validate that the parsed externals description contains all necessary - fields. - - """ - def print_compare_difference(data_a, data_b, loc_a, loc_b): - """Look through the data structures and print the differences. - - """ - for item in data_a: - if item in data_b: - if not isinstance(data_b[item], type(data_a[item])): - printlog(" {item}: {loc} = {val} ({val_type})".format( - item=item, loc=loc_a, val=data_a[item], - val_type=type(data_a[item]))) - printlog(" {item} {loc} = {val} ({val_type})".format( - item=' ' * len(item), loc=loc_b, val=data_b[item], - val_type=type(data_b[item]))) - else: - printlog(" {item}: {loc} = {val} ({val_type})".format( - item=item, loc=loc_a, val=data_a[item], - val_type=type(data_a[item]))) - printlog(" {item} {loc} missing".format( - item=' ' * len(item), loc=loc_b)) - - def validate_data_struct(schema, data): - """Compare a data structure against a schema and validate all required - fields are present. - - """ - is_valid = False - in_ref = True - valid = True - if isinstance(schema, dict) and isinstance(data, dict): - # Both are dicts, recursively verify that all fields - # in schema are present in the data. - for key in schema: - in_ref = in_ref and (key in data) - if in_ref: - valid = valid and ( - validate_data_struct(schema[key], data[key])) - - is_valid = in_ref and valid - else: - # non-recursive structure. verify data and schema have - # the same type. - is_valid = isinstance(data, type(schema)) - - if not is_valid: - printlog(" Unmatched schema and input:") - if isinstance(schema, dict): - print_compare_difference(schema, data, 'schema', 'input') - print_compare_difference(data, schema, 'input', 'schema') - else: - printlog(" schema = {0} ({1})".format( - schema, type(schema))) - printlog(" input = {0} ({1})".format(data, type(data))) - - return is_valid - - for field in self: - valid = validate_data_struct(self._source_schema, self[field]) - if not valid: - PPRINTER.pprint(self._source_schema) - PPRINTER.pprint(self[field]) - msg = 'ERROR: source for "{0}" did not validate'.format(field) - fatal_error(msg) - - -class ExternalsDescriptionDict(ExternalsDescription): - """Create a externals description object from a dictionary using the API - representations. Primarily used to simplify creating model - description files for unit testing. - - """ - - def __init__(self, model_data, components=None): - """Parse a native dictionary into a externals description. - """ - ExternalsDescription.__init__(self) - self._schema_major = 1 - self._schema_minor = 0 - self._schema_patch = 0 - self._input_major = 1 - self._input_minor = 0 - self._input_patch = 0 - self._verify_schema_version() - if components: - for key in model_data.items(): - if key not in components: - del model_data[key] - - self.update(model_data) - self._check_user_input() - - -class ExternalsDescriptionConfigV1(ExternalsDescription): - """Create a externals description object from a config_parser object, - schema version 1. - - """ - - def __init__(self, model_data, components=None, parent_repo=None): - """Convert the config data into a standardized dict that can be used to - construct the source objects - - """ - ExternalsDescription.__init__(self, parent_repo=parent_repo) - self._schema_major = 1 - self._schema_minor = 1 - self._schema_patch = 0 - self._input_major, self._input_minor, self._input_patch = \ - get_cfg_schema_version(model_data) - self._verify_schema_version() - self._remove_metadata(model_data) - self._parse_cfg(model_data, components=components) - self._check_user_input() - - @staticmethod - def _remove_metadata(model_data): - """Remove the metadata section from the model configuration file so - that it is simpler to look through the file and construct the - externals description. - - """ - model_data.remove_section(DESCRIPTION_SECTION) - - def _parse_cfg(self, cfg_data, components=None): - """Parse a config_parser object into a externals description. - """ - def list_to_dict(input_list, convert_to_lower_case=True): - """Convert a list of key-value pairs into a dictionary. - """ - output_dict = {} - for item in input_list: - key = config_string_cleaner(item[0].strip()) - value = config_string_cleaner(item[1].strip()) - if convert_to_lower_case: - key = key.lower() - output_dict[key] = value - return output_dict - - for section in cfg_data.sections(): - name = config_string_cleaner(section.lower().strip()) - if components and name not in components: - continue - self[name] = {} - self[name].update(list_to_dict(cfg_data.items(section))) - self[name][self.REPO] = {} - loop_keys = self[name].copy().keys() - for item in loop_keys: - if item in self._source_schema: - if isinstance(self._source_schema[item], bool): - self[name][item] = str_to_bool(self[name][item]) - elif item in self._source_schema[self.REPO]: - self[name][self.REPO][item] = self[name][item] - del self[name][item] - else: - msg = ('Invalid input: "{sect}" contains unknown ' - 'item "{item}".'.format(sect=name, item=item)) - fatal_error(msg) diff --git a/util/manage_externals/manic/externals_status.py b/util/manage_externals/manic/externals_status.py deleted file mode 100644 index d3d238f289..0000000000 --- a/util/manage_externals/manic/externals_status.py +++ /dev/null @@ -1,164 +0,0 @@ -"""ExternalStatus - -Class to store status and state information about repositories and -create a string representation. - -""" -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -from .global_constants import EMPTY_STR -from .utils import printlog, indent_string -from .global_constants import VERBOSITY_VERBOSE, VERBOSITY_DUMP - - -class ExternalStatus(object): - """Class to represent the status of a given source repository or tree. - - Individual repositories determine their own status in the - Repository objects. This object is just resposible for storing the - information and passing it up to a higher level for reporting or - global decisions. - - There are two states of concern: - - * If the repository is in-sync with the externals description file. - - * If the repostiory working copy is clean and there are no pending - transactions (e.g. add, remove, rename, untracked files). - - """ - DEFAULT = '-' - UNKNOWN = '?' - EMPTY = 'e' - MODEL_MODIFIED = 's' # a.k.a. out-of-sync - DIRTY = 'M' - - STATUS_OK = ' ' - STATUS_ERROR = '!' - - # source types - OPTIONAL = 'o' - STANDALONE = 's' - MANAGED = ' ' - - def __init__(self): - self.sync_state = self.DEFAULT - self.clean_state = self.DEFAULT - self.source_type = self.DEFAULT - self.path = EMPTY_STR - self.current_version = EMPTY_STR - self.expected_version = EMPTY_STR - self.status_output = EMPTY_STR - - def log_status_message(self, verbosity): - """Write status message to the screen and log file - """ - self._default_status_message() - if verbosity >= VERBOSITY_VERBOSE: - self._verbose_status_message() - if verbosity >= VERBOSITY_DUMP: - self._dump_status_message() - - def _default_status_message(self): - """Return the default terse status message string - """ - msg = '{sync}{clean}{src_type} {path}'.format( - sync=self.sync_state, clean=self.clean_state, - src_type=self.source_type, path=self.path) - printlog(msg) - - def _verbose_status_message(self): - """Return the verbose status message string - """ - clean_str = self.DEFAULT - if self.clean_state == self.STATUS_OK: - clean_str = 'clean sandbox' - elif self.clean_state == self.DIRTY: - clean_str = 'modified sandbox' - - sync_str = 'on {0}'.format(self.current_version) - if self.sync_state != self.STATUS_OK: - sync_str = '{current} --> {expected}'.format( - current=self.current_version, expected=self.expected_version) - msg = ' {clean}, {sync}'.format(clean=clean_str, sync=sync_str) - printlog(msg) - - def _dump_status_message(self): - """Return the dump status message string - """ - msg = indent_string(self.status_output, 12) - printlog(msg) - - def safe_to_update(self): - """Report if it is safe to update a repository. Safe is defined as: - - * If a repository is empty, it is safe to update. - - * If a repository exists and has a clean working copy state - with no pending transactions. - - """ - safe_to_update = False - repo_exists = self.exists() - if not repo_exists: - safe_to_update = True - else: - # If the repo exists, it must be in ok or modified - # sync_state. Any other sync_state at this point - # represents a logic error that should have been handled - # before now! - sync_safe = ((self.sync_state == ExternalStatus.STATUS_OK) or - (self.sync_state == ExternalStatus.MODEL_MODIFIED)) - if sync_safe: - # The clean_state must be STATUS_OK to update. Otherwise we - # are dirty or there was a missed error previously. - if self.clean_state == ExternalStatus.STATUS_OK: - safe_to_update = True - return safe_to_update - - def exists(self): - """Determine if the repo exists. This is indicated by: - - * sync_state is not EMPTY - - * if the sync_state is empty, then the valid states for - clean_state are default, empty or unknown. Anything else - and there was probably an internal logic error. - - NOTE(bja, 2017-10) For the moment we are considering a - sync_state of default or unknown to require user intervention, - but we may want to relax this convention. This is probably a - result of a network error or internal logic error but more - testing is needed. - - """ - is_empty = (self.sync_state == ExternalStatus.EMPTY) - clean_valid = ((self.clean_state == ExternalStatus.DEFAULT) or - (self.clean_state == ExternalStatus.EMPTY) or - (self.clean_state == ExternalStatus.UNKNOWN)) - - if is_empty and clean_valid: - exists = False - else: - exists = True - return exists - - -def check_safe_to_update_repos(tree_status): - """Check if *ALL* repositories are in a safe state to update. We don't - want to do a partial update of the repositories then die, leaving - the model in an inconsistent state. - - Note: if there is an update to do, the repositories will by - definiation be out of synce with the externals description, so we - can't use that as criteria for updating. - - """ - safe_to_update = True - for comp in tree_status: - stat = tree_status[comp] - safe_to_update &= stat.safe_to_update() - - return safe_to_update diff --git a/util/manage_externals/manic/global_constants.py b/util/manage_externals/manic/global_constants.py deleted file mode 100644 index 0e91cffc90..0000000000 --- a/util/manage_externals/manic/global_constants.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Globals shared across modules -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import pprint - -EMPTY_STR = '' -LOCAL_PATH_INDICATOR = '.' -VERSION_SEPERATOR = '.' -LOG_FILE_NAME = 'manage_externals.log' -PPRINTER = pprint.PrettyPrinter(indent=4) - -VERBOSITY_DEFAULT = 0 -VERBOSITY_VERBOSE = 1 -VERBOSITY_DUMP = 2 diff --git a/util/manage_externals/manic/repository.py b/util/manage_externals/manic/repository.py deleted file mode 100644 index ea4230fb7b..0000000000 --- a/util/manage_externals/manic/repository.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Base class representation of a repository -""" - -from .externals_description import ExternalsDescription -from .utils import fatal_error -from .global_constants import EMPTY_STR - - -class Repository(object): - """ - Class to represent and operate on a repository description. - """ - - def __init__(self, component_name, repo): - """ - Parse repo externals description - """ - self._name = component_name - self._protocol = repo[ExternalsDescription.PROTOCOL] - self._tag = repo[ExternalsDescription.TAG] - self._branch = repo[ExternalsDescription.BRANCH] - self._hash = repo[ExternalsDescription.HASH] - self._url = repo[ExternalsDescription.REPO_URL] - self._sparse = repo[ExternalsDescription.SPARSE] - - if self._url is EMPTY_STR: - fatal_error('repo must have a URL') - - if ((self._tag is EMPTY_STR) and (self._branch is EMPTY_STR) and - (self._hash is EMPTY_STR)): - fatal_error('{0} repo must have a branch, tag or hash element') - - ref_count = 0 - if self._tag is not EMPTY_STR: - ref_count += 1 - if self._branch is not EMPTY_STR: - ref_count += 1 - if self._hash is not EMPTY_STR: - ref_count += 1 - if ref_count != 1: - fatal_error('repo {0} must have exactly one of ' - 'tag, branch or hash.'.format(self._name)) - - def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the source. - If the repo destination directory does not exist, checkout the correce - branch or tag. - NB: is include as an argument for compatibility with - git functionality (repository_git.py) - """ - msg = ('DEV_ERROR: checkout method must be implemented in all ' - 'repository classes! {0}'.format(self.__class__.__name__)) - fatal_error(msg) - - def status(self, stat, repo_dir_path): # pylint: disable=unused-argument - """Report the status of the repo - - """ - msg = ('DEV_ERROR: status method must be implemented in all ' - 'repository classes! {0}'.format(self.__class__.__name__)) - fatal_error(msg) - - def submodules_file(self, repo_path=None): - # pylint: disable=no-self-use,unused-argument - """Stub for use by non-git VC systems""" - return None - - def url(self): - """Public access of repo url. - """ - return self._url - - def tag(self): - """Public access of repo tag - """ - return self._tag - - def branch(self): - """Public access of repo branch. - """ - return self._branch - - def hash(self): - """Public access of repo hash. - """ - return self._hash - - def name(self): - """Public access of repo name. - """ - return self._name - - def protocol(self): - """Public access of repo protocol. - """ - return self._protocol diff --git a/util/manage_externals/manic/repository_factory.py b/util/manage_externals/manic/repository_factory.py deleted file mode 100644 index 80a92a9d8a..0000000000 --- a/util/manage_externals/manic/repository_factory.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Factory for creating and initializing the appropriate repository class -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -from .repository_git import GitRepository -from .repository_svn import SvnRepository -from .externals_description import ExternalsDescription -from .utils import fatal_error - - -def create_repository(component_name, repo_info, svn_ignore_ancestry=False): - """Determine what type of repository we have, i.e. git or svn, and - create the appropriate object. - - """ - protocol = repo_info[ExternalsDescription.PROTOCOL].lower() - if protocol == 'git': - repo = GitRepository(component_name, repo_info) - elif protocol == 'svn': - repo = SvnRepository(component_name, repo_info, ignore_ancestry=svn_ignore_ancestry) - elif protocol == 'externals_only': - repo = None - else: - msg = 'Unknown repo protocol "{0}"'.format(protocol) - fatal_error(msg) - return repo diff --git a/util/manage_externals/manic/repository_git.py b/util/manage_externals/manic/repository_git.py deleted file mode 100644 index f986051001..0000000000 --- a/util/manage_externals/manic/repository_git.py +++ /dev/null @@ -1,819 +0,0 @@ -"""Class for interacting with git repositories -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import copy -import os - -from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR -from .global_constants import VERBOSITY_VERBOSE -from .repository import Repository -from .externals_status import ExternalStatus -from .externals_description import ExternalsDescription, git_submodule_status -from .utils import expand_local_url, split_remote_url, is_remote_url -from .utils import fatal_error, printlog -from .utils import execute_subprocess - - -class GitRepository(Repository): - """Class to represent and operate on a repository description. - - For testing purpose, all system calls to git should: - - * be isolated in separate functions with no application logic - * of the form: - - cmd = ['git', ...] - - value = execute_subprocess(cmd, output_to_caller={T|F}, - status_to_caller={T|F}) - - return value - * be static methods (not rely on self) - * name as _git_subcommand_args(user_args) - - This convention allows easy unit testing of the repository logic - by mocking the specific calls to return predefined results. - - """ - - def __init__(self, component_name, repo): - """ - Parse repo (a XML element). - """ - Repository.__init__(self, component_name, repo) - self._gitmodules = None - self._submods = None - - # ---------------------------------------------------------------- - # - # Public API, defined by Repository - # - # ---------------------------------------------------------------- - def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the source. - If the repo destination directory does not exist, checkout the correct - branch or tag. - """ - repo_dir_path = os.path.join(base_dir_path, repo_dir_name) - repo_dir_exists = os.path.exists(repo_dir_path) - if (repo_dir_exists and not os.listdir( - repo_dir_path)) or not repo_dir_exists: - self._clone_repo(base_dir_path, repo_dir_name, verbosity) - self._checkout_ref(repo_dir_path, verbosity, recursive) - gmpath = os.path.join(repo_dir_path, - ExternalsDescription.GIT_SUBMODULES_FILENAME) - if os.path.exists(gmpath): - self._gitmodules = gmpath - self._submods = git_submodule_status(repo_dir_path) - else: - self._gitmodules = None - self._submods = None - - def status(self, stat, repo_dir_path): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the source. - If the repo destination directory does not exist, checkout the correct - branch or tag. - """ - self._check_sync(stat, repo_dir_path) - if os.path.exists(repo_dir_path): - self._status_summary(stat, repo_dir_path) - - def submodules_file(self, repo_path=None): - if repo_path is not None: - gmpath = os.path.join(repo_path, - ExternalsDescription.GIT_SUBMODULES_FILENAME) - if os.path.exists(gmpath): - self._gitmodules = gmpath - self._submods = git_submodule_status(repo_path) - - return self._gitmodules - - # ---------------------------------------------------------------- - # - # Internal work functions - # - # ---------------------------------------------------------------- - def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): - """Prepare to execute the clone by managing directory location - """ - cwd = os.getcwd() - os.chdir(base_dir_path) - self._git_clone(self._url, repo_dir_name, verbosity) - os.chdir(cwd) - - def _current_ref(self): - """Determine the *name* associated with HEAD. - - If we're on a branch, then returns the branch name; otherwise, - if we're on a tag, then returns the tag name; otherwise, returns - the current hash. Returns an empty string if no reference can be - determined (e.g., if we're not actually in a git repository). - """ - ref_found = False - - # If we're on a branch, then use that as the current ref - branch_found, branch_name = self._git_current_branch() - if branch_found: - current_ref = branch_name - ref_found = True - - if not ref_found: - # Otherwise, if we're exactly at a tag, use that as the - # current ref - tag_found, tag_name = self._git_current_tag() - if tag_found: - current_ref = tag_name - ref_found = True - - if not ref_found: - # Otherwise, use current hash as the current ref - hash_found, hash_name = self._git_current_hash() - if hash_found: - current_ref = hash_name - ref_found = True - - if not ref_found: - # If we still can't find a ref, return empty string. This - # can happen if we're not actually in a git repo - current_ref = '' - - return current_ref - - def _check_sync(self, stat, repo_dir_path): - """Determine whether a git repository is in-sync with the model - description. - - Because repos can have multiple remotes, the only criteria is - whether the branch or tag is the same. - - """ - if not os.path.exists(repo_dir_path): - # NOTE(bja, 2017-10) condition should have been determined - # by _Source() object and should never be here! - stat.sync_state = ExternalStatus.STATUS_ERROR - else: - git_dir = os.path.join(repo_dir_path, '.git') - if not os.path.exists(git_dir): - # NOTE(bja, 2017-10) directory exists, but no git repo - # info.... Can't test with subprocess git command - # because git will move up directory tree until it - # finds the parent repo git dir! - stat.sync_state = ExternalStatus.UNKNOWN - else: - self._check_sync_logic(stat, repo_dir_path) - - def _check_sync_logic(self, stat, repo_dir_path): - """Compare the underlying hashes of the currently checkout ref and the - expected ref. - - Output: sets the sync_state as well as the current and - expected ref in the input status object. - - """ - def compare_refs(current_ref, expected_ref): - """Compare the current and expected ref. - - """ - if current_ref == expected_ref: - status = ExternalStatus.STATUS_OK - else: - status = ExternalStatus.MODEL_MODIFIED - return status - - cwd = os.getcwd() - os.chdir(repo_dir_path) - - # get the full hash of the current commit - _, current_ref = self._git_current_hash() - - if self._branch: - if self._url == LOCAL_PATH_INDICATOR: - expected_ref = self._branch - else: - remote_name = self._determine_remote_name() - if not remote_name: - # git doesn't know about this remote. by definition - # this is a modified state. - expected_ref = "unknown_remote/{0}".format(self._branch) - else: - expected_ref = "{0}/{1}".format(remote_name, self._branch) - elif self._hash: - expected_ref = self._hash - elif self._tag: - expected_ref = self._tag - else: - msg = 'In repo "{0}": none of branch, hash or tag are set'.format( - self._name) - fatal_error(msg) - - # record the *names* of the current and expected branches - stat.current_version = self._current_ref() - stat.expected_version = copy.deepcopy(expected_ref) - - if current_ref == EMPTY_STR: - stat.sync_state = ExternalStatus.UNKNOWN - else: - # get the underlying hash of the expected ref - revparse_status, expected_ref_hash = self._git_revparse_commit( - expected_ref) - if revparse_status: - # We failed to get the hash associated with - # expected_ref. Maybe we should assign this to some special - # status, but for now we're just calling this out-of-sync to - # remain consistent with how this worked before. - stat.sync_state = ExternalStatus.MODEL_MODIFIED - else: - # compare the underlying hashes - stat.sync_state = compare_refs(current_ref, expected_ref_hash) - - os.chdir(cwd) - - def _determine_remote_name(self): - """Return the remote name. - - Note that this is for the *future* repo url and branch, not - the current working copy! - - """ - git_output = self._git_remote_verbose() - git_output = git_output.splitlines() - remote_name = '' - for line in git_output: - data = line.strip() - if not data: - continue - data = data.split() - name = data[0].strip() - url = data[1].strip() - if self._url == url: - remote_name = name - break - return remote_name - - def _create_remote_name(self): - """The url specified in the externals description file was not known - to git. We need to add it, which means adding a unique and - safe name.... - - The assigned name needs to be safe for git to use, e.g. can't - look like a path 'foo/bar' and work with both remote and local paths. - - Remote paths include but are not limited to: git, ssh, https, - github, gitlab, bitbucket, custom server, etc. - - Local paths can be relative or absolute. They may contain - shell variables, e.g. ${REPO_ROOT}/repo_name, or username - expansion, i.e. ~/ or ~someuser/. - - Relative paths must be at least one layer of redirection, i.e. - container/../ext_repo, but may be many layers deep, e.g. - container/../../../../../ext_repo - - NOTE(bja, 2017-11) - - The base name below may not be unique, for example if the - user has local paths like: - - /path/to/my/repos/nice_repo - /path/to/other/repos/nice_repo - - But the current implementation should cover most common - use cases for remotes and still provide usable names. - - """ - url = copy.deepcopy(self._url) - if is_remote_url(url): - url = split_remote_url(url) - else: - url = expand_local_url(url, self._name) - url = url.split('/') - repo_name = url[-1] - base_name = url[-2] - # repo name should nominally already be something that git can - # deal with. We need to remove other possibly troublesome - # punctuation, e.g. /, $, from the base name. - unsafe_characters = '!@#$%^&*()[]{}\\/,;~' - for unsafe in unsafe_characters: - base_name = base_name.replace(unsafe, '') - remote_name = "{0}_{1}".format(base_name, repo_name) - return remote_name - - def _checkout_ref(self, repo_dir, verbosity, submodules): - """Checkout the user supplied reference - if is True, recursively initialize and update - the repo's submodules - """ - # import pdb; pdb.set_trace() - cwd = os.getcwd() - os.chdir(repo_dir) - if self._url.strip() == LOCAL_PATH_INDICATOR: - self._checkout_local_ref(verbosity, submodules) - else: - self._checkout_external_ref(verbosity, submodules) - - if self._sparse: - self._sparse_checkout(repo_dir, verbosity) - os.chdir(cwd) - - - def _checkout_local_ref(self, verbosity, submodules): - """Checkout the reference considering the local repo only. Do not - fetch any additional remotes or specify the remote when - checkout out the ref. - if is True, recursively initialize and update - the repo's submodules - """ - if self._tag: - ref = self._tag - elif self._branch: - ref = self._branch - else: - ref = self._hash - - self._check_for_valid_ref(ref) - self._git_checkout_ref(ref, verbosity, submodules) - - def _checkout_external_ref(self, verbosity, submodules): - """Checkout the reference from a remote repository - if is True, recursively initialize and update - the repo's submodules - """ - if self._tag: - ref = self._tag - elif self._branch: - ref = self._branch - else: - ref = self._hash - - remote_name = self._determine_remote_name() - if not remote_name: - remote_name = self._create_remote_name() - self._git_remote_add(remote_name, self._url) - self._git_fetch(remote_name) - - # NOTE(bja, 2018-03) we need to send separate ref and remote - # name to check_for_vaild_ref, but the combined name to - # checkout_ref! - self._check_for_valid_ref(ref, remote_name) - - if self._branch: - ref = '{0}/{1}'.format(remote_name, ref) - self._git_checkout_ref(ref, verbosity, submodules) - - def _sparse_checkout(self, repo_dir, verbosity): - """Use git read-tree to thin the working tree.""" - cwd = os.getcwd() - - cmd = ['cp', self._sparse, os.path.join(repo_dir, - '.git/info/sparse-checkout')] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) - os.chdir(repo_dir) - self._git_sparse_checkout(verbosity) - - os.chdir(cwd) - - def _check_for_valid_ref(self, ref, remote_name=None): - """Try some basic sanity checks on the user supplied reference so we - can provide a more useful error message than calledprocess - error... - - """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) - - is_valid = is_tag or is_branch or is_hash - if not is_valid: - msg = ('In repo "{0}": reference "{1}" does not appear to be a ' - 'valid tag, branch or hash! Please verify the reference ' - 'name (e.g. spelling), is available from: {2} '.format( - self._name, ref, self._url)) - fatal_error(msg) - - if is_tag: - is_unique_tag, msg = self._is_unique_tag(ref, remote_name) - if not is_unique_tag: - msg = ('In repo "{0}": tag "{1}" {2}'.format( - self._name, self._tag, msg)) - fatal_error(msg) - - return is_valid - - def _is_unique_tag(self, ref, remote_name): - """Verify that a reference is a valid tag and is unique (not a branch) - - Tags may be tag names, or SHA id's. It is also possible that a - branch and tag have the some name. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - - """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) - - msg = '' - is_unique_tag = False - if is_tag and not is_branch: - # unique tag - msg = 'is ok' - is_unique_tag = True - elif is_tag and is_branch: - msg = ('is both a branch and a tag. git may checkout the branch ' - 'instead of the tag depending on your version of git.') - is_unique_tag = False - elif not is_tag and is_branch: - msg = ('is a branch, and not a tag. If you intended to checkout ' - 'a branch, please change the externals description to be ' - 'a branch. If you intended to checkout a tag, it does not ' - 'exist. Please check the name.') - is_unique_tag = False - else: # not is_tag and not is_branch: - if is_hash: - # probably a sha1 or HEAD, etc, we call it a tag - msg = 'is ok' - is_unique_tag = True - else: - # undetermined state. - msg = ('does not appear to be a valid tag, branch or hash! ' - 'Please check the name and repository.') - is_unique_tag = False - - return is_unique_tag, msg - - def _ref_is_tag(self, ref): - """Verify that a reference is a valid tag according to git. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - """ - is_tag = False - value = self._git_showref_tag(ref) - if value == 0: - is_tag = True - return is_tag - - def _ref_is_branch(self, ref, remote_name=None): - """Verify if a ref is any kind of branch (local, tracked remote, - untracked remote). - - """ - local_branch = False - remote_branch = False - if remote_name: - remote_branch = self._ref_is_remote_branch(ref, remote_name) - local_branch = self._ref_is_local_branch(ref) - - is_branch = False - if local_branch or remote_branch: - is_branch = True - return is_branch - - def _ref_is_local_branch(self, ref): - """Verify that a reference is a valid branch according to git. - - show-ref branch returns local branches that have been - previously checked out. It will not necessarily pick up - untracked remote branches. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - - """ - is_branch = False - value = self._git_showref_branch(ref) - if value == 0: - is_branch = True - return is_branch - - def _ref_is_remote_branch(self, ref, remote_name): - """Verify that a reference is a valid branch according to git. - - show-ref branch returns local branches that have been - previously checked out. It will not necessarily pick up - untracked remote branches. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - - """ - is_branch = False - value = self._git_lsremote_branch(ref, remote_name) - if value == 0: - is_branch = True - return is_branch - - def _ref_is_commit(self, ref): - """Verify that a reference is a valid commit according to git. - - This could be a tag, branch, sha1 id, HEAD and potentially others... - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - """ - is_commit = False - value, _ = self._git_revparse_commit(ref) - if value == 0: - is_commit = True - return is_commit - - def _ref_is_hash(self, ref): - """Verify that a reference is a valid hash according to git. - - Git doesn't seem to provide an exact way to determine if user - supplied reference is an actual hash. So we verify that the - ref is a valid commit and return the underlying commit - hash. Then check that the commit hash begins with the user - supplied string. - - Note: values returned by git_showref_* and git_revparse are - shell return codes, which are zero for success, non-zero for - error! - - """ - is_hash = False - status, git_output = self._git_revparse_commit(ref) - if status == 0: - if git_output.strip().startswith(ref): - is_hash = True - return is_hash - - def _status_summary(self, stat, repo_dir_path): - """Determine the clean/dirty status of a git repository - - """ - cwd = os.getcwd() - os.chdir(repo_dir_path) - git_output = self._git_status_porcelain_v1z() - is_dirty = self._status_v1z_is_dirty(git_output) - if is_dirty: - stat.clean_state = ExternalStatus.DIRTY - else: - stat.clean_state = ExternalStatus.STATUS_OK - - # Now save the verbose status output incase the user wants to - # see it. - stat.status_output = self._git_status_verbose() - os.chdir(cwd) - - @staticmethod - def _status_v1z_is_dirty(git_output): - """Parse the git status output from --porcelain=v1 -z and determine if - the repo status is clean or dirty. Dirty means: - - * modified files - * missing files - * added files - * removed - * renamed - * unmerged - - Whether untracked files are considered depends on how the status - command was run (i.e., whether it was run with the '-u' option). - - NOTE: Based on the above definition, the porcelain status - should be an empty string to be considered 'clean'. Of course - this assumes we only get an empty string from an status - command on a clean checkout, and not some error - condition... Could alse use 'git diff --quiet'. - - """ - is_dirty = False - if git_output: - is_dirty = True - return is_dirty - - # ---------------------------------------------------------------- - # - # system call to git for information gathering - # - # ---------------------------------------------------------------- - @staticmethod - def _git_current_hash(): - """Return the full hash of the currently checked-out version. - - Returns a tuple, (hash_found, hash), where hash_found is a - logical specifying whether a hash was found for HEAD (False - could mean we're not in a git repository at all). (If hash_found - is False, then hash is ''.) - """ - status, git_output = GitRepository._git_revparse_commit("HEAD") - hash_found = not status - if not hash_found: - git_output = '' - return hash_found, git_output - - @staticmethod - def _git_current_branch(): - """Determines the name of the current branch. - - Returns a tuple, (branch_found, branch_name), where branch_found - is a logical specifying whether a branch name was found for - HEAD. (If branch_found is False, then branch_name is ''.) - """ - cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] - status, git_output = execute_subprocess(cmd, - output_to_caller=True, - status_to_caller=True) - branch_found = not status - if branch_found: - git_output = git_output.strip() - else: - git_output = '' - return branch_found, git_output - - @staticmethod - def _git_current_tag(): - """Determines the name tag corresponding to HEAD (if any). - - Returns a tuple, (tag_found, tag_name), where tag_found is a - logical specifying whether we found a tag name corresponding to - HEAD. (If tag_found is False, then tag_name is ''.) - """ - # git describe --exact-match --tags HEAD - cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] - status, git_output = execute_subprocess(cmd, - output_to_caller=True, - status_to_caller=True) - tag_found = not status - if tag_found: - git_output = git_output.strip() - else: - git_output = '' - return tag_found, git_output - - @staticmethod - def _git_showref_tag(ref): - """Run git show-ref check if the user supplied ref is a tag. - - could also use git rev-parse --quiet --verify tagname^{tag} - """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/tags/{0}'.format(ref), ] - status = execute_subprocess(cmd, status_to_caller=True) - return status - - @staticmethod - def _git_showref_branch(ref): - """Run git show-ref check if the user supplied ref is a local or - tracked remote branch. - - """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/heads/{0}'.format(ref), ] - status = execute_subprocess(cmd, status_to_caller=True) - return status - - @staticmethod - def _git_lsremote_branch(ref, remote_name): - """Run git ls-remote to check if the user supplied ref is a remote - branch that is not being tracked - - """ - cmd = ['git', 'ls-remote', '--exit-code', '--heads', - remote_name, ref, ] - status = execute_subprocess(cmd, status_to_caller=True) - return status - - @staticmethod - def _git_revparse_commit(ref): - """Run git rev-parse to detect if a reference is a SHA, HEAD or other - valid commit. - - """ - cmd = ['git', 'rev-parse', '--quiet', '--verify', - '{0}^{1}'.format(ref, '{commit}'), ] - status, git_output = execute_subprocess(cmd, status_to_caller=True, - output_to_caller=True) - git_output = git_output.strip() - return status, git_output - - @staticmethod - def _git_status_porcelain_v1z(): - """Run git status to obtain repository information. - - This is run with '--untracked=no' to ignore untracked files. - - The machine-portable format that is guaranteed not to change - between git versions or *user configuration*. - - """ - cmd = ['git', 'status', '--untracked-files=no', '--porcelain', '-z'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output - - @staticmethod - def _git_status_verbose(): - """Run the git status command to obtain repository information. - """ - cmd = ['git', 'status'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output - - @staticmethod - def _git_remote_verbose(): - """Run the git remote command to obtain repository information. - """ - cmd = ['git', 'remote', '--verbose'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output - - @staticmethod - def has_submodules(repo_dir_path=None): - """Return True iff the repository at (or the current - directory if is None) has a '.gitmodules' file - """ - if repo_dir_path is None: - fname = ExternalsDescription.GIT_SUBMODULES_FILENAME - else: - fname = os.path.join(repo_dir_path, - ExternalsDescription.GIT_SUBMODULES_FILENAME) - - return os.path.exists(fname) - - # ---------------------------------------------------------------- - # - # system call to git for sideffects modifying the working tree - # - # ---------------------------------------------------------------- - @staticmethod - def _git_clone(url, repo_dir_name, verbosity): - """Run git clone for the side effect of creating a repository. - """ - cmd = ['git', 'clone', '--quiet'] - subcmd = None - - cmd.extend([url, repo_dir_name]) - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) - if subcmd is not None: - os.chdir(repo_dir_name) - execute_subprocess(subcmd) - - @staticmethod - def _git_remote_add(name, url): - """Run the git remote command for the side effect of adding a remote - """ - cmd = ['git', 'remote', 'add', name, url] - execute_subprocess(cmd) - - @staticmethod - def _git_fetch(remote_name): - """Run the git fetch command for the side effect of updating the repo - """ - cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] - execute_subprocess(cmd) - - @staticmethod - def _git_checkout_ref(ref, verbosity, submodules): - """Run the git checkout command for the side effect of updating the repo - - Param: ref is a reference to a local or remote object in the - form 'origin/my_feature', or 'tag1'. - - """ - cmd = ['git', 'checkout', '--quiet', ref] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) - if submodules: - GitRepository._git_update_submodules(verbosity) - - @staticmethod - def _git_sparse_checkout(verbosity): - """Configure repo via read-tree.""" - cmd = ['git', 'config', 'core.sparsecheckout', 'true'] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) - cmd = ['git', 'read-tree', '-mu', 'HEAD'] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) - - @staticmethod - def _git_update_submodules(verbosity): - """Run git submodule update for the side effect of updating this - repo's submodules. - """ - # First, verify that we have a .gitmodules file - if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): - cmd = ['git', 'submodule', 'update', '--init', '--recursive'] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - - execute_subprocess(cmd) diff --git a/util/manage_externals/manic/repository_svn.py b/util/manage_externals/manic/repository_svn.py deleted file mode 100644 index 2f0d4d848c..0000000000 --- a/util/manage_externals/manic/repository_svn.py +++ /dev/null @@ -1,284 +0,0 @@ -"""Class for interacting with svn repositories -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import os -import re -import xml.etree.ElementTree as ET - -from .global_constants import EMPTY_STR, VERBOSITY_VERBOSE -from .repository import Repository -from .externals_status import ExternalStatus -from .utils import fatal_error, indent_string, printlog -from .utils import execute_subprocess - - -class SvnRepository(Repository): - """ - Class to represent and operate on a repository description. - - For testing purpose, all system calls to svn should: - - * be isolated in separate functions with no application logic - * of the form: - - cmd = ['svn', ...] - - value = execute_subprocess(cmd, output_to_caller={T|F}, - status_to_caller={T|F}) - - return value - * be static methods (not rely on self) - * name as _svn_subcommand_args(user_args) - - This convention allows easy unit testing of the repository logic - by mocking the specific calls to return predefined results. - - """ - RE_URLLINE = re.compile(r'^URL:') - - def __init__(self, component_name, repo, ignore_ancestry=False): - """ - Parse repo (a XML element). - """ - Repository.__init__(self, component_name, repo) - self._ignore_ancestry = ignore_ancestry - if self._branch: - self._url = os.path.join(self._url, self._branch) - elif self._tag: - self._url = os.path.join(self._url, self._tag) - else: - msg = "DEV_ERROR in svn repository. Shouldn't be here!" - fatal_error(msg) - - # ---------------------------------------------------------------- - # - # Public API, defined by Repository - # - # ---------------------------------------------------------------- - def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument - """Checkout or update the working copy - - If the repo destination directory exists, switch the sandbox to - match the externals description. - - If the repo destination directory does not exist, checkout the - correct branch or tag. - NB: is include as an argument for compatibility with - git functionality (repository_git.py) - - """ - repo_dir_path = os.path.join(base_dir_path, repo_dir_name) - if os.path.exists(repo_dir_path): - cwd = os.getcwd() - os.chdir(repo_dir_path) - self._svn_switch(self._url, self._ignore_ancestry, verbosity) - # svn switch can lead to a conflict state, but it gives a - # return code of 0. So now we need to make sure that we're - # in a clean (non-conflict) state. - self._abort_if_dirty(repo_dir_path, - "Expected clean state following switch") - os.chdir(cwd) - else: - self._svn_checkout(self._url, repo_dir_path, verbosity) - - def status(self, stat, repo_dir_path): - """ - Check and report the status of the repository - """ - self._check_sync(stat, repo_dir_path) - if os.path.exists(repo_dir_path): - self._status_summary(stat, repo_dir_path) - - # ---------------------------------------------------------------- - # - # Internal work functions - # - # ---------------------------------------------------------------- - def _check_sync(self, stat, repo_dir_path): - """Check to see if repository directory exists and is at the expected - url. Return: status object - - """ - if not os.path.exists(repo_dir_path): - # NOTE(bja, 2017-10) this state should have been handled by - # the source object and we never get here! - stat.sync_state = ExternalStatus.STATUS_ERROR - else: - svn_output = self._svn_info(repo_dir_path) - if not svn_output: - # directory exists, but info returned nothing. .svn - # directory removed or incomplete checkout? - stat.sync_state = ExternalStatus.UNKNOWN - else: - stat.sync_state, stat.current_version = \ - self._check_url(svn_output, self._url) - stat.expected_version = '/'.join(self._url.split('/')[3:]) - - def _abort_if_dirty(self, repo_dir_path, message): - """Check if the repo is in a dirty state; if so, abort with a - helpful message. - - """ - - stat = ExternalStatus() - self._status_summary(stat, repo_dir_path) - if stat.clean_state != ExternalStatus.STATUS_OK: - status = self._svn_status_verbose(repo_dir_path) - status = indent_string(status, 4) - errmsg = """In directory - {cwd} - -svn status now shows: -{status} - -ERROR: {message} - -One possible cause of this problem is that there may have been untracked -files in your working directory that had the same name as tracked files -in the new revision. - -To recover: Clean up the above directory (resolving conflicts, etc.), -then rerun checkout_externals. -""".format(cwd=repo_dir_path, message=message, status=status) - - fatal_error(errmsg) - - @staticmethod - def _check_url(svn_output, expected_url): - """Determine the svn url from svn info output and return whether it - matches the expected value. - - """ - url = None - for line in svn_output.splitlines(): - if SvnRepository.RE_URLLINE.match(line): - url = line.split(': ')[1].strip() - break - if not url: - status = ExternalStatus.UNKNOWN - elif url == expected_url: - status = ExternalStatus.STATUS_OK - else: - status = ExternalStatus.MODEL_MODIFIED - - if url: - current_version = '/'.join(url.split('/')[3:]) - else: - current_version = EMPTY_STR - - return status, current_version - - def _status_summary(self, stat, repo_dir_path): - """Report whether the svn repository is in-sync with the model - description and whether the sandbox is clean or dirty. - - """ - svn_output = self._svn_status_xml(repo_dir_path) - is_dirty = self.xml_status_is_dirty(svn_output) - if is_dirty: - stat.clean_state = ExternalStatus.DIRTY - else: - stat.clean_state = ExternalStatus.STATUS_OK - - # Now save the verbose status output incase the user wants to - # see it. - stat.status_output = self._svn_status_verbose(repo_dir_path) - - @staticmethod - def xml_status_is_dirty(svn_output): - """Parse svn status xml output and determine if the working copy is - clean or dirty. Dirty is defined as: - - * modified files - * added files - * deleted files - * missing files - - Unversioned files do not affect the clean/dirty status. - - 'external' is also an acceptable state - - """ - # pylint: disable=invalid-name - SVN_EXTERNAL = 'external' - SVN_UNVERSIONED = 'unversioned' - # pylint: enable=invalid-name - - is_dirty = False - try: - xml_status = ET.fromstring(svn_output) - except BaseException: - fatal_error( - "SVN returned invalid XML message {}".format(svn_output)) - xml_target = xml_status.find('./target') - entries = xml_target.findall('./entry') - for entry in entries: - status = entry.find('./wc-status') - item = status.get('item') - if item == SVN_EXTERNAL: - continue - if item == SVN_UNVERSIONED: - continue - else: - is_dirty = True - break - return is_dirty - - # ---------------------------------------------------------------- - # - # system call to svn for information gathering - # - # ---------------------------------------------------------------- - @staticmethod - def _svn_info(repo_dir_path): - """Return results of svn info command - """ - cmd = ['svn', 'info', repo_dir_path] - output = execute_subprocess(cmd, output_to_caller=True) - return output - - @staticmethod - def _svn_status_verbose(repo_dir_path): - """capture the full svn status output - """ - cmd = ['svn', 'status', repo_dir_path] - svn_output = execute_subprocess(cmd, output_to_caller=True) - return svn_output - - @staticmethod - def _svn_status_xml(repo_dir_path): - """ - Get status of the subversion sandbox in repo_dir - """ - cmd = ['svn', 'status', '--xml', repo_dir_path] - svn_output = execute_subprocess(cmd, output_to_caller=True) - return svn_output - - # ---------------------------------------------------------------- - # - # system call to svn for sideffects modifying the working tree - # - # ---------------------------------------------------------------- - @staticmethod - def _svn_checkout(url, repo_dir_path, verbosity): - """ - Checkout a subversion repository (repo_url) to checkout_dir. - """ - cmd = ['svn', 'checkout', '--quiet', url, repo_dir_path] - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) - - @staticmethod - def _svn_switch(url, ignore_ancestry, verbosity): - """ - Switch branches for in an svn sandbox - """ - cmd = ['svn', 'switch', '--quiet'] - if ignore_ancestry: - cmd.append('--ignore-ancestry') - cmd.append(url) - if verbosity >= VERBOSITY_VERBOSE: - printlog(' {0}'.format(' '.join(cmd))) - execute_subprocess(cmd) diff --git a/util/manage_externals/manic/sourcetree.py b/util/manage_externals/manic/sourcetree.py deleted file mode 100644 index 3a63835c78..0000000000 --- a/util/manage_externals/manic/sourcetree.py +++ /dev/null @@ -1,351 +0,0 @@ -""" - -FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! -""" - -import errno -import logging -import os - -from .externals_description import ExternalsDescription -from .externals_description import read_externals_description_file -from .externals_description import create_externals_description -from .repository_factory import create_repository -from .repository_git import GitRepository -from .externals_status import ExternalStatus -from .utils import fatal_error, printlog -from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR -from .global_constants import VERBOSITY_VERBOSE - -class _External(object): - """ - _External represents an external object inside a SourceTree - """ - - # pylint: disable=R0902 - - def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): - """Parse an external description file into a dictionary of externals. - - Input: - - root_dir : string - the root directory path where - 'local_path' is relative to. - - name : string - name of the ext_description object. may or may not - correspond to something in the path. - - ext_description : dict - source ExternalsDescription object - - svn_ignore_ancestry : bool - use --ignore-externals with svn switch - - """ - self._name = name - self._repo = None - self._externals = EMPTY_STR - self._externals_sourcetree = None - self._stat = ExternalStatus() - self._sparse = None - # Parse the sub-elements - - # _path : local path relative to the containing source tree - self._local_path = ext_description[ExternalsDescription.PATH] - # _repo_dir : full repository directory - repo_dir = os.path.join(root_dir, self._local_path) - self._repo_dir_path = os.path.abspath(repo_dir) - # _base_dir : base directory *containing* the repository - self._base_dir_path = os.path.dirname(self._repo_dir_path) - # repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path - self._repo_dir_name = os.path.basename(self._repo_dir_path) - assert(os.path.join(self._base_dir_path, self._repo_dir_name) - == self._repo_dir_path) - - self._required = ext_description[ExternalsDescription.REQUIRED] - self._externals = ext_description[ExternalsDescription.EXTERNALS] - # Treat a .gitmodules file as a backup externals config - if not self._externals: - if GitRepository.has_submodules(self._repo_dir_path): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME - - repo = create_repository( - name, ext_description[ExternalsDescription.REPO], - svn_ignore_ancestry=svn_ignore_ancestry) - if repo: - self._repo = repo - - if self._externals and (self._externals.lower() != 'none'): - self._create_externals_sourcetree() - - def get_name(self): - """ - Return the external object's name - """ - return self._name - - def get_local_path(self): - """ - Return the external object's path - """ - return self._local_path - - def status(self): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce - branch or tag. - If load_all is True, also load all of the the externals sub-externals. - """ - - self._stat.path = self.get_local_path() - if not self._required: - self._stat.source_type = ExternalStatus.OPTIONAL - elif self._local_path == LOCAL_PATH_INDICATOR: - # LOCAL_PATH_INDICATOR, '.' paths, are standalone - # component directories that are not managed by - # checkout_externals. - self._stat.source_type = ExternalStatus.STANDALONE - else: - # managed by checkout_externals - self._stat.source_type = ExternalStatus.MANAGED - - ext_stats = {} - - if not os.path.exists(self._repo_dir_path): - self._stat.sync_state = ExternalStatus.EMPTY - msg = ('status check: repository directory for "{0}" does not ' - 'exist.'.format(self._name)) - logging.info(msg) - self._stat.current_version = 'not checked out' - # NOTE(bja, 2018-01) directory doesn't exist, so we cannot - # use repo to determine the expected version. We just take - # a best-guess based on the assumption that only tag or - # branch should be set, but not both. - if not self._repo: - self._stat.expected_version = 'unknown' - else: - self._stat.expected_version = self._repo.tag() + self._repo.branch() - else: - if self._repo: - self._repo.status(self._stat, self._repo_dir_path) - - if self._externals and self._externals_sourcetree: - # we expect externals and they exist - cwd = os.getcwd() - # SourceTree expects to be called from the correct - # root directory. - os.chdir(self._repo_dir_path) - ext_stats = self._externals_sourcetree.status(self._local_path) - os.chdir(cwd) - - all_stats = {} - # don't add the root component because we don't manage it - # and can't provide useful info about it. - if self._local_path != LOCAL_PATH_INDICATOR: - # store the stats under tha local_path, not comp name so - # it will be sorted correctly - all_stats[self._stat.path] = self._stat - - if ext_stats: - all_stats.update(ext_stats) - - return all_stats - - def checkout(self, verbosity, load_all): - """ - If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correct - branch or tag. - If load_all is True, also load all of the the externals sub-externals. - """ - if load_all: - pass - # Make sure we are in correct location - - if not os.path.exists(self._repo_dir_path): - # repository directory doesn't exist. Need to check it - # out, and for that we need the base_dir_path to exist - try: - os.makedirs(self._base_dir_path) - except OSError as error: - if error.errno != errno.EEXIST: - msg = 'Could not create directory "{0}"'.format( - self._base_dir_path) - fatal_error(msg) - - if self._stat.source_type != ExternalStatus.STANDALONE: - if verbosity >= VERBOSITY_VERBOSE: - # NOTE(bja, 2018-01) probably do not want to pass - # verbosity in this case, because if (verbosity == - # VERBOSITY_DUMP), then the previous status output would - # also be dumped, adding noise to the output. - self._stat.log_status_message(VERBOSITY_VERBOSE) - - if self._repo: - if self._stat.sync_state == ExternalStatus.STATUS_OK: - # If we're already in sync, avoid showing verbose output - # from the checkout command, unless the verbosity level - # is 2 or more. - checkout_verbosity = verbosity - 1 - else: - checkout_verbosity = verbosity - - self._repo.checkout(self._base_dir_path, self._repo_dir_name, - checkout_verbosity, self.clone_recursive()) - - def checkout_externals(self, verbosity, load_all): - """Checkout the sub-externals for this object - """ - if self.load_externals(): - if self._externals_sourcetree: - # NOTE(bja, 2018-02): the subtree externals objects - # were created during initial status check. Updating - # the external may have changed which sub-externals - # are needed. We need to delete those objects and - # re-read the potentially modified externals - # description file. - self._externals_sourcetree = None - self._create_externals_sourcetree() - self._externals_sourcetree.checkout(verbosity, load_all) - - def load_externals(self): - 'Return True iff an externals file should be loaded' - load_ex = False - if os.path.exists(self._repo_dir_path): - if self._externals: - if self._externals.lower() != 'none': - load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals)) - - return load_ex - - def clone_recursive(self): - 'Return True iff any .gitmodules files should be processed' - # Try recursive unless there is an externals entry - recursive = not self._externals - - return recursive - - def _create_externals_sourcetree(self): - """ - """ - if not os.path.exists(self._repo_dir_path): - # NOTE(bja, 2017-10) repository has not been checked out - # yet, can't process the externals file. Assume we are - # checking status before code is checkoud out and this - # will be handled correctly later. - return - - cwd = os.getcwd() - os.chdir(self._repo_dir_path) - if self._externals.lower() == 'none': - msg = ('Internal: Attempt to create source tree for ' - 'externals = none in {}'.format(self._repo_dir_path)) - fatal_error(msg) - - if not os.path.exists(self._externals): - if GitRepository.has_submodules(): - self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME - - if not os.path.exists(self._externals): - # NOTE(bja, 2017-10) this check is redundent with the one - # in read_externals_description_file! - msg = ('External externals description file "{0}" ' - 'does not exist! In directory: {1}'.format( - self._externals, self._repo_dir_path)) - fatal_error(msg) - - externals_root = self._repo_dir_path - model_data = read_externals_description_file(externals_root, - self._externals) - externals = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, externals) - os.chdir(cwd) - -class SourceTree(object): - """ - SourceTree represents a group of managed externals - """ - - def __init__(self, root_dir, model, svn_ignore_ancestry=False): - """ - Build a SourceTree object from a model description - """ - self._root_dir = os.path.abspath(root_dir) - self._all_components = {} - self._required_compnames = [] - for comp in model: - src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) - self._all_components[comp] = src - if model[comp][ExternalsDescription.REQUIRED]: - self._required_compnames.append(comp) - - def status(self, relative_path_base=LOCAL_PATH_INDICATOR): - """Report the status components - - FIXME(bja, 2017-10) what do we do about situations where the - user checked out the optional components, but didn't add - optional for running status? What do we do where the user - didn't add optional to the checkout but did add it to the - status. -- For now, we run status on all components, and try - to do the right thing based on the results.... - - """ - load_comps = self._all_components.keys() - - summary = {} - for comp in load_comps: - printlog('{0}, '.format(comp), end='') - stat = self._all_components[comp].status() - for name in stat.keys(): - # check if we need to append the relative_path_base to - # the path so it will be sorted in the correct order. - if not stat[name].path.startswith(relative_path_base): - stat[name].path = os.path.join(relative_path_base, - stat[name].path) - # store under key = updated path, and delete the - # old key. - comp_stat = stat[name] - del stat[name] - stat[comp_stat.path] = comp_stat - summary.update(stat) - - return summary - - def checkout(self, verbosity, load_all, load_comp=None): - """ - Checkout or update indicated components into the the configured - subdirs. - - If load_all is True, recursively checkout all externals. - If load_all is False, load_comp is an optional set of components to load. - If load_all is True and load_comp is None, only load the required externals. - """ - if verbosity >= VERBOSITY_VERBOSE: - printlog('Checking out externals: ') - else: - printlog('Checking out externals: ', end='') - - if load_all: - load_comps = self._all_components.keys() - elif load_comp is not None: - load_comps = [load_comp] - else: - load_comps = self._required_compnames - - # checkout the primary externals - for comp in load_comps: - if verbosity < VERBOSITY_VERBOSE: - printlog('{0}, '.format(comp), end='') - else: - # verbose output handled by the _External object, just - # output a newline - printlog(EMPTY_STR) - self._all_components[comp].checkout(verbosity, load_all) - printlog('') - - # now give each external an opportunitity to checkout it's externals. - for comp in load_comps: - self._all_components[comp].checkout_externals(verbosity, load_all) diff --git a/util/manage_externals/manic/utils.py b/util/manage_externals/manic/utils.py deleted file mode 100644 index f57f43930c..0000000000 --- a/util/manage_externals/manic/utils.py +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python -""" -Common public utilities for manic package - -""" - -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function - -import logging -import os -import subprocess -import sys -from threading import Timer - -from .global_constants import LOCAL_PATH_INDICATOR - -# --------------------------------------------------------------------- -# -# screen and logging output and functions to massage text for output -# -# --------------------------------------------------------------------- - - -def log_process_output(output): - """Log each line of process output at debug level so it can be - filtered if necessary. By default, output is a single string, and - logging.debug(output) will only put log info heading on the first - line. This makes it hard to filter with grep. - - """ - output = output.split('\n') - for line in output: - logging.debug(line) - - -def printlog(msg, **kwargs): - """Wrapper script around print to ensure that everything printed to - the screen also gets logged. - - """ - logging.info(msg) - if kwargs: - print(msg, **kwargs) - else: - print(msg) - sys.stdout.flush() - - -def last_n_lines(the_string, n_lines, truncation_message=None): - """Returns the last n lines of the given string - - Args: - the_string: str - n_lines: int - truncation_message: str, optional - - Returns a string containing the last n lines of the_string - - If truncation_message is provided, the returned string begins with - the given message if and only if the string is greater than n lines - to begin with. - """ - - lines = the_string.splitlines(True) - if len(lines) <= n_lines: - return_val = the_string - else: - lines_subset = lines[-n_lines:] - str_truncated = ''.join(lines_subset) - if truncation_message: - str_truncated = truncation_message + '\n' + str_truncated - return_val = str_truncated - - return return_val - - -def indent_string(the_string, indent_level): - """Indents the given string by a given number of spaces - - Args: - the_string: str - indent_level: int - - Returns a new string that is the same as the_string, except that - each line is indented by 'indent_level' spaces. - - In python3, this can be done with textwrap.indent. - """ - - lines = the_string.splitlines(True) - padding = ' ' * indent_level - lines_indented = [padding + line for line in lines] - return ''.join(lines_indented) - -# --------------------------------------------------------------------- -# -# error handling -# -# --------------------------------------------------------------------- - - -def fatal_error(message): - """ - Error output function - """ - logging.error(message) - raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) - - -# --------------------------------------------------------------------- -# -# Data conversion / manipulation -# -# --------------------------------------------------------------------- -def str_to_bool(bool_str): - """Convert a sting representation of as boolean into a true boolean. - - Conversion should be case insensitive. - """ - value = None - str_lower = bool_str.lower() - if str_lower in ('true', 't'): - value = True - elif str_lower in ('false', 'f'): - value = False - if value is None: - msg = ('ERROR: invalid boolean string value "{0}". ' - 'Must be "true" or "false"'.format(bool_str)) - fatal_error(msg) - return value - - -REMOTE_PREFIXES = ['http://', 'https://', 'ssh://', 'git@'] - - -def is_remote_url(url): - """check if the user provided a local file path instead of a - remote. If so, it must be expanded to an absolute - path. - - """ - remote_url = False - for prefix in REMOTE_PREFIXES: - if url.startswith(prefix): - remote_url = True - return remote_url - - -def split_remote_url(url): - """check if the user provided a local file path or a - remote. If remote, try to strip off protocol info. - - """ - remote_url = is_remote_url(url) - if not remote_url: - return url - - for prefix in REMOTE_PREFIXES: - url = url.replace(prefix, '') - - if '@' in url: - url = url.split('@')[1] - - if ':' in url: - url = url.split(':')[1] - - return url - - -def expand_local_url(url, field): - """check if the user provided a local file path instead of a - remote. If so, it must be expanded to an absolute - path. - - Note: local paths of LOCAL_PATH_INDICATOR have special meaning and - represent local copy only, don't work with the remotes. - - """ - remote_url = is_remote_url(url) - if not remote_url: - if url.strip() == LOCAL_PATH_INDICATOR: - pass - else: - url = os.path.expandvars(url) - url = os.path.expanduser(url) - if not os.path.isabs(url): - msg = ('WARNING: Externals description for "{0}" contains a ' - 'url that is not remote and does not expand to an ' - 'absolute path. Version control operations may ' - 'fail.\n\nurl={1}'.format(field, url)) - printlog(msg) - else: - url = os.path.normpath(url) - return url - - -# --------------------------------------------------------------------- -# -# subprocess -# -# --------------------------------------------------------------------- - -# Give the user a helpful message if we detect that a command seems to -# be hanging. -_HANGING_SEC = 300 - - -def _hanging_msg(working_directory, command): - print(""" - -Command '{command}' -from directory {working_directory} -has taken {hanging_sec} seconds. It may be hanging. - -The command will continue to run, but you may want to abort -manage_externals with ^C and investigate. A possible cause of hangs is -when svn or git require authentication to access a private -repository. On some systems, svn and git requests for authentication -information will not be displayed to the user. In this case, the program -will appear to hang. Ensure you can run svn and git manually and access -all repositories without entering your authentication information. - -""".format(command=command, - working_directory=working_directory, - hanging_sec=_HANGING_SEC)) - - -def execute_subprocess(commands, status_to_caller=False, - output_to_caller=False): - """Wrapper around subprocess.check_output to handle common - exceptions. - - check_output runs a command with arguments and waits - for it to complete. - - check_output raises an exception on a nonzero return code. if - status_to_caller is true, execute_subprocess returns the subprocess - return code, otherwise execute_subprocess treats non-zero return - status as an error and raises an exception. - - """ - cwd = os.getcwd() - msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd) - logging.info(msg) - commands_str = ' '.join(commands) - logging.info(commands_str) - return_to_caller = status_to_caller or output_to_caller - status = -1 - output = '' - hanging_timer = Timer(_HANGING_SEC, _hanging_msg, - kwargs={"working_directory": cwd, - "command": commands_str}) - hanging_timer.start() - try: - output = subprocess.check_output(commands, stderr=subprocess.STDOUT, - universal_newlines=True) - log_process_output(output) - status = 0 - except OSError as error: - msg = failed_command_msg( - 'Command execution failed. Does the executable exist?', - commands) - logging.error(error) - fatal_error(msg) - except ValueError as error: - msg = failed_command_msg( - 'DEV_ERROR: Invalid arguments trying to run subprocess', - commands) - logging.error(error) - fatal_error(msg) - except subprocess.CalledProcessError as error: - # Only report the error if we are NOT returning to the - # caller. If we are returning to the caller, then it may be a - # simple status check. If returning, it is the callers - # responsibility determine if an error occurred and handle it - # appropriately. - if not return_to_caller: - msg_context = ('Process did not run successfully; ' - 'returned status {0}'.format(error.returncode)) - msg = failed_command_msg(msg_context, commands, - output=error.output) - logging.error(error) - logging.error(msg) - log_process_output(error.output) - fatal_error(msg) - status = error.returncode - finally: - hanging_timer.cancel() - - if status_to_caller and output_to_caller: - ret_value = (status, output) - elif status_to_caller: - ret_value = status - elif output_to_caller: - ret_value = output - else: - ret_value = None - - return ret_value - - -def failed_command_msg(msg_context, command, output=None): - """Template for consistent error messages from subprocess calls. - - If 'output' is given, it should provide the output from the failed - command - """ - - if output: - output_truncated = last_n_lines(output, 20, - truncation_message='[... Output truncated for brevity ...]') - errmsg = ('Failed with output:\n' + - indent_string(output_truncated, 4) + - '\nERROR: ') - else: - errmsg = '' - - command_str = ' '.join(command) - errmsg += """In directory - {cwd} -{context}: - {command} -""".format(cwd=os.getcwd(), context=msg_context, command=command_str) - - if output: - errmsg += 'See above for output from failed command.\n' - - return errmsg From d9ea1acab54f65a987b32d56587dfd1b6bcd037c Mon Sep 17 00:00:00 2001 From: "Kate.Friedman" Date: Thu, 6 Feb 2020 16:03:11 +0000 Subject: [PATCH 03/11] Issue #3 - reduce hashes down to minimum 8 characters --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 9fb968a378..297ba94bd9 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -8,14 +8,14 @@ protocol = git required = True [GSI] -hash = cb8f69d82f38dcf85669b45aaf95dad068f0103c +hash = cb8f69d8 local_path = sorc/gsi.fd repo_url = ssh://vlab.ncep.noaa.gov:29418/ProdGSI protocol = git required = True [EMC_post] -hash = ba7e59b290c8149ff1c2fee98d01e99e4ef92ee6 +hash = ba7e59b2 local_path = sorc/gfs_post.fd repo_url = https://github.com/NOAA-EMC/EMC_post.git protocol = git From 4bd0e20300cc2a79e79433b2ec8cdb15c8f01c9e Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 6 Feb 2020 11:55:31 -0500 Subject: [PATCH 04/11] Update README.md --- README.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/README.md b/README.md index 6c3f627e53..f99d3e0d15 100644 --- a/README.md +++ b/README.md @@ -3,5 +3,47 @@ Global Superstructure/Workflow currently supporting the Finite-Volume on a Cubed The global-workflow depends on the following prerequisities to be available on the system: +* workload management platform / scheduler - LSF or SLURM +* workflow manager - ROCOTO (https://github.com/christopherwharrop/rocoto) +* modules - NCEPLIBS (various), esmf v8.0.0bs48, hdf5, intel/ips v18, impi v18, wgrib2, netcdf v4.7.0, hpss, gempak (see module files under /modulefiles for additional details) * manage_externals - A utility from ESMCI to checkout external dependencies. Manage_externals can be obtained at the following address and should in the users PATH: https://github.com/ESMCI/manage_externals +The global-workflow current supports the following machines: + +* WCOSS-Dell +* WCOSS-Cray +* Hera + +## Build global-workflow: + +### 1. Check out components + +The global-workflow uses the manage_externals utility to handle checking out its components. The manic-v1.1.6 manage_externals tag is supported. + +Run manage_externals (checkout_externals) while at top of clone: + +``` +$ checkout_externals +``` + +If checkout_externals is not in PATH then use full path to it: + +* WCOSS-Dell: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/manage_externals/manic-v1.1.6/checkout_externals +* WCOSS-Cray: /gpfs/hps3/emc/global/noscrub/emc.glopara/git/manage_externals/manic-v1.1.6/checkout_externals +* Hera: /scratch1/NCEPDEV/global/glopara/git/manage_externals/manic-v1.1.6/checkout_externals + +### 2. Build components + +While in /sorc folder: +``` +$ sh build_all.sh +``` + +### 3. Link components + +While in /sorc folder: +``` +$ sh link_fv3gfs.sh emc $MACHINE +``` + +...where $MACHINE is "dell", "cray", or "hera". From e3196a84a0ecd2b54d59abfdc9184622a9c605ca Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 13 Feb 2020 15:59:06 -0500 Subject: [PATCH 05/11] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f99d3e0d15..c6d4867f36 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The global-workflow depends on the following prerequisities to be available on t * workload management platform / scheduler - LSF or SLURM * workflow manager - ROCOTO (https://github.com/christopherwharrop/rocoto) * modules - NCEPLIBS (various), esmf v8.0.0bs48, hdf5, intel/ips v18, impi v18, wgrib2, netcdf v4.7.0, hpss, gempak (see module files under /modulefiles for additional details) -* manage_externals - A utility from ESMCI to checkout external dependencies. Manage_externals can be obtained at the following address and should in the users PATH: https://github.com/ESMCI/manage_externals +* manage_externals - A utility from ESMCI to checkout external dependencies. Manage_externals can be obtained at the following address and should be in the users PATH: https://github.com/ESMCI/manage_externals The global-workflow current supports the following machines: @@ -26,7 +26,7 @@ Run manage_externals (checkout_externals) while at top of clone: $ checkout_externals ``` -If checkout_externals is not in PATH then use full path to it: +If checkout_externals is not in your $PATH then use full path to it: * WCOSS-Dell: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/manage_externals/manic-v1.1.6/checkout_externals * WCOSS-Cray: /gpfs/hps3/emc/global/noscrub/emc.glopara/git/manage_externals/manic-v1.1.6/checkout_externals From f662fffa25a99617828e4322bf789978cf523248 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 14 Feb 2020 15:57:05 +0000 Subject: [PATCH 06/11] Issue #3 - Updated README with new manic tag v1.1.7 --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c6d4867f36..d3e69b820f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The global-workflow current supports the following machines: ### 1. Check out components -The global-workflow uses the manage_externals utility to handle checking out its components. The manic-v1.1.6 manage_externals tag is supported. +The global-workflow uses the manage_externals utility to handle checking out its components. The manic-v1.1.7 manage_externals tag is supported. Run manage_externals (checkout_externals) while at top of clone: @@ -28,9 +28,9 @@ $ checkout_externals If checkout_externals is not in your $PATH then use full path to it: -* WCOSS-Dell: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/manage_externals/manic-v1.1.6/checkout_externals -* WCOSS-Cray: /gpfs/hps3/emc/global/noscrub/emc.glopara/git/manage_externals/manic-v1.1.6/checkout_externals -* Hera: /scratch1/NCEPDEV/global/glopara/git/manage_externals/manic-v1.1.6/checkout_externals +* WCOSS-Dell: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/manage_externals/manic-v1.1.7/checkout_externals +* WCOSS-Cray: /gpfs/hps3/emc/global/noscrub/emc.glopara/git/manage_externals/manic-v1.1.7/checkout_externals +* Hera: /scratch1/NCEPDEV/global/glopara/git/manage_externals/manic-v1.1.7/checkout_externals ### 2. Build components From 830c73f430d70cc516dea419a8969c6fd9fc0910 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 6 Mar 2020 15:21:45 +0000 Subject: [PATCH 07/11] Issue #3 - update EMC_verif-global tag in Externals.cfg after sync with develop --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 297ba94bd9..c58ec88ac9 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -29,7 +29,7 @@ protocol = git required = True [EMC_verif-global] -tag = verif_global_v1.2.2 +tag = verif_global_v1.5.0 local_path = sorc/verif-global.fd repo_url = ssh://vlab.ncep.noaa.gov:29418/EMC_verif-global protocol = git From e602cd3d536b55b86b04285aedd5781d8d3a9f82 Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 6 Mar 2020 16:27:57 +0000 Subject: [PATCH 08/11] Issue #3 - updated link_fv3gfs.sh to adjust wafs links --- sorc/link_fv3gfs.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh index 8671b3dba0..1e7ef263a8 100755 --- a/sorc/link_fv3gfs.sh +++ b/sorc/link_fv3gfs.sh @@ -74,10 +74,10 @@ cd ${pwd}/../util ||exit 8 done -#------------------------------ -#--add gfs_wafs link if on Dell -if [ $machine = dell -o $machine = hera ]; then -#------------------------------ +#----------------------------------- +#--add gfs_wafs link if checked out +if [ -d ${pwd}/gfs_wafs.fd ]; then +#----------------------------------- cd ${pwd}/../jobs ||exit 8 $LINK ../sorc/gfs_wafs.fd/jobs/* . cd ${pwd}/../parm ||exit 8 @@ -173,7 +173,7 @@ $LINK ../sorc/fv3gfs.fd/NEMS/exe/global_fv3gfs.x . [[ -s gfs_ncep_post ]] && rm -f gfs_ncep_post $LINK ../sorc/gfs_post.fd/exec/ncep_post gfs_ncep_post -if [ $machine = dell -o $machine = hera ]; then +if [ -d ${pwd}/gfs_wafs.fd ]; then for wafsexe in wafs_awc_wafavn wafs_blending wafs_cnvgrib2 wafs_gcip wafs_makewafs wafs_setmissing; do [[ -s $wafsexe ]] && rm -f $wafsexe $LINK ../sorc/gfs_wafs.fd/exec/$wafsexe . @@ -229,7 +229,7 @@ cd ${pwd}/../sorc || exit 8 done - if [ $machine = dell -o $machine = hera ]; then + if [ -d ${pwd}/gfs_wafs.fd ]; then $SLINK gfs_wafs.fd/sorc/wafs_awc_wafavn.fd wafs_awc_wafavn.fd $SLINK gfs_wafs.fd/sorc/wafs_blending.fd wafs_blending.fd $SLINK gfs_wafs.fd/sorc/wafs_cnvgrib2.fd wafs_cnvgrib2.fd @@ -254,5 +254,3 @@ fi exit 0 - - From 8699b46aa1797e8dd29edff1d4bd3b511ad5cb1c Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 6 Mar 2020 16:30:54 +0000 Subject: [PATCH 09/11] Issue #3 - updated README with new manic version --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d3e69b820f..875694750e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ The global-workflow current supports the following machines: ### 1. Check out components -The global-workflow uses the manage_externals utility to handle checking out its components. The manic-v1.1.7 manage_externals tag is supported. +The global-workflow uses the manage_externals utility to handle checking out its components. The manic-v1.1.8 manage_externals tag is supported. Run manage_externals (checkout_externals) while at top of clone: @@ -28,9 +28,9 @@ $ checkout_externals If checkout_externals is not in your $PATH then use full path to it: -* WCOSS-Dell: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/manage_externals/manic-v1.1.7/checkout_externals -* WCOSS-Cray: /gpfs/hps3/emc/global/noscrub/emc.glopara/git/manage_externals/manic-v1.1.7/checkout_externals -* Hera: /scratch1/NCEPDEV/global/glopara/git/manage_externals/manic-v1.1.7/checkout_externals +* WCOSS-Dell: /gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/manage_externals/manic-v1.1.8/checkout_externals +* WCOSS-Cray: /gpfs/hps3/emc/global/noscrub/emc.glopara/git/manage_externals/manic-v1.1.8/checkout_externals +* Hera: /scratch1/NCEPDEV/global/glopara/git/manage_externals/manic-v1.1.8/checkout_externals ### 2. Build components From e83b90d50999f64ed5208d94a8cceb8179c9395f Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 6 Mar 2020 17:00:15 +0000 Subject: [PATCH 10/11] Issue #3 - remove prod_util and grib_util sections from build_all.sh, removed elsewhere already --- sorc/build_all.sh | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 820c06ee75..50a31e14f5 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -284,24 +284,6 @@ if [ $target = wcoss -o $target = wcoss_cray -o $target = wcoss_dell_p3 ]; then } fi -#------------------------------------ -# build prod_util -#------------------------------------ -$Build_prod_util && { -echo " .... prod_util build not currently supported .... " -#echo " .... Building prod_util .... " -#./build_prod_util.sh > $logs_dir/build_prod_util.log 2>&1 -} - -#------------------------------------ -# build grib_util -#------------------------------------ -$Build_grib_util && { -echo " .... grib_util build not currently supported .... " -#echo " .... Building grib_util .... " -#./build_grib_util.sh > $logs_dir/build_grib_util.log 2>&1 -} - #------------------------------------ # Exception Handling #------------------------------------ From 622167d5fb3322921a1702639ebccb42da1f5e1b Mon Sep 17 00:00:00 2001 From: "kate.friedman" Date: Fri, 6 Mar 2020 18:20:31 +0000 Subject: [PATCH 11/11] Issue #3 - added explicit config flag example for checkout_externals in README and blurb about this replacing checkout.sh --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 875694750e..3fac2751aa 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,12 @@ The global-workflow current supports the following machines: ### 1. Check out components -The global-workflow uses the manage_externals utility to handle checking out its components. The manic-v1.1.8 manage_externals tag is supported. +The global-workflow uses the manage_externals utility to handle checking out its components. The manic-v1.1.8 manage_externals tag is supported. The manage_externals utility will be replacing the current checkout.sh script. Run manage_externals (checkout_externals) while at top of clone: ``` -$ checkout_externals +$ checkout_externals -e Externals.cfg ``` If checkout_externals is not in your $PATH then use full path to it: