From 99ac857281075b5b24028467993bb7642eeb4557 Mon Sep 17 00:00:00 2001 From: jansule Date: Mon, 28 Jan 2019 16:54:29 +0100 Subject: [PATCH 1/9] file check and small fixes --- Dockerfile | 4 ++-- bin/s2_changedetection | 22 ++++++++++++++++------ bin/s2_clip | 16 ++++++++++++---- bin/s2_pipeline | 2 +- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index f3435fe..e952218 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM r-base WORKDIR /app RUN apt-get update && \ - apt-get install -y python2 python-pip gdal-bin file musl-dev && \ + apt-get install -y python2 python-pip proj-bin libproj-dev libgdal-dev gdal-bin file musl-dev && \ apt-get clean && apt-get autoremove -y # install Sen2Cor @@ -16,7 +16,7 @@ COPY requirements.txt ./ RUN pip install -r requirements.txt # install R dependencies -RUN Rscript -e 'install.packages("raster")' +RUN Rscript -e 'install.packages(c("raster", "rgdal"))' # install our code COPY . . diff --git a/bin/s2_changedetection b/bin/s2_changedetection index 67982f9..31f537b 100755 --- a/bin/s2_changedetection +++ b/bin/s2_changedetection @@ -36,20 +36,30 @@ if not len(processes): for i in range(0, len(in_files)): for process_id, script, suffix, inputs_required in processes: if len(in_files) < inputs_required: - logger.warn("not enough input files for {}; skipping!".format(process_id)) + logger.warn("Not enough input files for {}; skipping!".format(process_id)) continue fname = path.basename(in_files[i]).replace('.tif', suffix) outfile = path.join(out_dir, fname) infile = path.abspath(in_files[i]) + # check if output file already exists. If so, skip loop + if path.isfile(outfile): + logger.warn("{} already exists. Skipping change detection.".format(outfile)) + continue cmd = "Rscript --vanilla {} {} {}".format(script, outfile, infile) if (i < len(in_files) - 1) and (inputs_required == 2): cmd += " " + path.abspath(in_files[i+1]) - + elif (i == len(in_files) - 1) and (inputs_required == 2): + print "Not enough input files for {}; skipping!".format(process_id) + continue + print "Running script {} with outfile {} and infile {}".format(script, outfile, infile) + # temporary solution until default threshold value is fixed in dNBR.R + #if process_id == "dnbr": + #cmd += " FALSE" out = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, cwd=path.dirname(script)) stdout, stderr = out.communicate() - # logger.warn(stdout) # DEBUG: rscript also puts errors to stdout - # if stderr: - # logger.critical(stderr) - # sys.exit(1) + logger.warn(stdout) # DEBUG: rscript also puts errors to stdout + if stderr: + logger.critical(stderr) + sys.exit(1) diff --git a/bin/s2_clip b/bin/s2_clip index edb4eb0..f044d4e 100755 --- a/bin/s2_clip +++ b/bin/s2_clip @@ -85,6 +85,18 @@ else echo "no matching tiles for that date ($INPUTDATE) found" 1>> $log 2>> $log exit fi +# output tiff is named inputfolder_inputdate_l2a.tiff +DATE=$(basename ${l2a_tiles%% *}) +TIFFNAME=${DATE:45:8} +stitchedpath="${OUTDIR}/${TIFFNAME}_l2a" + +# check if file already exists. If so skip process +if [ -f "${stitchedpath}.tif" ]; then + echo "${stitchedpath}.tif already exists. Skipping clipping" 1>> $log 2>> $log + echo "${stitchedpath}.tif" + exit +fi + # for every tile, do for tile in $l2a_tiles do @@ -102,10 +114,6 @@ do done echo "done with tiles" >> $log -# output tiff is named inputfolder_inputdate.tiff -DATE=$(basename ${l2a_tiles%% *}) -TIFFNAME=${DATE:45:8} -stitchedpath="${OUTDIR}/${TIFFNAME}_l2a" # Stitching echo "start stitching" >> $log diff --git a/bin/s2_pipeline b/bin/s2_pipeline index 95331e1..39a61cb 100755 --- a/bin/s2_pipeline +++ b/bin/s2_pipeline @@ -119,7 +119,7 @@ echo -e "\n====> pipeline 3/4: change detection\n" # calling change detection script(s) $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" -echo -e "\n====> pipeline 3/4: web visualization\n" +echo -e "\n====> pipeline 4/4: web visualization\n" # create web visualization at last $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_l2a\.tif$" From 8ed0b5c21a4d2f681a797cac0a13f575f2fb0681 Mon Sep 17 00:00:00 2001 From: jansule Date: Mon, 28 Jan 2019 17:18:00 +0100 Subject: [PATCH 2/9] add dnbr and bais2 tifs to visualization --- bin/s2_pipeline | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/s2_pipeline b/bin/s2_pipeline index 39a61cb..d31300b 100755 --- a/bin/s2_pipeline +++ b/bin/s2_pipeline @@ -121,7 +121,7 @@ $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" echo -e "\n====> pipeline 4/4: web visualization\n" # create web visualization at last -$SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_l2a\.tif$" +$SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_(l2a|dnbr|bais2)\.tif$" # clean up intermediate files if [ ! -z $cleanup ]; then From 6fc62ef7eec6bf80c7ececcd7f17fe78f9cca19a Mon Sep 17 00:00:00 2001 From: Norwin Roosen Date: Tue, 29 Jan 2019 13:25:25 +0100 Subject: [PATCH 3/9] generate separate visualizations for dnbr & bais2 --- bin/s2_pipeline | 7 ++++++- bin/s2_visualize | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bin/s2_pipeline b/bin/s2_pipeline index d31300b..4050ee2 100755 --- a/bin/s2_pipeline +++ b/bin/s2_pipeline @@ -121,7 +121,12 @@ $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" echo -e "\n====> pipeline 4/4: web visualization\n" # create web visualization at last -$SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_(l2a|dnbr|bais2)\.tif$" +$SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_bais2\.tif$" --outfile "viz_bais2.html" + +# dnbr only works if there are two input files, so we should check that +if [ ! -z "$(find $outdir/data/ -maxdepth 1 -name '*_dnbr.tif')" ]; then + $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_dnbr\.tif$" --outfile "viz_dnbr.html" +fi # clean up intermediate files if [ ! -z $cleanup ]; then diff --git a/bin/s2_visualize b/bin/s2_visualize index cf40ccf..8345a06 100755 --- a/bin/s2_visualize +++ b/bin/s2_visualize @@ -26,6 +26,7 @@ parser.add_argument('indir', type = str, help = 'directory to retrieve the input parser.add_argument('aoi', type = str, help = 'area of interest geojson string') parser.add_argument('--filepattern', help = 'RegEx pattern to use to match input files', default = '.*_dnbr\.tif$') parser.add_argument('--template', help = 'Template to use for visualization generation', default = 'leaflet_villages') +parser.add_argument('--outfile', help = 'Filename of the resulting visualization', default = 'visualization.html') args = parser.parse_args() # prepare area of interest (make sure it is a feature) @@ -64,7 +65,7 @@ if not (templateFiles.get(args.template) and path.isfile(templateFiles[args.temp # read template, apply content, write result to /visualization.html with open(templateFiles[args.template], 'r') as templateFile: - outFile = path.join(args.indir, 'visualization.html') + outFile = path.join(args.indir, args.outfile) tifList = ','.join(['"./{}"'.format(path.basename(f)) for f in thumbs]) template = templateFile.read() template = template.replace('%%AOI%%', aoi) From fb0ddf96bce39a865bbb70c8bd512245cbef793e Mon Sep 17 00:00:00 2001 From: jansule Date: Tue, 29 Jan 2019 14:23:36 +0100 Subject: [PATCH 4/9] update changedetection and add progress logs --- bin/s2_changedetection | 26 +++++++++++++++++--------- bin/s2_clip | 4 +++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/bin/s2_changedetection b/bin/s2_changedetection index 31f537b..752cfbb 100755 --- a/bin/s2_changedetection +++ b/bin/s2_changedetection @@ -44,22 +44,30 @@ for i in range(0, len(in_files)): infile = path.abspath(in_files[i]) # check if output file already exists. If so, skip loop if path.isfile(outfile): - logger.warn("{} already exists. Skipping change detection.".format(outfile)) + logger.warn("{} already exists. Skipping {}.".format(outfile, process_id)) continue - cmd = "Rscript --vanilla {} {} {}".format(script, outfile, infile) - if (i < len(in_files) - 1) and (inputs_required == 2): - cmd += " " + path.abspath(in_files[i+1]) - elif (i == len(in_files) - 1) and (inputs_required == 2): + cmd = "Rscript --vanilla {} {}".format(script, outfile) + + # adding input files to cmd + restfiles = len(in_files) - i + if (restfiles >= inputs_required): + print "Running {} with:".format(process_id) + for j in range(0, inputs_required): + cmd += " " + path.abspath(in_files[i+j]) + print " {}".format(in_files[i+j]) + print " Saving to {}".format(outfile) + else: print "Not enough input files for {}; skipping!".format(process_id) continue - print "Running script {} with outfile {} and infile {}".format(script, outfile, infile) + + # TODO # temporary solution until default threshold value is fixed in dNBR.R - #if process_id == "dnbr": - #cmd += " FALSE" + if process_id == "dnbr": + cmd += " FALSE" out = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, cwd=path.dirname(script)) stdout, stderr = out.communicate() logger.warn(stdout) # DEBUG: rscript also puts errors to stdout if stderr: logger.critical(stderr) - sys.exit(1) + #sys.exit(1) diff --git a/bin/s2_clip b/bin/s2_clip index f044d4e..9829a21 100755 --- a/bin/s2_clip +++ b/bin/s2_clip @@ -92,7 +92,7 @@ stitchedpath="${OUTDIR}/${TIFFNAME}_l2a" # check if file already exists. If so skip process if [ -f "${stitchedpath}.tif" ]; then - echo "${stitchedpath}.tif already exists. Skipping clipping" 1>> $log 2>> $log + echo "${stitchedpath}.tif already exists. Skipping clipping." 1>&2 echo "${stitchedpath}.tif" exit fi @@ -117,12 +117,14 @@ echo "done with tiles" >> $log # Stitching echo "start stitching" >> $log +echo "stitching `cat $vrtlist`..." 1>&2 gdalbuildvrt -resolution highest -input_file_list $vrtlist -overwrite $stitchedpath.vrt 1>> $log 2>> $log gdal_translate -of GTiff $stitchedpath.vrt ${stitchedpath}_tmp.tif 1>> $log 2>> $log echo "done\n" >> $log # CLIP image echo "start clipping" >> $log +echo "clipping $stitchedpath.vrt..." 1>&2 gdalwarp -dstnodata 0 -crop_to_cutline -cutline $AOI ${stitchedpath}_tmp.tif -overwrite $stitchedpath.tif 1>> $log 2>> $log echo done >> $log echo "" >> $log From 03cde2c9f2a6a73c50e47c92ee4dbd281a7c022d Mon Sep 17 00:00:00 2001 From: jansule Date: Tue, 29 Jan 2019 15:06:09 +0100 Subject: [PATCH 5/9] run viz computation in parallel. add filesexist function --- bin/s2_pipeline | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/bin/s2_pipeline b/bin/s2_pipeline index 4050ee2..7cb481f 100755 --- a/bin/s2_pipeline +++ b/bin/s2_pipeline @@ -80,6 +80,15 @@ function parallelism { echo -e "$cpu_instances\n$mem_instances" | sort | head -n1 # minimum of cores & ram } +function filesexist { + # if [ ! -z $(find $outdir/data/ -maxdepth 1 -name $1) ]; then + if [ ! -z "$(find $outdir/data/ -maxdepth 1 -name $1)" ]; then + return 0 + else + return 1 + fi +} + sen2cor_instances=2 # `parallelism 5000` if [ "$sen2cor_instances" -eq "0" ]; then @@ -121,11 +130,17 @@ $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" echo -e "\n====> pipeline 4/4: web visualization\n" # create web visualization at last -$SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_bais2\.tif$" --outfile "viz_bais2.html" +# always checks if files with matching suffix exist +if filesexist '*_bais2.tif'; then + $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_bais2\.tif$" --outfile "viz_bais2.html" & +fi + +if filesexist '*_dnbr.tif'; then + $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_dnbr\.tif$" --outfile "viz_dnbr.html" & +fi -# dnbr only works if there are two input files, so we should check that -if [ ! -z "$(find $outdir/data/ -maxdepth 1 -name '*_dnbr.tif')" ]; then - $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_dnbr\.tif$" --outfile "viz_dnbr.html" +if filesexist '*_l2a.tif'; then + $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_l2a\.tif$" --outfile "viz_l2a.html" fi # clean up intermediate files From 477ccf11dea999ea239ff614a51a30f7a7c483fa Mon Sep 17 00:00:00 2001 From: jansule Date: Tue, 29 Jan 2019 15:15:09 +0100 Subject: [PATCH 6/9] add exit traps to s2_clip --- bin/s2_clip | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bin/s2_clip b/bin/s2_clip index 9829a21..7e7a7e4 100755 --- a/bin/s2_clip +++ b/bin/s2_clip @@ -97,6 +97,17 @@ if [ -f "${stitchedpath}.tif" ]; then exit fi +function cleanup { + # cleanup + #xargs rm -f < $vrtlist + #rm -f $vrtlist ${stitchedpath}.vrt ${stitchedpath}_tmp.tif + #rm -f $log + xargs rm -f < $1 + rm -f $1 ${2}.vrt ${2}_tmp.tif + rm -f $3 +} +trap cleanup $vrtlist $stitchedpath $log + # for every tile, do for tile in $l2a_tiles do @@ -114,7 +125,6 @@ do done echo "done with tiles" >> $log - # Stitching echo "start stitching" >> $log echo "stitching `cat $vrtlist`..." 1>&2 @@ -129,10 +139,5 @@ gdalwarp -dstnodata 0 -crop_to_cutline -cutline $AOI ${stitchedpath}_tmp.tif -ov echo done >> $log echo "" >> $log -# cleanup -xargs rm -f < $vrtlist -rm -f $vrtlist ${stitchedpath}.vrt ${stitchedpath}_tmp.tif -rm -f $log - # output required for pipeline echo $stitchedpath.tif From 733ece528e5a52487635935db5dcec5c72d282b2 Mon Sep 17 00:00:00 2001 From: jansule Date: Tue, 29 Jan 2019 16:42:29 +0100 Subject: [PATCH 7/9] fix exit traps --- bin/s2_clip | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/bin/s2_clip b/bin/s2_clip index 7e7a7e4..43f8549 100755 --- a/bin/s2_clip +++ b/bin/s2_clip @@ -90,6 +90,15 @@ DATE=$(basename ${l2a_tiles%% *}) TIFFNAME=${DATE:45:8} stitchedpath="${OUTDIR}/${TIFFNAME}_l2a" +function cleanup { + # cleanup + echo "deleting $vrtlist" 1>&2 + xargs rm -f < $vrtlist + rm -f $vrtlist ${stitchedpath}.vrt ${stitchedpath}_tmp.tif + rm -f $log +} +trap cleanup EXIT + # check if file already exists. If so skip process if [ -f "${stitchedpath}.tif" ]; then echo "${stitchedpath}.tif already exists. Skipping clipping." 1>&2 @@ -97,17 +106,6 @@ if [ -f "${stitchedpath}.tif" ]; then exit fi -function cleanup { - # cleanup - #xargs rm -f < $vrtlist - #rm -f $vrtlist ${stitchedpath}.vrt ${stitchedpath}_tmp.tif - #rm -f $log - xargs rm -f < $1 - rm -f $1 ${2}.vrt ${2}_tmp.tif - rm -f $3 -} -trap cleanup $vrtlist $stitchedpath $log - # for every tile, do for tile in $l2a_tiles do From 09a9d89d636f755da7ec606aeb3da464edc6a0a2 Mon Sep 17 00:00:00 2001 From: jansule Date: Tue, 29 Jan 2019 16:55:38 +0100 Subject: [PATCH 8/9] add --algorithm flag to pipeline --- bin/s2_pipeline | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/bin/s2_pipeline b/bin/s2_pipeline index 7cb481f..81c1acd 100755 --- a/bin/s2_pipeline +++ b/bin/s2_pipeline @@ -3,6 +3,7 @@ set -e aoi='' +algorithm='' startdate='NOW-5DAYS' enddate='NOW' cloud='20' @@ -22,6 +23,7 @@ while test $# -gt 0; do echo "options:" echo "-h, --help show brief help." echo "-a, --aoi area of interest as geojson polygon geometry json file. Required." + echo "--algorithm select single algorithm. Possible is 'dnbr' or 'bais2'. Defaults to both." echo "-s, --start start date 'YYYYMMDD' or 'NOW-XDAYS'. Default 'NOW-5DAYS'." echo "-e, --end end date 'YYYYMMDD' or 'NOW-XDAYS'. Default 'NOW'." echo "-o, --outdir path to output directory. Defaults to directory where the script was executed." @@ -35,6 +37,11 @@ while test $# -gt 0; do aoistring=$(cat $aoi) shift ;; + --algorithm) + shift + algorithm=$1 + shift + ;; -s|--start) shift startdate=$1 @@ -126,7 +133,13 @@ tiffs=`$SCRIPTDIR/s2_grouporbit --input $outdir/data/ | echo -e "\n====> pipeline 3/4: change detection\n" # calling change detection script(s) -$SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" +# add --algorithm flag if it was defined +if [ ! -z $algorithm ] + then + $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" --process $algorithm + else + $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" +fi echo -e "\n====> pipeline 4/4: web visualization\n" # create web visualization at last From d16701d0370224e12663bdf61c7f68aec98622dd Mon Sep 17 00:00:00 2001 From: Norwin Roosen Date: Tue, 29 Jan 2019 17:18:30 +0100 Subject: [PATCH 9/9] simplify cleanup --- bin/s2_pipeline | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/s2_pipeline b/bin/s2_pipeline index 81c1acd..539a48d 100755 --- a/bin/s2_pipeline +++ b/bin/s2_pipeline @@ -159,7 +159,7 @@ fi # clean up intermediate files if [ ! -z $cleanup ]; then echo -e "\n====> cleanup of intermediate files" - rm -rf $outdir/data/{*MSIL1C*,*MSIL2A*,*.vrt,*.zip} + rm -rf $outdir/data/{*.SAFE,*.zip} fi echo -e "\n====> done!"