file check and small fixes

Dockerfile

@@ -3,7 +3,7 @@ FROM r-base
 WORKDIR /app
 
 RUN apt-get update && \
-    apt-get install -y python2 python-pip gdal-bin file musl-dev && \
+    apt-get install -y python2 python-pip proj-bin libproj-dev libgdal-dev gdal-bin file musl-dev && \
     apt-get clean && apt-get autoremove -y
 
 # install Sen2Cor
@@ -16,7 +16,7 @@ COPY requirements.txt ./
 RUN pip install -r requirements.txt
 
 # install R dependencies
-RUN Rscript -e 'install.packages("raster")'
+RUN Rscript -e 'install.packages(c("raster", "rgdal"))'
 
 # install our code
 COPY . .

bin/s2_changedetection

@@ -36,20 +36,38 @@ if not len(processes):
 for i in range(0, len(in_files)):
     for process_id, script, suffix, inputs_required in processes:
         if len(in_files) < inputs_required:
-            logger.warn("not enough input files for {}; skipping!".format(process_id))
+            logger.warn("Not enough input files for {}; skipping!".format(process_id))
             continue
 
         fname = path.basename(in_files[i]).replace('.tif', suffix)
         outfile = path.join(out_dir, fname)
         infile = path.abspath(in_files[i])
+        # check if output file already exists. If so, skip loop
+        if path.isfile(outfile):
+            logger.warn("{} already exists. Skipping {}.".format(outfile, process_id))
+            continue
+
+        cmd = "Rscript --vanilla {} {}".format(script, outfile)
 
-        cmd = "Rscript --vanilla {} {} {}".format(script, outfile, infile)
-        if (i < len(in_files) - 1) and (inputs_required == 2):
-            cmd += " " + path.abspath(in_files[i+1])
+        # adding input files to cmd
+        restfiles = len(in_files) - i
+        if (restfiles >= inputs_required):
+            print "Running {} with:".format(process_id)
+            for j in range(0, inputs_required):
+                cmd += " " + path.abspath(in_files[i+j])
+                print "    {}".format(in_files[i+j])
+            print "    Saving to {}".format(outfile)
+        else:
+            print "Not enough input files for {}; skipping!".format(process_id)
+            continue
 
+        # TODO
+        # temporary solution until default threshold value is fixed in dNBR.R
+        if process_id == "dnbr":
+            cmd += " FALSE"
         out = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, cwd=path.dirname(script))
         stdout, stderr = out.communicate()
-        # logger.warn(stdout) # DEBUG: rscript also puts errors to stdout
-        # if stderr:
-        #     logger.critical(stderr)
-        #     sys.exit(1)
+        logger.warn(stdout) # DEBUG: rscript also puts errors to stdout
+        if stderr:
+            logger.critical(stderr)
+            #sys.exit(1)

bin/s2_clip

@@ -85,6 +85,27 @@ else echo "no matching tiles for that date ($INPUTDATE) found" 1>> $log 2>> $log
  exit
 fi
 
+# output tiff is named inputfolder_inputdate_l2a.tiff
+DATE=$(basename ${l2a_tiles%% *})
+TIFFNAME=${DATE:45:8}
+stitchedpath="${OUTDIR}/${TIFFNAME}_l2a"
+
+function cleanup {
+  # cleanup
+  echo "deleting $vrtlist" 1>&2
+  xargs rm -f < $vrtlist
+  rm -f $vrtlist ${stitchedpath}.vrt ${stitchedpath}_tmp.tif
+  rm -f $log
+}
+trap cleanup EXIT
+
+# check if file already exists. If so skip process
+if [ -f "${stitchedpath}.tif" ]; then
+  echo "${stitchedpath}.tif already exists. Skipping clipping." 1>&2
+  echo "${stitchedpath}.tif"
+  exit
+fi
+
 # for every tile, do
 for tile in $l2a_tiles
 do
@@ -102,27 +123,19 @@ do
 done
 echo "done with tiles" >> $log
 
-# output tiff is named inputfolder_inputdate.tiff
-DATE=$(basename ${l2a_tiles%% *})
-TIFFNAME=${DATE:45:8}
-stitchedpath="${OUTDIR}/${TIFFNAME}_l2a"
-
 # Stitching
 echo "start stitching" >> $log
+echo "stitching `cat $vrtlist`..." 1>&2
 gdalbuildvrt -resolution highest -input_file_list $vrtlist -overwrite $stitchedpath.vrt 1>> $log 2>> $log
 gdal_translate -of GTiff $stitchedpath.vrt ${stitchedpath}_tmp.tif 1>> $log 2>> $log
 echo "done\n" >> $log
 
 # CLIP image
 echo "start clipping" >> $log
+echo "clipping $stitchedpath.vrt..." 1>&2
 gdalwarp -dstnodata 0 -crop_to_cutline -cutline $AOI ${stitchedpath}_tmp.tif -overwrite $stitchedpath.tif 1>> $log 2>> $log
 echo done >> $log
 echo "" >> $log
 
-# cleanup
-xargs rm -f < $vrtlist
-rm -f $vrtlist ${stitchedpath}.vrt ${stitchedpath}_tmp.tif
-rm -f $log
-
 # output required for pipeline
 echo $stitchedpath.tif

bin/s2_pipeline

@@ -3,6 +3,7 @@
 set -e
 
 aoi=''
+algorithm=''
 startdate='NOW-5DAYS'
 enddate='NOW'
 cloud='20'
@@ -22,6 +23,7 @@ while test $# -gt 0; do
       echo "options:"
       echo "-h, --help                show brief help."
       echo "-a, --aoi                 area of interest as geojson polygon geometry json file. Required."
+      echo "--algorithm               select single algorithm. Possible is 'dnbr' or 'bais2'. Defaults to both."
       echo "-s, --start               start date 'YYYYMMDD' or 'NOW-XDAYS'. Default 'NOW-5DAYS'."
       echo "-e, --end                 end date 'YYYYMMDD' or 'NOW-XDAYS'. Default 'NOW'."
       echo "-o, --outdir              path to output directory. Defaults to directory where the script was executed."
@@ -35,6 +37,11 @@ while test $# -gt 0; do
       aoistring=$(cat $aoi)
       shift
       ;;
+    --algorithm)
+      shift
+      algorithm=$1
+      shift
+      ;;
     -s|--start)
       shift
       startdate=$1
@@ -80,6 +87,15 @@ function parallelism {
   echo -e "$cpu_instances\n$mem_instances" | sort | head -n1 # minimum of cores & ram
 }
 
+function filesexist {
+ # if [ ! -z $(find $outdir/data/ -maxdepth 1 -name $1) ]; then
+  if [ ! -z "$(find $outdir/data/ -maxdepth 1 -name $1)" ]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
 sen2cor_instances=2 # `parallelism 5000`
 
 if [ "$sen2cor_instances" -eq "0" ]; then
@@ -117,16 +133,33 @@ tiffs=`$SCRIPTDIR/s2_grouporbit --input $outdir/data/ |
 
 echo -e "\n====> pipeline 3/4: change detection\n"
 # calling change detection script(s)
-$SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data"
+# add --algorithm flag if it was defined
+if [ ! -z $algorithm ]
+  then
+    $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data" --process $algorithm
+  else
+    $SCRIPTDIR/s2_changedetection --input $tiffs --outputdir "$outdir/data"
+fi
 
-echo -e "\n====> pipeline 3/4: web visualization\n"
+echo -e "\n====> pipeline 4/4: web visualization\n"
 # create web visualization at last
-$SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_l2a\.tif$"
+# always checks if files with matching suffix exist
+if filesexist '*_bais2.tif'; then
+  $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_bais2\.tif$" --outfile "viz_bais2.html" &
+fi
+
+if filesexist '*_dnbr.tif'; then
+  $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_dnbr\.tif$" --outfile "viz_dnbr.html" &
+fi
+
+if filesexist '*_l2a.tif'; then
+  $SCRIPTDIR/s2_visualize "$outdir/data" "$aoistring" --filepattern ".*_l2a\.tif$" --outfile "viz_l2a.html"
+fi
 
 # clean up intermediate files
 if [ ! -z $cleanup ]; then
   echo -e "\n====> cleanup of intermediate files"
-  rm -rf $outdir/data/{*MSIL1C*,*MSIL2A*,*.vrt,*.zip}
+  rm -rf $outdir/data/{*.SAFE,*.zip}
 fi
 
 echo -e "\n====> done!"

bin/s2_visualize

@@ -26,6 +26,7 @@ parser.add_argument('indir', type = str, help = 'directory to retrieve the input
 parser.add_argument('aoi', type = str, help = 'area of interest geojson string')
 parser.add_argument('--filepattern', help = 'RegEx pattern to use to match input files', default = '.*_dnbr\.tif$')
 parser.add_argument('--template', help = 'Template to use for visualization generation', default = 'leaflet_villages')
+parser.add_argument('--outfile', help = 'Filename of the resulting visualization', default = 'visualization.html')
 args = parser.parse_args()
 
 # prepare area of interest (make sure it is a feature)
@@ -64,7 +65,7 @@ if not (templateFiles.get(args.template) and path.isfile(templateFiles[args.temp
 
 # read template, apply content, write result to <indir>/visualization.html
 with open(templateFiles[args.template], 'r') as templateFile:
-    outFile = path.join(args.indir, 'visualization.html')
+    outFile = path.join(args.indir, args.outfile)
     tifList = ','.join(['"./{}"'.format(path.basename(f)) for f in thumbs])
     template = templateFile.read()
     template = template.replace('%%AOI%%', aoi)