Merged 'swath' into 'develop'; solved merge conflicts.

Merge branch 'swath' into develop

# Conflicts:
#	NEWS
#	NEWS.md
#	R/getCollection.R
#	R/minorFuns.R

.Rbuildignore

@@ -8,3 +8,9 @@
 ^resample\.log$
 ^R/getUTMZone\.R$
 ^inst/external/UTM_Zone_Boundaries\.rds$
+^tests/testthat/test-runGdal\.R$
+^tests/testthat/test-EarthdataLogin\.R$
+^inst/external/MOD13A2\.A2016145\.h18v04\.006\.2016166145124\.hdf$
+^R/getGranule\.R$
+^R/MODISswath\.R$
+^R/MODISswath$

NAMESPACE

@@ -31,6 +31,8 @@ export(temporalComposite)
 export(transDate)
 export(whittaker.raster)
 exportClasses(MODISextent)
+exportClasses(MODISfile)
+exportClasses(MODISproduct)
 if (.Platform$OS.type=="windows") importFrom(utils,shortPathName)
 
 import(bitops)

NEWS

@@ -3,7 +3,12 @@ MODIS 1.1.5
 Changes:
 
   * Moved internal datasets (products, collections, etc.) from manual creation in zz_lazyload.R, which also had an unnecessary installed.packages() call in it, to R/sysdata.rda.
-
+
+Bugfixes:
+
+  * Missing compatibility of repDoy() with results from extractDate().
+  * Duplicate server issue when passing multiple products on to runGdal().
+
 
 MODIS 1.1.4
 
@@ -13,10 +18,12 @@ New features:
   * Interactive feature drawing is now implemented in getTile(), check out the documentation.
   * getTile() supports 'sp' or 'sf' point geometries with length = 1 (i.e. a single point), in which case the target extent is the MODIS tile covering the specified location.
   * Multiple source download using aria2 whenever more than one server is available (e.g. LAADS and LP DAAC)
+  * New classes 'MODISproduct' and 'MODISfile' created by getProduct().
 
 Changes:
 
   * When running getHdf() (or runGdal()) with LP DAAC or NSIDC as target download server and Earthdata login credentials in ~/.netrc are missing, the user is forced to insert the required information on the command line. This is intended to avoid repeat authentication failures.
+  * Functions checkIntegrity(), OutProj(), PixelSize(), ResamplingType(), BlockSize(), OutputCompression(), QuietOutput(), genString(), checkTools(), ModisFileDownloader(), and doCheckIntegrity() no longer have an 'opts' argument.
 
 
 MODIS 1.1.3

NEWS.md

@@ -3,7 +3,12 @@
 Changes:
 
   * Moved internal datasets (products, collections, etc.) from manual creation in zz_lazyload.R, which also had an unnecessary installed.packages() call in it, to R/sysdata.rda.
-
+
+Bugfixes:
+
+  * Missing compatibility of repDoy() with results from extractDate().
+  * Duplicate server issue when passing multiple products on to runGdal().
+
 
 ## MODIS 1.1.4
 
@@ -13,10 +18,12 @@ New features:
   * Interactive feature drawing is now implemented in getTile(), check out the documentation.
   * getTile() supports 'sp' or 'sf' point geometries with length = 1 (i.e. a single point), in which case the target extent is the MODIS tile covering the specified location.
   * Multiple source download using aria2 whenever more than one server is available (e.g. LAADS and LP DAAC)
+  * New classes 'MODISproduct' and 'MODISfile' created by getProduct().
 
 Changes:
 
   * When running getHdf() (or runGdal()) with LP DAAC or NSIDC as target download server and Earthdata login credentials in ~/.netrc are missing, the user is forced to insert the required information on the command line. This is intended to avoid repeat authentication failures.
+  * Functions checkIntegrity(), OutProj(), PixelSize(), ResamplingType(), BlockSize(), OutputCompression(), QuietOutput(), genString(), checkTools(), ModisFileDownloader(), and doCheckIntegrity() no longer have an 'opts' argument.
 
 
 ## MODIS 1.1.3

R/.gitignore

@@ -0,0 +1 @@
+MODISswath

R/EarthdataLogin.R

@@ -12,8 +12,7 @@
 #' username and password are read from the terminal.
 #' 
 #' @return 
-#' An \code{invisible()} absolute file path of the created \code{.netrc} file as 
-#' \code{character}.
+#' The \code{\link{invisible}} Earthdata login credentials as \code{list}.
 #' 
 #' @author 
 #' Matteo Mattiuzzi and Florian Detsch
@@ -44,6 +43,14 @@ EarthdataLogin <- function(usr = NULL, pwd = NULL) {
   # get servers found on .netrc
   machine <- unlist(listPather(lns,'machine'))
 
+  # if earthdata server is not in the netrc file
+  ind = if (!server %in% machine) {
+    # if server not present, add it in a new line
+    length(lns)+1
+  } else {
+    which(machine == server)[1]
+  }
+
   # create file
   if(!file.exists(nrc))
   {
@@ -54,7 +61,12 @@ EarthdataLogin <- function(usr = NULL, pwd = NULL) {
   {
     if(sum(machine %in% server)>0)
     {
-      insert <- tolower(readline(paste0("Earthdata credentials seem to be present, do you want to change them? (y/n) \n",sep="")))
+      if (any(is.null(lns[[ind]]$login), is.null(lns[[ind]]$password))) {
+        cat("'", nrc, "' with defective Earthdata login credentials found! Please correct them now...\n", sep = "")
+        insert = "y"
+      } else {
+        insert <- tolower(readline(paste0("Earthdata credentials seem to be present, do you want to change them? (y/n) \n",sep="")))
+      }
     } else
     {
       cat("'",nrc,"' without Earthdata login credentials found! Please add them now...\n",sep="")
@@ -77,14 +89,6 @@ EarthdataLogin <- function(usr = NULL, pwd = NULL) {
   }
   # Y: add (new) credentials. N: reformat .netrc file
 
-  # if earthdata server is not in the netrc file
-  ind = if (!server %in% machine) {
-    # if server not present, add it in a new line
-    length(lns)+1
-  } else {
-    which(machine == server)
-  }
-
   # if credentials are present, do not change, unless specified by arguments or confirmed
   if(is.null(usr))
   {
@@ -122,19 +126,15 @@ EarthdataLogin <- function(usr = NULL, pwd = NULL) {
 
   Sys.chmod(nrc, mode = "600", use_umask = TRUE)
 
-  return(invisible(nrc))
+  return(invisible(credentials()))
 }
 
 ## Earthdata login credentials from .netrc file
 readCredentials = function() {
 
-  # e .netrc file
+  # ~/.netrc file
   nrc = path.expand("~/.netrc")
 
-  # if (!file.exists(nrc)) {
-  #   stop("~/.netrc file required. Either run EarthdataLogin() or set"
-  #        , " MODISoptions(MODISserverOrder = 'LAADS').")
-
   if (file.exists(nrc))
   {  
     lns = readLines(nrc)
@@ -157,14 +157,14 @@ readCredentials = function() {
       if(length(strsplit(lns[i]," ")[[1]])==2)
       {
         machine = strsplit(lns[i], " ")[[1]][2]
-        login = strsplit(lns[i+1], " ")[[1]][2]
-        password = strsplit(lns[i+2], " ")[[1]][2]
+        login = if (!is.na(tmp <- strsplit(lns[i+1], " ")[[1]][2])) tmp
+        password = if (!is.na(tmp <- strsplit(lns[i+2], " ")[[1]][2])) tmp
       } else
       {
         # if credentials are within a single line
         machine = strsplit(lns[i], " ")[[1]][2]
-        login = strsplit(lns[i], " ")[[1]][4]
-        password = strsplit(lns[i], " ")[[1]][6]   
+        login = if (!is.na(tmp <- strsplit(lns[i], " ")[[1]][4])) tmp
+        password = if (!is.na(tmp <- strsplit(lns[i], " ")[[1]][6])) tmp
       }
       result[[j]] <- list(machine=machine,login=login,password=password)
     }

R/MODIS_QC.R

@@ -246,6 +246,36 @@ MYD15A3H_QC <- data.frame(
   bitMask=c(1,1,1,3,7)
 )
 
+#### M.D16
+
+MOD16A2_QC <- data.frame(
+  LongName=c("MODLAND_QC bits", "Sensor", "DeadDetector", "CloudState (inherited from Aggregate_QC bits {0,1} cloud state)",
+             "SCF_QC (five level confidence score)"),
+  bitShift=c(0,1,2,3,5),
+  bitMask=c(1,1,1,3,7)
+)
+
+MYD16A2_QC <- data.frame(
+  LongName=c("MODLAND_QC bits", "Sensor", "DeadDetector", "CloudState (inherited from Aggregate_QC bits {0,1} cloud state)",
+             "SCF_QC (five level confidence score)"),
+  bitShift=c(0,1,2,3,5),
+  bitMask=c(1,1,1,3,7)
+)
+
+MOD16A3_QC <- data.frame(
+  LongName=c("MODLAND_QC bits", "Sensor", "DeadDetector", "CloudState (inherited from Aggregate_QC bits {0,1} cloud state)",
+             "SCF_QC (five level confidence score)"),
+  bitShift=c(0,1,2,3,5),
+  bitMask=c(1,1,1,3,7)
+)
+
+MYD16A3_QC <- data.frame(
+  LongName=c("MODLAND_QC bits", "Sensor", "DeadDetector", "CloudState (inherited from Aggregate_QC bits {0,1} cloud state)",
+             "SCF_QC (five level confidence score)"),
+  bitShift=c(0,1,2,3,5),
+  bitMask=c(1,1,1,3,7)
+)
+
 #### M.D17
 
 MOD17A2_QC <- data.frame(

R/MODISoptions.R

@@ -43,6 +43,12 @@
 #' internal online download call via \code{\link{download.file}} or 
 #' \code{\link[curl]{curl}}. Reduces the chance of connection errors that 
 #' frequently occur after many requests.
+#' @param cellchunk Default 1 (=use raster default), comparable with chunksize
+#' in \code{\link{rasterOptions}}. But as no effect was found in adapting chunksize, 
+#' MODIS applies its own variant:
+#' \strong{minrows <- max(floor(cellchunk/ncol(x)),1) 
+#' blockSize(x,minrows=minrows)}. 
+#' On a reasonable working station you can easily increase cellchunk to 500000.
 #' @param systemwide A \code{logical} determining whether changes made to 
 #' \code{\link{MODISoptions}} are to be applied system or user-wide (default), 
 #' see 'Details'.
@@ -124,9 +130,9 @@
 #' @name MODISoptions
 MODISoptions <- function(localArcPath, outDirPath, pixelSize, outProj, 
                          resamplingType, dataFormat, gdalPath, MODISserverOrder, 
-                         dlmethod, stubbornness, wait, quiet, 
-                         systemwide = FALSE, save = TRUE, checkTools = TRUE
-                         , checkWriteDrivers = TRUE, ask = TRUE)
+                         dlmethod, stubbornness, wait, quiet, cellchunk,
+                         systemwide = FALSE, save = TRUE, checkTools = TRUE,
+                         checkWriteDrivers = TRUE, ask = TRUE)
 {
   # This function collects the package options from up to 3 files and creates 
   # the .MODIS_Opts.R file (location depending on systemwide=T/F, see below):
@@ -296,7 +302,7 @@ MODISoptions <- function(localArcPath, outDirPath, pixelSize, outProj,
     opt$gdalPath <- correctPath(gdalPath)
     if(all(!grepl("gdalinfo", dir(opt$gdalPath))))
     {
-      stop(paste0("The 'gdalPath' you have provided '",normalizePath(opt$gdalPath,"/",FALSE) ,"' does not contain any gdal utilities, make sure to address the folder with GDAL executables (ie: gdalinfo)!"))
+      stop(paste0("The 'gdalPath' you have provided '",normalizePath(opt$gdalPath,"/",FALSE) ,"' does not contain any gdal utilities, make sure to address the folder with GDAL executables (i.e.: gdalinfo)!"))
     }
   }
   opt$gdalPath <- correctPath(opt$gdalPath)
@@ -335,15 +341,17 @@ MODISoptions <- function(localArcPath, outDirPath, pixelSize, outProj,
     if (isOk) 
     {
       opt$gdalOk  <- TRUE
-      gdalVersion <- checkTools(tool="GDAL",quiet=TRUE, opts = opt)$GDAL$version
+      opt2 = opt; opt2 = opt2[names(opt) != "quiet"]
+      gdalVersion <- do.call("checkTools", c(list(tool = "GDAL", quiet = TRUE), opt2))$GDAL$version
     } else
     {
       opt$gdalOk  <- FALSE
       gdalVersion <- "Not available. Use 'MODIS:::checkTools('GDAL')' for more information!"
     }
 
     # MRT
-    mrt <- checkTools(tool="MRT",quiet=TRUE, opts = opt)$MRT
+    opt2 = opt; opt2 = opt2[names(opt) != "quiet"]
+    mrt <- do.call("checkTools", c(list(tool = "MRT", quiet = TRUE), opt2))$MRT
     if(mrt$MRT)
     {
       opt$mrtOk  <- TRUE
@@ -381,6 +389,13 @@ MODISoptions <- function(localArcPath, outDirPath, pixelSize, outProj,
     opt$gdalOutDriver <- gdalWriteDriver(renew = FALSE, quiet = FALSE, gdalPath=opt$gdalPath,outDirPath=opt$outDirPath)
   }
 
+  if(!missing(cellchunk))
+  {
+    opt$cellchunk <- cellchunk
+  }
+
+  #########
+
   if (save) {
 
     #  let user decide whether to make settings permanent
@@ -443,7 +458,15 @@ MODISoptions <- function(localArcPath, outDirPath, pixelSize, outProj,
     write(paste0('dataFormat     <- \'',opt$dataFormat,'\''),filename)
     write('  ', filename)	
     write('#########################', filename)
-    write('# 4.) Set path to GDAL _bin_ directory', filename)
+    write('# 4) Defaults related to raster package:', filename)	 
+    write('# Cellchunk: Comparable with chunksize in ?rasterOption.',filename)
+    write('# But as no effect was found in adapting chunksize,', filename)	
+    write('# MODIS applies its own variant:minrows <- max(floor(cellchunk/ncol(x)),1) blockSize(x,minrows=minrows).', filename)	
+    write('# On a reasonable working station you can easily increase this to 500000, set 1 for raster defaults', filename)	
+    write(paste0('cellchunk <- ',opt$cellchunk), filename)
+    write('  ', filename)    
+    write('#########################', filename)
+    write('# 5.) Set path to GDAL _bin_ directory', filename)
     write('# More related to Windows, but also to other OS in case of a non standard location of GDAL', filename)
     write('# ON WINDOWS install \'OSGeo4W\' (recommanded) or \'FWTools\'', filename)
     write('# consult \'?MODISoptions\' for more details', filename)        
@@ -487,8 +510,9 @@ MODISoptions <- function(localArcPath, outDirPath, pixelSize, outProj,
   cat('pixelSize      :', opt$pixelSize, '\n')
   cat('outProj        :', opt$outProj, '\n')
   cat('resamplingType :', opt$resamplingType, '\n')
-  cat('dataFormat     :', opt$dataFormat, '\n\n\n')
-
+  cat('dataFormat     :', opt$dataFormat, '\n')
+  cat('cellchunk      :', opt$cellchunk,'\n\n\n')
+
   # remove ftpstring* from opt (old "~/.MODIS_Opts.R" style)
   oldftp <- grep(names(opt),pattern="^ftpstring*")