raster module memory: set new value globally (#922)

* raster module memory: set other value globally This change allows to globally override the default 300 MB with a user defined value: r.in.gdal --help Imports raster data into a GRASS raster map using GDAL library. Usage: r.in.gdal [-ojeflakcrp] input=name output=name ... Parameters: input Name of raster file to be imported output Name for output raster map band Band(s) to select (default is all bands) memory Maximum memory to be used (in MB) default: 300 target Name of GCPs target location title Title for resultant raster map offset Offset to be added to band numbers default: 0 g.gisenv set="MEMORYMB=6000" r.in.gdal --help Imports raster data into a GRASS raster map using GDAL library. Usage: r.in.gdal [-ojeflakcrp] input=name output=name ... Parameters: input Name of raster file to be imported output Name for output raster map band Band(s) to select (default is all bands) memory Maximum memory to be used (in MB) default: 6000 target Name of GCPs target location title Title for resultant raster map offset Offset to be added to band numbers default: 0 g.gisenv set="MEMORYMB=300" Based upon demo patch my @metzm * fix double description * code simplification by defining new G_define_standard_option(G_OPT_MEMORYMB) * update STD_OPT_STRINGS after include/gis.h modification * Patch added to also set default answer at compile time * document new GRASS variable MEMORYMB - fixes the creation of man pages #922 (comment) Contributed by @metzm
OSGeo · Aug 26, 2020 · 0c4f02e · 0c4f02e
1 parent dbeb5f3
commit 0c4f02e
Show file tree

Hide file tree

Showing 21 changed files with 64 additions and 118 deletions.
diff --git a/general/g.parser/standard_option.c b/general/g.parser/standard_option.c
@@ -12,6 +12,7 @@ static char* STD_OPT_STRINGS[] = {"G_OPT_UNDEFINED",
                                   "G_OPT_DB_KEYCOLUMN",
                                   "G_OPT_I_GROUP",
                                   "G_OPT_I_SUBGROUP",
+                                  "G_OPT_MEMORYMB",
                                   "G_OPT_R_INPUT",
                                   "G_OPT_R_INPUTS",
                                   "G_OPT_R_OUTPUT",

diff --git a/imagery/i.ortho.photo/i.ortho.rectify/main.c b/imagery/i.ortho.photo/i.ortho.rectify/main.c
@@ -81,6 +81,7 @@ int main(int argc, char *argv[])
     struct Flag *c, *a, *pan_flag;
     struct GModule *module;
 
+
     G_gisinit(argv[0]);
 
     module = G_define_module();
@@ -107,13 +108,7 @@ int main(int argc, char *argv[])
     tres->required = NO;
     tres->description = _("Target resolution (ignored if -c flag used)");
 
-    mem = G_define_option();
-    mem->key = "memory";
-    mem->type = TYPE_DOUBLE;
-    mem->key_desc = "memory in MB";
-    mem->required = NO;
-    mem->answer = "300";
-    mem->description = _("Amount of memory to use in MB");
+    mem = G_define_standard_option(G_OPT_MEMORYMB);
 
     ipolname = make_ipol_list();
 

diff --git a/imagery/i.rectify/main.c b/imagery/i.rectify/main.c
@@ -77,6 +77,7 @@ int main(int argc, char *argv[])
     struct Flag *c, *a, *t;
     struct GModule *module;
 
+
     G_gisinit(argv[0]);
 
     module = G_define_module();
@@ -113,14 +114,8 @@ int main(int argc, char *argv[])
     tres->required = NO;
     tres->description = _("Target resolution (ignored if -c flag used)");
 
-    mem = G_define_option();
-    mem->key = "memory";
-    mem->type = TYPE_DOUBLE;
-    mem->key_desc = "memory in MB";
-    mem->required = NO;
-    mem->answer = "300";
-    mem->description = _("Amount of memory to use in MB");
-
+    mem = G_define_standard_option(G_OPT_MEMORYMB);
+
     ipolname = make_ipol_list();
 
     interpol = G_define_option();

diff --git a/imagery/i.segment/parse_args.c b/imagery/i.segment/parse_args.c
@@ -20,6 +20,7 @@ int parse_args(int argc, char *argv[], struct globals *globals)
     struct Option *gof, *endt;
     int bands;
 
+
     /* required parameters */
     group = G_define_standard_option(G_OPT_R_INPUTS);
     group->key = "group";
@@ -112,12 +113,7 @@ int parse_args(int argc, char *argv[], struct globals *globals)
     smooth_weight->guisection = _("Settings");
 #endif
 
-    mem = G_define_option();
-    mem->key = "memory";
-    mem->type = TYPE_INTEGER;
-    mem->required = NO;
-    mem->answer = "300";
-    mem->description = _("Memory in MB");
+    mem = G_define_standard_option(G_OPT_MEMORYMB);
 
     /* TODO input for distance function */
 

diff --git a/include/gis.h b/include/gis.h
@@ -241,6 +241,8 @@ typedef enum
 
     G_OPT_I_GROUP,		/*!< old input imagery group */
     G_OPT_I_SUBGROUP,		/*!< old input imagery subgroup */
+
+    G_OPT_MEMORYMB,		/*!< Maximum memory to be used (in MB): cache size for raster rows */
     G_OPT_R_INPUT,		/*!< old input raster map */
     G_OPT_R_INPUTS,		/*!< old input raster maps */
     G_OPT_R_OUTPUT,		/*!< new output raster map */

diff --git a/lib/gis/parser_standard_options.c b/lib/gis/parser_standard_options.c
@@ -52,6 +52,7 @@
    - G_OPT_I_SUBGROUP
 
   - raster:
+   - G_OPT_MEMORYMB
    - G_OPT_R_INPUT
    - G_OPT_R_INPUTS
    - G_OPT_R_OUTPUT
@@ -137,6 +138,7 @@
 struct Option *G_define_standard_option(int opt)
 {
     struct Option *Opt;
+    char *memstr;
 
     Opt = G_define_option();
 
@@ -245,6 +247,22 @@ struct Option *G_define_standard_option(int opt)
 	break;
 
 	/* raster maps */
+    case G_OPT_MEMORYMB:
+	Opt->key = "memory";
+	Opt->type = TYPE_INTEGER;
+	Opt->key_desc = "memory in MB";
+	Opt->required = NO;
+	Opt->multiple = NO;
+	Opt->answer = "300";
+	/* start dynamic answer */
+	/* check MEMORYMB in GISRC, set with g.gisenv */
+	memstr = G_store(G_getenv_nofatal("MEMORYMB"));
+	if (memstr && *memstr)
+	    Opt->answer = memstr;
+	/* end dynamic answer */
+	Opt->label = _("Maximum memory to be used (in MB)");
+	Opt->description = _("Cache size for raster rows");
+	break;
     case G_OPT_R_INPUT:
 	Opt->key = "input";
 	Opt->type = TYPE_STRING;

diff --git a/lib/init/variables.html b/lib/init/variables.html
@@ -531,6 +531,14 @@ <h2>List of selected GRASS gisenv variables</h2>
   <dt>MAPSET</dt>
   <dd>initial mapset</dd>
 
+  <dt>MEMORYMB</dt>
+  <dd>[entire GRASS with focus on raster related data processing]<br>
+    sets the maximum memory to be used (in MB), i.e. the cache size for raster rows
+<div class="code"><pre>
+# set to 6 GB (default: 300 MB)
+g.gisenv set="MEMORYMB=6000"
+</pre></div>
+
   <dt>OVERWRITE</dt>
   <dd>[all modules]<br>
     toggles map overwrite.

diff --git a/man/parser_standard_options.py b/man/parser_standard_options.py
@@ -50,7 +50,14 @@ def split_opt_line(line):
     def parse_glines(glines):
         res = {}
         key = None
+        dynamic_answer = False
         for line in glines:
+            if line.strip() == "/* start dynamic answer */":
+                dynamic_answer = True
+            if line.strip() == "/* end dynamic answer */":
+                dynamic_answer = False
+            if dynamic_answer or line.startswith('/*'):
+                continue
             if line.startswith('/*'):
                 continue
             if line.startswith(startswith) and line.endswith(';'):

diff --git a/raster/r.cost/main.c b/raster/r.cost/main.c
@@ -154,6 +154,7 @@ int main(int argc, char *argv[])
     double peak = 0.0;
     int dsize, nearest_size;
     double disk_mb, mem_mb, pq_mb;
+
     int dir_bin;
     DCELL mysolvedir[2], solvedir[2];
 
@@ -249,14 +250,7 @@ int main(int argc, char *argv[])
 	_("Cost assigned to null cells. By default, null cells are excluded");
     opt6->guisection = _("NULL cells");
 
-    opt10 = G_define_option();
-    opt10->key = "memory";
-    opt10->type = TYPE_INTEGER;
-    opt10->key_desc = "value";
-    opt10->required = NO;
-    opt10->multiple = NO;
-    opt10->answer = "300";
-    opt10->description = _("Maximum memory to be used in MB");
+    opt10 = G_define_standard_option(G_OPT_MEMORYMB);
 
     flag2 = G_define_flag();
     flag2->key = 'k';

diff --git a/raster/r.in.gdal/main.c b/raster/r.in.gdal/main.c
@@ -116,6 +116,7 @@ int main(int argc, char *argv[])
     struct Flag *flag_o, *flag_e, *flag_k, *flag_f, *flag_l, *flag_c, *flag_p,
         *flag_j, *flag_a, *flag_r;
 
+
     /* -------------------------------------------------------------------- */
     /*      Initialize.                                                     */
     /* -------------------------------------------------------------------- */
@@ -144,16 +145,10 @@ int main(int argc, char *argv[])
     parm.band->description = _("Band(s) to select (default is all bands)");
     parm.band->guisection = _("Bands");
 
-    parm.memory = G_define_option();
-    parm.memory->key = "memory";
-    parm.memory->type = TYPE_INTEGER;
-    parm.memory->required = NO;
+    parm.memory = G_define_standard_option(G_OPT_MEMORYMB);
 #if GDAL_VERSION_NUM < 1800
     parm.memory->options = "0-2047";
 #endif
-    parm.memory->answer = "300";
-    parm.memory->label = _("Maximum memory to be used (in MB)");
-    parm.memory->description = _("Cache size for raster rows");
 
     parm.target = G_define_option();
     parm.target->key = "target";

diff --git a/raster/r.proj/main.c b/raster/r.proj/main.c
@@ -186,13 +186,7 @@ int main(int argc, char **argv)
     interpol->guisection = _("Target");
     interpol->descriptions = make_ipol_desc();
 
-    memory = G_define_option();
-    memory->key = "memory";
-    memory->type = TYPE_INTEGER;
-    memory->required = NO;
-    memory->answer = "300";
-    memory->label = _("Maximum memory to be used (in MB)");
-    memory->description = _("Cache size for raster rows");
+    memory = G_define_standard_option(G_OPT_MEMORYMB);
 
     res = G_define_option();
     res->key = "resolution";

diff --git a/raster/r.resamp.bspline/main.c b/raster/r.resamp.bspline/main.c
@@ -50,6 +50,7 @@ int main(int argc, char *argv[])
     int seg_mb, segments_in_memory;
     int have_mask;
 
+
     int inrastfd, outrastfd;
     DCELL *drastbuf, dval;
     struct History history;
@@ -145,13 +146,7 @@ int main(int argc, char *argv[])
     cross_corr_flag->description =
 	_("Find the best Tykhonov regularizing parameter using a \"leave-one-out\" cross validation method");
 
-    memory_opt = G_define_option();
-    memory_opt->key = "memory";
-    memory_opt->type = TYPE_INTEGER;
-    memory_opt->required = NO;
-    memory_opt->answer = "300";
-    memory_opt->label = _("Maximum memory to be used (in MB)");
-    memory_opt->description = _("Cache size for raster rows");
+    memory_opt = G_define_standard_option(G_OPT_MEMORYMB);
 
     /*----------------------------------------------------------------*/
     /* Parsing */

diff --git a/raster/r.stream.extract/main.c b/raster/r.stream.extract/main.c
@@ -132,13 +132,7 @@ int main(int argc, char *argv[])
     input.min_stream_length->description =
 	_("Applies only to first-order stream segments (springs/stream heads)");
 
-    input.memory = G_define_option();
-    input.memory->key = "memory";
-    input.memory->type = TYPE_INTEGER;
-    input.memory->required = NO;
-    input.memory->answer = "300";
-    input.memory->label = _("Maximum memory to be used (in MB)");
-    input.memory->description = _("Cache size for raster rows");
+    input.memory = G_define_standard_option(G_OPT_MEMORYMB);
 
     output.stream_rast = G_define_standard_option(G_OPT_R_OUTPUT);
     output.stream_rast->key = "stream_raster";

diff --git a/raster/r.terraflow/main.cpp b/raster/r.terraflow/main.cpp
@@ -128,22 +128,17 @@ parse_args(int argc, char *argv[]) {
       "SFD (D8) direction (meaningful only for MFD flow). "
       "If no answer is given it defaults to infinity.");
 
-  /* main memory */
+  /* raster cache memory */
   struct Option *mem;
-  mem = G_define_option() ;
-  mem->key         = "memory";
-  mem->type        = TYPE_INTEGER;
-  mem->required    = NO;
-  mem->answer      = (char *) "300";
-  mem->description = _("Maximum memory to be used (in MB)");
+  mem = G_define_standard_option(G_OPT_MEMORYMB);
 
   /* temporary STREAM path */
   struct Option *streamdir;
   streamdir = G_define_option() ;
   streamdir->key        = "directory";
   streamdir->type       = TYPE_STRING;
   streamdir->required   = NO;
-  //streamdir->answer     = "";
+  /* streamdir->answer     = ""; */
   streamdir->description=
      _("Directory to hold temporary files (they can be large)");
 

diff --git a/raster/r.walk/main.c b/raster/r.walk/main.c
@@ -193,6 +193,7 @@ int main(int argc, char *argv[])
     double peak = 0.0;
     int dtm_dsize, cost_dsize, nearest_size;
     double disk_mb, mem_mb, pq_mb;
+
     int dir_bin;
     DCELL mysolvedir[2], solvedir[2];
 
@@ -295,14 +296,7 @@ int main(int argc, char *argv[])
 	_("Cost assigned to null cells. By default, null cells are excluded");
     opt6->guisection = _("NULL cells");
 
-    opt10 = G_define_option();
-    opt10->key = "memory";
-    opt10->type = TYPE_INTEGER;
-    opt10->key_desc = "value";
-    opt10->required = NO;
-    opt10->multiple = NO;
-    opt10->answer = "300";
-    opt10->description = _("Maximum memory to be used in MB");
+    opt10 = G_define_standard_option(G_OPT_MEMORYMB);
 
     opt15 = G_define_option();
     opt15->key = "walk_coeff";

diff --git a/raster/r.watershed/front/main.c b/raster/r.watershed/front/main.c
@@ -68,6 +68,7 @@ int main(int argc, char *argv[])
     struct Flag *flag_flat;
     struct GModule *module;
 
+
     G_gisinit(argv[0]);
 
     /* Set description */
@@ -208,12 +209,7 @@ int main(int argc, char *argv[])
     opt15->description =
 	_("1 = most diverging flow, 10 = most converging flow. Recommended: 5");
 
-    opt16 = G_define_option();
-    opt16->key = "memory";
-    opt16->type = TYPE_INTEGER;
-    opt16->required = NO;
-    opt16->answer = "300";	/* 300MB default value, please keep r.terraflow in sync */
-    opt16->description = _("Maximum memory to be used with -m flag (in MB)");
+    opt16 = G_define_standard_option(G_OPT_MEMORYMB);
 
     flag_sfd = G_define_flag();
     flag_sfd->key = 's';

diff --git a/scripts/r.fillnulls/r.fillnulls.py b/scripts/r.fillnulls/r.fillnulls.py
@@ -97,14 +97,7 @@
 #% answer: 0.01
 #% guisection: Spline options
 #%end
-#%option
-#% key: memory
-#% type: integer
-#% required: no
-#% multiple: no
-#% label: Maximum memory to be used (in MB)
-#% description: Cache size for raster rows
-#% answer: 300
+#%option G_OPT_MEMORYMB
 #%end
 
 

diff --git a/scripts/r.import/r.import.py b/scripts/r.import/r.import.py
@@ -34,14 +34,7 @@
 #% description: Input band(s) to select (default is all bands)
 #% guisection: Input
 #%end
-#%option
-#% key: memory
-#% type: integer
-#% required: no
-#% multiple: no
-#% label: Maximum memory to be used (in MB)
-#% description: Cache size for raster rows
-#% answer: 300
+#%option G_OPT_MEMORYMB
 #%end
 #%option G_OPT_R_OUTPUT
 #% description: Name for output raster map

diff --git a/temporal/t.rast.import/t.rast.import.py b/temporal/t.rast.import/t.rast.import.py
@@ -75,14 +75,7 @@
 #% multiple: no
 #%end
 
-#%option
-#% key: memory
-#% type: integer
-#% description: Cache size for raster rows
-#% label: Maximum memory to be used (in MB)
-#% options: 0-2047
-#% answer: 300
-#% multiple: no
+#%option G_OPT_MEMORYMB
 #%end
 
 #%flag