diff --git a/README.rst b/README.rst index decbed8c..27a3b0ff 100644 --- a/README.rst +++ b/README.rst @@ -80,22 +80,19 @@ itself. .. code-block:: python - def execute(mp, resampling="nearest"): + def execute(mp, dem, land_polygons, resampling="nearest"): - # Open elevation model. - with mp.open("dem") as src: - # Skip tile if there is no data available or read data into a NumPy array. - if src.is_empty(1): - return "empty" - else: - dem = src.read(1, resampling=resampling) + # Skip tile if there is no data available or read data into a NumPy array. + if dem.is_empty(1): + return "empty" + else: + dem_data = dem.read(1, resampling=resampling) # Create hillshade using a built-in hillshade function. - hillshade = mp.hillshade(dem) + hillshade = mp.hillshade(dem_data) # Clip with polygons from vector file and return result. - with mp.open("land_polygons") as land_file: - return mp.clip(hillshade, land_file.read()) + return mp.clip(hillshade, land_polygons.read()) You can then interactively inspect the process output directly on a map in a diff --git a/doc/source/index.rst b/doc/source/index.rst index 1b852730..3cab9ca7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -61,22 +61,19 @@ vector dataset could look like this: # content of hillshade.py - def execute(mp, resampling="nearest"): + def execute(mp, dem, land_polygons, resampling="nearest"): - # Open elevation model. - with mp.open("dem") as src: - # Skip tile if there is no data available or read data into a NumPy array. - if src.is_empty(1): - return "empty" - else: - dem = src.read(1, resampling=resampling) + # Skip tile if there is no data available or read data into a NumPy array. + if dem.is_empty(1): + return "empty" + else: + dem_data = dem.read(1, resampling=resampling) # Create hillshade using a built-in hillshade function. - hillshade = mp.hillshade(dem) + hillshade = mp.hillshade(dem_data) # Clip with polygons from vector file and return result. - with mp.open("land_polygons") as land_file: - return mp.clip(hillshade, land_file.read()) + return mp.clip(hillshade, land_polygons.read()) Examine the result in your browser by serving the process by pointing it to diff --git a/mapchete/_processing.py b/mapchete/_processing.py index bf655415..468aec5d 100644 --- a/mapchete/_processing.py +++ b/mapchete/_processing.py @@ -87,18 +87,25 @@ def _execute(self): process_func = get_process_func( process_path=self.process_path, config_dir=self.config_dir ) + mp_obj = MapcheteProcess( + tile=self.tile, + params=self.process_func_params, + input=self.input, + output_params=self.output_params + ) try: with Timer() as t: + params = [ + # magic mp object + mp_obj if name == "mp" + # process input + else mp_obj.input[name] if name in mp_obj.input + # process parameter + else value + for name, value in self.process_func_params.items() + ] # Actually run process. - process_data = process_func( - MapcheteProcess( - tile=self.tile, - params=self.process_func_params, - input=self.input, - output_params=self.output_params - ), - **self.process_func_params - ) + process_data = process_func(*params) except MapcheteNodataTile: raise except Exception as e: diff --git a/mapchete/config.py b/mapchete/config.py index 0b745dd3..efebd60e 100644 --- a/mapchete/config.py +++ b/mapchete/config.py @@ -381,7 +381,7 @@ def input(self): for k, v in raw_inputs.items(): # for files and tile directories if isinstance(v, str): - logger.debug("load input reader for simple input %s", v) + logger.debug("load input reader for simple input %s", v) try: reader = load_input_reader( dict( @@ -481,10 +481,54 @@ def process_func(self): ) def get_process_func_params(self, zoom): - return { - k: v for k, v in self.params_at_zoom(zoom).items() - if k in inspect.signature(self.process_func).parameters - } + """ + Return process function parameters for zoom. + + The dictionary returned is a snapshot for given zoom which combines custom process + parameters and input datasets. + + This function also checks whether parameter names are also used as input names and + raises a MapcheteConfigError if this is the case. + + Parameters + ---------- + zoom : int + zoom level + + Returns + ------- + parameter map : dictionary + """ + if zoom not in self.init_zoom_levels: + raise ValueError("zoom level not available with current configuration") + process_func_params = inspect.signature(self.process_func).parameters + all_config_params = self.params_at_zoom(zoom) + + # look for config parameters which map on process function parameters + inputs = set(all_config_params["input"].keys()) & set(process_func_params) + custom_params = set(all_config_params.keys()) & set(process_func_params) + + # verify no parameter name intersection is configured + intersecting = custom_params & inputs + if intersecting: + raise MapcheteConfigError( + "custom parameters and inputs cannot have the same key: %s" % intersecting + ) + + # bring all together + return OrderedDict([ + # set mp value + (k, None) if k == "mp" + # input values from configuration + else (k, all_config_params["input"][k]) if k in all_config_params["input"] + # custom values from configuration + else (k, all_config_params[k]) if k in all_config_params + # default values from process function + else (k, v.default) + for k, v in process_func_params.items() + # excludes **kwargs from process function + if v.kind != v.VAR_KEYWORD + ]) def get_inputs_for_tile(self, tile): diff --git a/test/conftest.py b/test/conftest.py index 3ac70d19..e32d38a7 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -432,6 +432,13 @@ def output_single_gtiff_cog(): return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) +@pytest.fixture +def inputs_as_args(): + """Fixture for inputs_as_args.mapchete.""" + path = os.path.join(TESTDATA_DIR, "inputs_as_args.mapchete") + return ExampleConfig(path=path, dict=_dict_from_mapchete(path)) + + @pytest.fixture def s3_example_tile(gtiff_s3): """Example tile for fixture.""" diff --git a/test/example_process.py b/test/example_process.py index 673d9bf3..6b902918 100644 --- a/test/example_process.py +++ b/test/example_process.py @@ -1,13 +1,12 @@ """Example process file.""" -def execute(mp): +def execute(mp, file1): """User defined process.""" # Reading and writing data works like this: - with mp.open("file1") as raster_file: - if raster_file.is_empty(): - return "empty" - # This assures a transparent tile instead of a pink error tile - # is returned when using mapchete serve. - dem = raster_file.read(resampling="bilinear") + if file1.is_empty(): + return "empty" + # This assures a transparent tile instead of a pink error tile + # is returned when using mapchete serve. + dem = file1.read(resampling="bilinear") return dem diff --git a/test/test_config.py b/test/test_config.py index 58fe8907..52a57e44 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -257,3 +257,11 @@ def test_init_zoom(cleantopo_br): def test_process_module(process_module): mapchete.open(process_module.dict) + + +def test_inputs_as_args_intersection_error(mp_tmpdir, inputs_as_args): + config = inputs_as_args.dict + config.update(file1="some_str") + with pytest.raises(MapcheteConfigError): + with mapchete.open(config) as mp: + mp.execute((7, 61, 129)) diff --git a/test/test_mapchete.py b/test/test_mapchete.py index d71fa75d..cb9de79a 100644 --- a/test/test_mapchete.py +++ b/test/test_mapchete.py @@ -313,7 +313,6 @@ def test_baselevels_output_buffer(mp_tmpdir, baselevels_output_buffer): 171.46155, -87.27184, 174.45159, -84.31281, transform=src.transform ) subset = src.read(window=window, masked=True) - print(subset.shape) assert not subset.mask.any() pass @@ -567,3 +566,9 @@ def test_bufferedtiles(): assert a != tp_buffered.tile(5, 5, 5) assert a.get_neighbors() != a.get_neighbors(connectedness=4) + + +def test_inputs_as_args(mp_tmpdir, inputs_as_args): + config = inputs_as_args.dict + with mapchete.open(config) as mp: + mp.execute((7, 61, 129)) diff --git a/test/testdata/inputs_as_args.mapchete b/test/testdata/inputs_as_args.mapchete new file mode 100644 index 00000000..5c647b3c --- /dev/null +++ b/test/testdata/inputs_as_args.mapchete @@ -0,0 +1,40 @@ +# mandatory parameters +###################### +# this is the location of user python code: +process: inputs_as_args.py + +# zoom level range: +zoom_levels: + min: 7 + max: 11 +# or define single zoom level +# zoom_levels: 5 + +# geographical subset: +# bounds: [1.0, 2.0, 3.0, 4.0] + +# output pyramid definition + +pyramid: + grid: geodetic + metatiling: 1 # can be 1, 2, 4, 8, 16 (default 1) + + +input: + file1: + zoom>=10: dummy1.tif + file2: dummy2.tif +output: + path: tmp/example + format: GTiff + dtype: float32 + bands: 1 + +# free parameters +################# +some_integer_parameter: 12 +some_float_parameter: 5.3 +some_string_parameter: + zoom<=7: string1 + zoom>7: string2 +some_bool_parameter: true diff --git a/test/testdata/inputs_as_args.py b/test/testdata/inputs_as_args.py new file mode 100644 index 00000000..e2d50ffa --- /dev/null +++ b/test/testdata/inputs_as_args.py @@ -0,0 +1,8 @@ +"""Example process file.""" + + +def execute(file1, file2, herbert=None): + """User defined process.""" + assert file1 is None + dem = file2.read() + return dem