From 28d37ec809ef90b2a084e376089f053bb6719761 Mon Sep 17 00:00:00 2001 From: Meghan Jones Date: Tue, 23 Mar 2021 15:36:13 -0400 Subject: [PATCH 1/5] Wrap blockmean (#1092) * Rename blockmedian.py to blockm.py * Refactor blockmedian to support mean and mode * Wrap method blockmean * Add tests for blockmean method Co-authored-by: Dongdong Tian Co-authored-by: Wei Ji <23487320+weiji14@users.noreply.github.com> --- doc/api/index.rst | 1 + pygmt/__init__.py | 1 + pygmt/src/__init__.py | 2 +- pygmt/src/{blockmedian.py => blockm.py} | 146 +++++++++++++++++++----- pygmt/tests/test_blockmean.py | 78 +++++++++++++ 5 files changed, 199 insertions(+), 29 deletions(-) rename pygmt/src/{blockmedian.py => blockm.py} (51%) create mode 100644 pygmt/tests/test_blockmean.py diff --git a/doc/api/index.rst b/doc/api/index.rst index 58a41dd8cec..14a8fd31708 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -67,6 +67,7 @@ Operations on tabular data: .. autosummary:: :toctree: generated + blockmean blockmedian surface diff --git a/pygmt/__init__.py b/pygmt/__init__.py index 17e6f295b94..2aeb76d10a6 100644 --- a/pygmt/__init__.py +++ b/pygmt/__init__.py @@ -29,6 +29,7 @@ from pygmt.session_management import begin as _begin from pygmt.session_management import end as _end from pygmt.src import ( + blockmean, blockmedian, config, grd2cpt, diff --git a/pygmt/src/__init__.py b/pygmt/src/__init__.py index 93930a3a55c..1ea68b899fc 100644 --- a/pygmt/src/__init__.py +++ b/pygmt/src/__init__.py @@ -3,7 +3,7 @@ """ # pylint: disable=import-outside-toplevel from pygmt.src.basemap import basemap -from pygmt.src.blockmedian import blockmedian +from pygmt.src.blockm import blockmean, blockmedian from pygmt.src.coast import coast from pygmt.src.colorbar import colorbar from pygmt.src.config import config diff --git a/pygmt/src/blockmedian.py b/pygmt/src/blockm.py similarity index 51% rename from pygmt/src/blockmedian.py rename to pygmt/src/blockm.py index 350f12ba5f0..a9cf45f4616 100644 --- a/pygmt/src/blockmedian.py +++ b/pygmt/src/blockm.py @@ -1,5 +1,5 @@ """ -blockmedian - Block average (x,y,z) data tables by median estimation. +blockm - Block average (x,y,z) data tables by mean or median estimation. """ import pandas as pd from pygmt.clib import Session @@ -15,6 +15,122 @@ ) +def _blockm(block_method, table, outfile, **kwargs): + r""" + Block average (x,y,z) data tables by mean or median estimation. + + Reads arbitrarily located (x,y,z) triples [or optionally weighted + quadruples (x,y,z,w)] from a table and writes to the output a mean or + median (depending on ``block_method``) position and value for every + non-empty block in a grid region defined by the ``region`` and ``spacing`` + parameters. + + Parameters + ---------- + block_method : str + Name of the GMT module to call. Must be "blockmean" or "blockmedian". + + Returns + ------- + output : pandas.DataFrame or None + Return type depends on whether the ``outfile`` parameter is set: + + - :class:`pandas.DataFrame` table with (x, y, z) columns if ``outfile`` + is not set + - None if ``outfile`` is set (filtered output will be stored in file + set by ``outfile``) + """ + + kind = data_kind(table) + with GMTTempFile(suffix=".csv") as tmpfile: + with Session() as lib: + if kind == "matrix": + if not hasattr(table, "values"): + raise GMTInvalidInput(f"Unrecognized data type: {type(table)}") + file_context = lib.virtualfile_from_matrix(table.values) + elif kind == "file": + if outfile is None: + raise GMTInvalidInput("Please pass in a str to 'outfile'") + file_context = dummy_context(table) + else: + raise GMTInvalidInput(f"Unrecognized data type: {type(table)}") + + with file_context as infile: + if outfile is None: + outfile = tmpfile.name + arg_str = " ".join([infile, build_arg_string(kwargs), "->" + outfile]) + lib.call_module(module=block_method, args=arg_str) + + # Read temporary csv output to a pandas table + if outfile == tmpfile.name: # if user did not set outfile, return pd.DataFrame + result = pd.read_csv(tmpfile.name, sep="\t", names=table.columns) + elif outfile != tmpfile.name: # return None if outfile set, output in outfile + result = None + + return result + + +@fmt_docstring +@use_alias( + I="spacing", + R="region", + V="verbose", + a="aspatial", + f="coltypes", + r="registration", +) +@kwargs_to_strings(R="sequence") +def blockmean(table, outfile=None, **kwargs): + r""" + Block average (x,y,z) data tables by mean estimation. + + Reads arbitrarily located (x,y,z) triples [or optionally weighted + quadruples (x,y,z,w)] from a table and writes to the output a mean + position and value for every non-empty block in a grid region defined by + the ``region`` and ``spacing`` parameters. + + Full option list at :gmt-docs:`blockmean.html` + + {aliases} + + Parameters + ---------- + table : pandas.DataFrame or str + Either a pandas dataframe with (x, y, z) or (longitude, latitude, + elevation) values in the first three columns, or a file name to an + ASCII data table. + + spacing : str + *xinc*\[\ *unit*\][**+e**\|\ **n**] + [/*yinc*\ [*unit*][**+e**\|\ **n**]]. + *xinc* [and optionally *yinc*] is the grid spacing. + + region : str or list + *xmin/xmax/ymin/ymax*\[\ **+r**\][**+u**\ *unit*]. + Specify the region of interest. + + outfile : str + Required if ``table`` is a file. The file name for the output ASCII + file. + + {V} + {a} + {f} + {r} + + Returns + ------- + output : pandas.DataFrame or None + Return type depends on whether the ``outfile`` parameter is set: + + - :class:`pandas.DataFrame` table with (x, y, z) columns if ``outfile`` + is not set + - None if ``outfile`` is set (filtered output will be stored in file + set by ``outfile``) + """ + return _blockm(block_method="blockmean", table=table, outfile=outfile, **kwargs) + + @fmt_docstring @use_alias( I="spacing", @@ -73,30 +189,4 @@ def blockmedian(table, outfile=None, **kwargs): - None if ``outfile`` is set (filtered output will be stored in file set by ``outfile``) """ - kind = data_kind(table) - with GMTTempFile(suffix=".csv") as tmpfile: - with Session() as lib: - if kind == "matrix": - if not hasattr(table, "values"): - raise GMTInvalidInput(f"Unrecognized data type: {type(table)}") - file_context = lib.virtualfile_from_matrix(table.values) - elif kind == "file": - if outfile is None: - raise GMTInvalidInput("Please pass in a str to 'outfile'") - file_context = dummy_context(table) - else: - raise GMTInvalidInput(f"Unrecognized data type: {type(table)}") - - with file_context as infile: - if outfile is None: - outfile = tmpfile.name - arg_str = " ".join([infile, build_arg_string(kwargs), "->" + outfile]) - lib.call_module(module="blockmedian", args=arg_str) - - # Read temporary csv output to a pandas table - if outfile == tmpfile.name: # if user did not set outfile, return pd.DataFrame - result = pd.read_csv(tmpfile.name, sep="\t", names=table.columns) - elif outfile != tmpfile.name: # return None if outfile set, output in outfile - result = None - - return result + return _blockm(block_method="blockmedian", table=table, outfile=outfile, **kwargs) diff --git a/pygmt/tests/test_blockmean.py b/pygmt/tests/test_blockmean.py new file mode 100644 index 00000000000..76adb922f6b --- /dev/null +++ b/pygmt/tests/test_blockmean.py @@ -0,0 +1,78 @@ +""" +Tests for blockmean. +""" +import os + +import numpy.testing as npt +import pandas as pd +import pytest +from pygmt import blockmean +from pygmt.datasets import load_sample_bathymetry +from pygmt.exceptions import GMTInvalidInput +from pygmt.helpers import GMTTempFile, data_kind + + +def test_blockmean_input_dataframe(): + """ + Run blockmean by passing in a pandas.DataFrame as input. + """ + dataframe = load_sample_bathymetry() + output = blockmean(table=dataframe, spacing="5m", region=[245, 255, 20, 30]) + assert isinstance(output, pd.DataFrame) + assert all(dataframe.columns == output.columns) + assert output.shape == (5849, 3) + npt.assert_allclose(output.iloc[0], [245.888877, 29.978707, -384.0]) + + return output + + +def test_blockmean_wrong_kind_of_input_table_matrix(): + """ + Run blockmean using table input that is not a pandas.DataFrame but still a + matrix. + """ + dataframe = load_sample_bathymetry() + invalid_table = dataframe.values + assert data_kind(invalid_table) == "matrix" + with pytest.raises(GMTInvalidInput): + blockmean(table=invalid_table, spacing="5m", region=[245, 255, 20, 30]) + + +def test_blockmean_wrong_kind_of_input_table_grid(): + """ + Run blockmean using table input that is not a pandas.DataFrame or file but + a grid. + """ + dataframe = load_sample_bathymetry() + invalid_table = dataframe.bathymetry.to_xarray() + assert data_kind(invalid_table) == "grid" + with pytest.raises(GMTInvalidInput): + blockmean(table=invalid_table, spacing="5m", region=[245, 255, 20, 30]) + + +def test_blockmean_input_filename(): + """ + Run blockmean by passing in an ASCII text file as input. + """ + with GMTTempFile() as tmpfile: + output = blockmean( + table="@tut_ship.xyz", + spacing="5m", + region=[245, 255, 20, 30], + outfile=tmpfile.name, + ) + assert output is None # check that output is None since outfile is set + assert os.path.exists(path=tmpfile.name) # check that outfile exists at path + output = pd.read_csv(tmpfile.name, sep="\t", header=None) + assert output.shape == (5849, 3) + npt.assert_allclose(output.iloc[0], [245.888877, 29.978707, -384.0]) + + return output + + +def test_blockmean_without_outfile_setting(): + """ + Run blockmean by not passing in outfile parameter setting. + """ + with pytest.raises(GMTInvalidInput): + blockmean(table="@tut_ship.xyz", spacing="5m", region=[245, 255, 20, 30]) From c064d4a57497e8f07960fb0c30d68a604015f6f0 Mon Sep 17 00:00:00 2001 From: Will Schlitzer Date: Tue, 23 Mar 2021 19:38:24 +0000 Subject: [PATCH 2/5] Update the check_figures_equal testing section in CONTRIBUTING.md (#1108) Co-authored-by: Wei Ji <23487320+weiji14@users.noreply.github.com> Co-authored-by: Meghan Jones --- CONTRIBUTING.md | 48 ++++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9668e76e023..23c75b934d1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -369,35 +369,13 @@ or run tests which contain names that match a specific keyword expression: Writing an image-based test is only slightly more difficult than a simple test. The main consideration is that you must specify the "baseline" or reference image, and compare it with a "generated" or test image. This is handled using -the *decorator* functions `@check_figures_equal` and -`@pytest.mark.mpl_image_compare` whose usage are further described below. - -#### Using check_figures_equal - -This approach draws the same figure using two different methods (the reference -method and the tested method), and checks that both of them are the same. -It takes two `pygmt.Figure` objects ('fig_ref' and 'fig_test'), generates a png -image, and checks for the Root Mean Square (RMS) error between the two. -Here's an example: - -```python -@check_figures_equal() -def test_my_plotting_case(): - "Test that my plotting function works" - fig_ref, fig_test = Figure(), Figure() - fig_ref.grdimage("@earth_relief_01d_g", projection="W120/15c", cmap="geo") - fig_test.grdimage(grid, projection="W120/15c", cmap="geo") - return fig_ref, fig_test -``` - -Note: This is the recommended way to test plots whenever possible, such as when -we want to compare a reference GMT plot created from NetCDF files with one -generated by PyGMT that passes through several layers of virtualfile machinery. -Using this method will help save space in the git repository by not having to -store baseline images as with the other method below. +the *decorator* functions `@pytest.mark.mpl_image_compare` and `@check_figures_equal` +whose usage are further described below. #### Using mpl_image_compare +> **This is the preferred way to test plots whenever possible.** + This method uses the [pytest-mpl](https://github.com/matplotlib/pytest-mpl) plug-in to test plot generating code. Every time the tests are run, `pytest-mpl` compares the generated plots with known @@ -502,6 +480,24 @@ summarized as follows: git push dvc push +#### Using check_figures_equal + +This approach draws the same figure using two different methods (the reference +method and the tested method), and checks that both of them are the same. +It takes two `pygmt.Figure` objects ('fig_ref' and 'fig_test'), generates a png +image, and checks for the Root Mean Square (RMS) error between the two. +Here's an example: + +```python +@check_figures_equal() +def test_my_plotting_case(): + "Test that my plotting function works" + fig_ref, fig_test = Figure(), Figure() + fig_ref.grdimage("@earth_relief_01d_g", projection="W120/15c", cmap="geo") + fig_test.grdimage(grid, projection="W120/15c", cmap="geo") + return fig_ref, fig_test +``` + ### Documentation #### Building the documentation From 1b42aa47ac9f83489a771f887ebde89492dfe0b9 Mon Sep 17 00:00:00 2001 From: Michael Grund <23025878+michaelgrund@users.noreply.github.com> Date: Tue, 23 Mar 2021 22:22:38 +0100 Subject: [PATCH 3/5] Rename tutorial.py to samples.py (#1110) * Rename tutorial.py to samples.py Co-authored-by: Wei Ji <23487320+weiji14@users.noreply.github.com> --- pygmt/datasets/__init__.py | 2 +- pygmt/datasets/{tutorial.py => samples.py} | 2 +- .../{test_datasets_tutorial.py => test_datasets_samples.py} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename pygmt/datasets/{tutorial.py => samples.py} (98%) rename pygmt/tests/{test_datasets_tutorial.py => test_datasets_samples.py} (96%) diff --git a/pygmt/datasets/__init__.py b/pygmt/datasets/__init__.py index 3c2a82408bb..a7cfdf17998 100644 --- a/pygmt/datasets/__init__.py +++ b/pygmt/datasets/__init__.py @@ -3,7 +3,7 @@ # Load sample data included with GMT (downloaded from the GMT cache server). from pygmt.datasets.earth_relief import load_earth_relief -from pygmt.datasets.tutorial import ( +from pygmt.datasets.samples import ( load_japan_quakes, load_ocean_ridge_points, load_sample_bathymetry, diff --git a/pygmt/datasets/tutorial.py b/pygmt/datasets/samples.py similarity index 98% rename from pygmt/datasets/tutorial.py rename to pygmt/datasets/samples.py index 226f0b89ab3..729ee1962b5 100644 --- a/pygmt/datasets/tutorial.py +++ b/pygmt/datasets/samples.py @@ -1,5 +1,5 @@ """ -Functions to load sample data from the GMT tutorials. +Functions to load sample data. """ import pandas as pd from pygmt.src import which diff --git a/pygmt/tests/test_datasets_tutorial.py b/pygmt/tests/test_datasets_samples.py similarity index 96% rename from pygmt/tests/test_datasets_tutorial.py rename to pygmt/tests/test_datasets_samples.py index ecd4c881b79..81148fc2a04 100644 --- a/pygmt/tests/test_datasets_tutorial.py +++ b/pygmt/tests/test_datasets_samples.py @@ -1,5 +1,5 @@ """ -Test basic functionality for loading datasets for tutorials. +Test basic functionality for loading sample datasets. """ from pygmt.datasets import ( load_japan_quakes, From 1ac4c59713b6deb242288c8876c89b4c7f5e8f20 Mon Sep 17 00:00:00 2001 From: Dongdong Tian Date: Tue, 23 Mar 2021 18:26:12 -0400 Subject: [PATCH 4/5] Update "GMT Dev Tests" workflow to test macOS-11.0 and pre-release Python packages (#1105) - Install pre-release Python packages using pip install --pre, so that the workflow also tests dev Python packages - Change "macOS-10.15" to "macOS-11.0" - Change "ubuntu-20.04" to "ubuntu-latest" (ubuntu-latest is just an alias of ubuntu-20.04) - Add "activate-environment: pygmt" to the setup-miniconda step - Add more paths to paths-ignore Co-authored-by: Wei Ji <23487320+weiji14@users.noreply.github.com> --- .github/workflows/ci_tests_dev.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_tests_dev.yaml b/.github/workflows/ci_tests_dev.yaml index d106dac1fe6..fe072c01fdb 100644 --- a/.github/workflows/ci_tests_dev.yaml +++ b/.github/workflows/ci_tests_dev.yaml @@ -1,5 +1,4 @@ -# This workflow installs PyGMT dependencies, builds documentation and runs tests on GMT dev version -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions +# This workflow installs PyGMT and runs tests on GMT dev version name: GMT Dev Tests @@ -10,10 +9,13 @@ on: types: [ready_for_review] paths-ignore: - 'doc/**' + - 'examples/**' - '*.md' - '*.json' - 'README.rst' - 'LICENSE.txt' + - '.gitignore' + - '.pylintrc' repository_dispatch: types: [test-gmt-dev-command] # Schedule daily tests @@ -28,7 +30,7 @@ jobs: fail-fast: false matrix: python-version: [3.9] - os: [ubuntu-20.04, macOS-10.15, windows-latest] + os: [ubuntu-latest, macOS-11.0, windows-latest] gmt_git_ref: [master] defaults: run: @@ -73,6 +75,7 @@ jobs: - name: Setup Miniconda uses: conda-incubator/setup-miniconda@v2.0.1 with: + activate-environment: pygmt python-version: ${{ matrix.python-version }} channels: conda-forge miniconda-version: "latest" @@ -81,8 +84,9 @@ jobs: - name: Install dependencies run: | conda install ninja cmake libblas libcblas liblapack fftw gdal \ - ghostscript libnetcdf hdf5 zlib curl pcre ipython \ - dvc pytest pytest-cov pytest-mpl + ghostscript libnetcdf hdf5 zlib curl pcre make dvc + pip install --pre numpy pandas xarray netCDF4 packaging \ + ipython pytest-cov pytest-mpl pytest>=6.0 sphinx-gallery # Build and install latest GMT from GitHub - name: Install GMT ${{ matrix.gmt_git_ref }} branch (Linux/macOS) From 65f5aee38b663af8053a309e10170cee2ef046f4 Mon Sep 17 00:00:00 2001 From: Yao Jiayuan Date: Wed, 24 Mar 2021 13:37:21 +0800 Subject: [PATCH 5/5] Allow passing an array as intensity for plot3d (#1109) Co-authored-by: Dongdong Tian --- pygmt/src/plot3d.py | 24 ++++++--- .../test_plot3d_varying_intensity.png.dvc | 4 ++ pygmt/tests/test_plot3d.py | 54 ++++++++++++------- 3 files changed, 55 insertions(+), 27 deletions(-) create mode 100644 pygmt/tests/baseline/test_plot3d_varying_intensity.png.dvc diff --git a/pygmt/src/plot3d.py b/pygmt/src/plot3d.py index fa49ed6f67a..85b4826cae4 100644 --- a/pygmt/src/plot3d.py +++ b/pygmt/src/plot3d.py @@ -111,11 +111,14 @@ def plot3d( Offset the plot symbol or line locations by the given amounts *dx*/*dy*\ [/*dz*] [Default is no offset]. {G} - intensity : float or bool - Provide an *intens* value (nominally in the -1 to +1 range) to - modulate the fill color by simulating illumination [Default is None]. - If using ``intensity=True``, we will instead read *intens* from the - first data column after the symbol parameters (if given). + intensity : float or bool or 1d array + Provide an *intensity* value (nominally in the -1 to +1 range) to + modulate the fill color by simulating illumination. If using + ``intensity=True``, we will instead read *intensity* from the first + data column after the symbol parameters (if given). *intensity* can + also be a 1d array to set varying intensity for symbols, but it is only + valid for ``x``/``y``/``z``. + close : str [**+b**\|\ **d**\|\ **D**][**+xl**\|\ **r**\|\ *x0*]\ [**+yl**\|\ **r**\|\ *y0*][**+p**\ *pen*]. @@ -183,9 +186,14 @@ def plot3d( ) extra_arrays.append(sizes) - if "t" in kwargs and is_nonstr_iter(kwargs["t"]): - extra_arrays.append(kwargs["t"]) - kwargs["t"] = "" + for flag in ["I", "t"]: + if flag in kwargs and is_nonstr_iter(kwargs[flag]): + if kind != "vectors": + raise GMTInvalidInput( + f"Can't use arrays for {plot3d.aliases[flag]} if data is matrix or file." + ) + extra_arrays.append(kwargs[flag]) + kwargs[flag] = "" with Session() as lib: # Choose how data will be passed in to the module diff --git a/pygmt/tests/baseline/test_plot3d_varying_intensity.png.dvc b/pygmt/tests/baseline/test_plot3d_varying_intensity.png.dvc new file mode 100644 index 00000000000..6003814adc7 --- /dev/null +++ b/pygmt/tests/baseline/test_plot3d_varying_intensity.png.dvc @@ -0,0 +1,4 @@ +outs: +- md5: 79f7d8062dbb6f29ffa0a3aaa7382f13 + size: 24052 + path: test_plot3d_varying_intensity.png diff --git a/pygmt/tests/test_plot3d.py b/pygmt/tests/test_plot3d.py index 82630af48c5..11ba0707897 100644 --- a/pygmt/tests/test_plot3d.py +++ b/pygmt/tests/test_plot3d.py @@ -130,30 +130,19 @@ def test_plot3d_fail_no_data(data, region): ) -def test_plot3d_fail_size_color(data, region): +def test_plot3d_fail_color_size_intensity(data, region): """ - Should raise an exception if array sizes and color are used with matrix. + Should raise an exception if array color, sizes and intensity are used with + matrix. """ fig = Figure() + kwargs = dict(data=data, region=region, projection="X10c", frame="afg") with pytest.raises(GMTInvalidInput): - fig.plot3d( - data=data, - region=region, - projection="X4i", - style="c0.2c", - color=data[:, 2], - frame="afg", - ) + fig.plot3d(style="c0.2c", color=data[:, 2], **kwargs) with pytest.raises(GMTInvalidInput): - fig.plot3d( - data=data, - region=region, - projection="X4i", - style="cc", - sizes=data[:, 2], - color="red", - frame="afg", - ) + fig.plot3d(style="cc", sizes=data[:, 2], color="red", **kwargs) + with pytest.raises(GMTInvalidInput): + fig.plot3d(style="cc", intensity=data[:, 2], color="red", **kwargs) @check_figures_equal() @@ -329,6 +318,33 @@ def test_plot3d_colors_sizes_proj(data, region): return fig_ref, fig_test +@pytest.mark.mpl_image_compare +def test_plot3d_varying_intensity(): + """ + Plot the data with array-like intensity. + """ + x = np.arange(-1, 1.1, 0.1) + y = np.zeros(x.size) + z = y + intensity = x + + fig = Figure() + fig.plot3d( + x=x, + y=y, + z=z, + region=[-1.1, 1.1, -0.5, 0.5, -0.5, 0.5], + projection="X15c/5c", + zsize="5c", + perspective=[135, 30], + frame=["Sltr", "xaf+lIntensity"], + style="c0.5c", + color="blue", + intensity=intensity, + ) + return fig + + @check_figures_equal() def test_plot3d_transparency(): """