Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

GEMPAK performance updates #3431

Merged
merged 2 commits into from
Mar 14, 2024
Merged

GEMPAK performance updates #3431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
050f1f5
Performance updates
nawendt Mar 11, 2024
d811c65
Minor test update
nawendt Mar 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .codespellignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
thta = 2
lambda grid: grid if grid.PARM in parameter else False,
col_head.SELV,
'SELV': col_head.SELV,
'SELV': col_head.SELV,
col_head.SELV,
row_head.SELV,
'SELV': row_head.SELV,
'SELV': col_head.SELV,
'SELV': col_head.SELV,
'SELV': row_head.SELV,
'SELV': col_head.SELV,
Klystron Warmup Integer*2 N/A 0 to 1 1 0=Normal, 1=Preheat 146
# GFS, NAM, RAP, or other gridded dataset (e.g., NARR).
# This attribute can be set to False if the vector components are grid relative (e.g., for NAM
Expand Down
557 changes: 279 additions & 278 deletions src/metpy/io/gempak.py
View file
Open in desktop

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions src/metpy/static-data-manifest.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,20 @@ cubic_test.npz 8b9d01c2177a057b3352bb6c5b775dae6b796d37ba35b4775fcb65300dc06ccf
dec9_sounding.txt 4f60955bee4a59e2da0c225d778b9a04a149e9a17b4dce6bfefc111240b3b165
gem_azimuthal.grd 2b1cb2a358135035dbf6926f0e8ba8122f8b3a0477da5183943a1299ee602093
gem_azimuthal.npz 57d9f69cfa08893ae2d388bda7fb15a1b306ba205e3f15b9e40e963e7db79b0f
gem_big_endian.grd cf2828181311677081e86253e2b7eb5778450ee92cf6b8161dc0635687b1fd9d
gem_climate.csv 24c562fac9d5b81c08092ce5cf5522b57c74a63a184864e211016110fbc983c6
gem_climate.sfc 92f7fc278e16a9f2fbdb054b6c2a0f9f2aea420658841687e34ad2ad9a6552a7
gem_conical.grd 0184d05a596d623171135fc3243060aef69da8601c5ef8448bb6f39914474c3d
gem_conical.npz fd77f4e4fb2884087bc022c957277b4ec11bb21a35c9d3e261b64df425b1da28
gem_cylindrical.grd 1b2ee56e4ee50d8d3012aa319f24a7c09e092db8e8a5a8f58e2d857a8fd1520f
gem_cylindrical.npz 68ea33a58b24b651254dd4d2c935c33fe4f414ffd404296c9098b6404a37e408
gem_little_endian.grd 12c5c094c6505697c89426f362aa9bd6ba7de12f25c690ea73fb6d0159925536
gem_merged_nopack.csv 57ac3d9dc6b2461f0d0b8739452dceaa88f11fc8d55f75d8f05ed0981af0bf97
gem_merged_nopack.snd 141a1122e9638b0dde5d2ec424c03748d93577b84f10bbe7c29d9d3228200a88
gem_model_mrg.csv 304afb3bb4b662f7adfc8803e8661fc217a691d0b1790d5e0e6d085f198c4257
gem_model_mrg.snd 6d229a7af8d3e86d8da02403023cdaf0f86b6c982eb8b6f7f84c3836811df014
gem_multi_time.grd a11746f162f2ea3944573246a477db94026f76e72ce7ecb98bd08c8475e2ca3f
gem_multilevel_multidate.grd a5642ad733e88e9512d979b7c95049e9572591622d9ac94645622c1723742909
gem_packing_dec.grd 547cda1ffb5e143623a33afce76a6cfa8c3c543eed04cf388f3005ccf1ba677d
gem_packing_dec.npz b4b3b838b245c0e0d0d803c049a9169c34fe1c09f267d98df5a0d0586297d18d
gem_packing_diff.grd 35de37fb378fe1c8cb1be56962ab3496ee079a3c6df4ef838cf1e2e8fd86da3f
Expand Down
Binary file added staticdata/gem_big_endian.grd
View file
Open in desktop
Binary file not shown.
2 changes: 2 additions & 0 deletions staticdata/gem_climate.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
STN,YYMMDD/HHMM,PMSL,ALTI,TMPC,DWPC,SKNT,DRCT,GUST,WNUM,CHC1,CHC2,CHC3,VSBY,P03D,P03I,MSUN,SNOW,WEQS,P24I,TDXC,TDNC,P03C,CTYL,CTYM,CTYH,P06I,T6XC,T6NC,CEIL,P01I,SNEW
LWC,210401/0000,1031.3,30.44,9.4,-9.4,12.0,340.0,-9999.0,-9999.0,1004.0,-9999.0,-9999.0,10.0,3005.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,0.5,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0,-9999.0
Binary file added staticdata/gem_climate.sfc
View file
Open in desktop
Binary file not shown.
Binary file added staticdata/gem_little_endian.grd
View file
Open in desktop
Binary file not shown.
93 changes: 93 additions & 0 deletions staticdata/gem_merged_nopack.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
PRES,TEMP,DWPT,DRCT,SPED,HGHT
1000.0,-9999.0,-9999.0,-9999.0,-9999.0,261.0
990.0,8.4,-3.6,10.0,3.6,357.0
959.11,5.88,-5.19,350.0,5.15,610.0
925.0,3.0,-7.0,0.0,5.15,899.0
923.28,2.85,-7.03,0.0,5.15,914.0
888.92,-0.17,-7.6,5.0,5.66,1219.0
883.0,-0.7,-7.7,4.12,6.56,1272.77
871.0,-0.9,-11.9,2.32,8.42,1382.02
864.0,1.2,-22.8,1.26,9.51,1446.65
855.73,1.35,-25.65,0.0,10.81,1524.0
853.0,1.4,-26.6,357.62,11.3,1549.65
850.0,1.6,-15.4,355.0,11.84,1578.0
849.0,1.6,-12.4,354.81,11.86,1587.48
833.0,1.6,-10.4,351.76,12.18,1740.69
823.89,1.3,-12.18,350.0,12.36,1829.0
793.17,0.28,-18.32,300.0,7.21,2134.0
785.0,0.0,-20.0,293.17,8.34,2217.13
763.67,0.56,-4.71,275.0,11.33,2438.0
762.0,0.6,-3.5,274.71,11.77,2455.54
735.15,-0.18,-4.1,270.0,19.05,2743.0
718.0,-0.7,-4.5,266.9,20.33,2932.19
707.67,-0.84,-6.83,265.0,21.11,3048.0
703.0,-0.9,-7.9,265.0,19.86,3100.88
700.0,-1.3,-8.3,265.0,19.05,3135.0
655.04,-5.14,-13.67,275.0,17.5,3658.0
642.0,-6.3,-15.3,272.44,19.08,3816.46
626.0,-6.3,-16.3,269.23,21.07,4013.39
605.56,-7.95,-17.1,265.0,23.68,4267.0
559.08,-11.93,-19.02,270.0,27.8,4877.0
516.23,-15.91,-20.93,270.0,31.92,5486.0
500.0,-17.5,-21.7,265.0,32.43,5730.0
476.0,-20.7,-27.7,265.0,32.43,6096.72
412.0,-28.9,-31.7,273.3,34.57,7148.81
400.0,-30.3,-35.3,275.0,35.01,7360.0
386.0,-31.7,-40.7,270.13,35.01,7613.15
385.62,-31.76,-40.7,270.0,35.01,7620.0
369.28,-34.33,-40.7,270.0,34.49,7925.0
367.0,-34.7,-40.7,270.0,34.83,7968.52
339.0,-38.5,-52.5,270.0,39.21,8519.31
313.0,-42.5,-55.5,270.0,43.61,9063.95
309.31,-42.95,-56.23,270.0,44.27,9144.0
300.0,-44.1,-58.1,270.0,46.33,9350.0
279.0,-45.9,-62.9,268.01,53.09,9835.73
250.0,-50.5,-63.5,265.0,63.32,10560.0
245.85,-50.94,-63.57,265.0,65.38,10668.0
234.49,-52.17,-63.78,265.0,71.04,10973.0
231.0,-52.56,-63.84,265.0,71.04,11069.62
228.0,-52.9,-63.9,265.45,70.85,11153.9
200.0,-60.3,-69.3,270.0,68.98,11980.0
193.0,-61.1,-70.1,270.0,65.89,12202.11
185.0,-57.7,-68.7,270.0,63.73,12467.66
174.0,-58.3,-70.3,270.0,60.6,12854.69
151.55,-62.77,-73.84,270.0,53.54,13716.0
150.0,-63.1,-74.1,270.0,53.54,13780.0
130.0,-66.3,-77.3,265.0,46.33,14651.96
124.1,-64.8,-76.68,255.0,43.76,14935.0
118.05,-63.19,-76.02,270.0,45.3,15240.0
117.0,-62.9,-75.9,270.27,44.64,15294.27
101.0,-66.7,-77.7,274.7,33.69,16189.89
100.0,-66.3,-77.3,275.0,32.95,16250.0
91.8,-68.57,-78.94,280.0,28.83,16764.0
87.3,-69.9,-79.9,265.16,27.3,17065.63
87.25,-69.85,-79.88,265.0,27.28,17069.0
81.9,-63.7,-77.7,279.73,23.16,17451.51
75.07,-64.81,-77.7,300.0,17.5,17983.0
71.41,-65.45,-77.7,290.0,15.44,18288.0
70.0,-65.7,-77.7,290.0,15.44,18410.0
64.59,-66.09,-78.09,275.0,13.9,18898.0
55.55,-66.81,-78.81,285.0,11.33,19812.0
52.3,-67.1,-79.1,302.96,9.48,20177.21
50.23,-65.66,-78.56,315.0,8.24,20422.0
50.0,-65.5,-78.5,315.0,7.21,20450.0
47.9,-63.1,-78.1,291.22,3.78,20712.52
47.79,-63.11,-78.1,290.0,3.6,20726.0
43.27,-63.78,-78.21,265.0,4.63,21336.0
40.0,-64.3,-78.3,316.48,6.26,21818.41
39.18,-63.86,-78.17,330.0,6.69,21946.0
35.51,-61.74,-77.57,0.0,12.36,22555.0
32.9,-60.1,-77.1,2.26,10.73,23025.97
30.0,-61.5,-77.5,5.0,8.75,23600.0
21.7,-60.3,-78.3,36.95,8.75,25615.8
20.0,-56.1,-77.1,45.0,8.75,26130.0
18.5,-55.1,-77.1,58.32,8.36,26626.25
16.3,-55.77,-78.45,80.0,7.72,27432.0
12.7,-57.1,-81.1,27.95,4.24,29015.18
10.8,-53.9,-80.9,354.11,1.97,30047.19
10.09,-54.6,-81.6,340.0,1.03,30480.0
10.0,-54.7,-81.7,320.0,2.06,30540.0
9.1,-57.3,-83.3,-9999.0,-9999.0,31139.48
7.9,-48.9,-81.9,-9999.0,-9999.0,32050.37
7.6,-50.1,-83.1,-9999.0,-9999.0,32303.82
7.3,-52.5,-83.5,-9999.0,-9999.0,32565.37
Binary file added staticdata/gem_merged_nopack.snd
View file
Open in desktop
Binary file not shown.
Binary file added staticdata/gem_multilevel_multidate.grd
View file
Open in desktop
Binary file not shown.
105 changes: 104 additions & 1 deletion tests/io/test_gempak.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import logging

import numpy as np
from numpy.testing import assert_allclose, assert_almost_equal
from numpy.testing import assert_allclose, assert_almost_equal, assert_equal
import pandas as pd
import pytest

Expand All @@ -17,6 +17,18 @@
logging.getLogger('metpy.io.gempak').setLevel(logging.ERROR)


@pytest.mark.parametrize('order', ['little', 'big'])
def test_byte_swap(order):
""""Test byte swapping."""
g = get_test_data(f'gem_{order}_endian.grd')
Dismissed Show dismissed Hide dismissed

grid = GempakGrid(g).gdxarray()[0].squeeze()

reference = np.ones((113, 151), dtype='int32')

assert_equal(grid, reference)


@pytest.mark.parametrize('grid_name', ['none', 'diff', 'dec', 'grib'])
def test_grid_loading(grid_name):
"""Test reading grids with different packing."""
Expand Down Expand Up @@ -94,6 +106,35 @@
np.testing.assert_allclose(gdtar, ddtar, rtol=1e-10, atol=1e-2)


def test_merged_sounding_no_packing():
"""Test loading a merged sounding without data packing."""
gso = GempakSounding(get_test_data('gem_merged_nopack.snd')).snxarray(
Dismissed Show dismissed Hide dismissed
station_id='OUN')

gpres = gso[0].pressure.values
gtemp = gso[0].temp.values.squeeze()
gdwpt = gso[0].dwpt.values.squeeze()
gdrct = gso[0].drct.values.squeeze()
gsped = gso[0].sped.values.squeeze()
ghght = gso[0].hght.values.squeeze()

gempak = pd.read_csv(get_test_data('gem_merged_nopack.csv', as_file_obj=False),
Dismissed Show dismissed Hide dismissed
na_values=-9999)
dpres = gempak.PRES.values
dtemp = gempak.TEMP.values
ddwpt = gempak.DWPT.values
ddrct = gempak.DRCT.values
dsped = gempak.SPED.values
dhght = gempak.HGHT.values

assert_allclose(gpres, dpres, rtol=1e-10, atol=1e-2)
assert_allclose(gtemp, dtemp, rtol=1e-10, atol=1e-2)
assert_allclose(gdwpt, ddwpt, rtol=1e-10, atol=1e-2)
assert_allclose(gdrct, ddrct, rtol=1e-10, atol=1e-2)
assert_allclose(gsped, dsped, rtol=1e-10, atol=1e-2)
assert_allclose(ghght, dhght, rtol=1e-10, atol=1e-1)


@pytest.mark.parametrize('gem,gio,station', [
('gem_sigw_hght_unmrg.csv', 'gem_sigw_hght_unmrg.snd', 'TOP'),
('gem_sigw_pres_unmrg.csv', 'gem_sigw_pres_unmrg.snd', 'WAML')
Expand Down Expand Up @@ -160,6 +201,24 @@
assert_allclose(ghght, dhght, rtol=1e-10, atol=1e-1)


def test_climate_surface():
"""Test to read a cliamte surface file."""
gsf = GempakSurface(get_test_data('gem_climate.sfc'))
Dismissed Show dismissed Hide dismissed
gstns = gsf.sfjson()

gempak = pd.read_csv(get_test_data('gem_climate.csv', as_file_obj=False))
Dismissed Show dismissed Hide dismissed
gempak['YYMMDD/HHMM'] = pd.to_datetime(gempak['YYMMDD/HHMM'], format='%y%m%d/%H%M')
gempak = gempak.set_index(['STN', 'YYMMDD/HHMM'])

for stn in gstns:
idx_key = (stn['properties']['station_id'],
stn['properties']['date_time'])
gemsfc = gempak.loc[idx_key, :]

for param, val in stn['values'].items():
assert val == pytest.approx(gemsfc[param.upper()])


def test_standard_surface():
"""Test to read a standard surface file."""
skip = ['text', 'spcl']
Expand Down Expand Up @@ -261,6 +320,20 @@
assert dat == datetime(2000, 1, 2)


@pytest.mark.parametrize('access_type', ['STID', 'STNM'])
def test_surface_access(access_type):
"""Test for proper surface retrieval with multi-parameter filter."""
g = get_test_data('gem_surface_with_text.sfc')
gsf = GempakSurface(g)

if access_type == 'STID':
gsf.sfjson(station_id='MSN', country='US', state='WI',
date_time='202109070000')
elif access_type == 'STNM':
gsf.sfjson(station_number=726410, country='US', state='WI',
date_time='202109070000')


@pytest.mark.parametrize('text_type,date_time', [
('text', '202109070000'), ('spcl', '202109071600')
])
Expand All @@ -276,6 +349,20 @@
assert text == gem_text


@pytest.mark.parametrize('access_type', ['STID', 'STNM'])
def test_sounding_access(access_type):
"""Test for proper sounding retrieval with multi-parameter filter."""
g = get_test_data('gem_merged_nopack.snd')
gso = GempakSounding(g)

if access_type == 'STID':
gso.snxarray(station_id='OUN', country='US', state='OK',
date_time='202101200000')
elif access_type == 'STNM':
gso.snxarray(station_number=72357, country='US', state='OK',
date_time='202101200000')


@pytest.mark.parametrize('text_type', ['txta', 'txtb', 'txtc', 'txpb'])
def test_sounding_text(text_type):
"""Test for proper decoding of coded message text."""
Expand Down Expand Up @@ -313,6 +400,22 @@
assert stn['vsby'] == 2


def test_multi_level_multi_time_access():
"""Test accessing data with multiple levels and times."""
g = get_test_data('gem_multilevel_multidate.grd')

grid = GempakGrid(g)

grid.gdxarray(
parameter='STPC',
date_time='202403040000',
coordinate='HGHT',
level=0,
date_time2='202403050000',
level2=1
)


def test_multi_time_grid():
"""Test files with multiple times on a single grid."""
g = get_test_data('gem_multi_time.grd')
Expand Down