Feature 1319 no pickle (#1720)

* Try path insert.

* sys.path insert.

* Per #1319, adding David's changes back into the feature_1319_no_pickle branch. It compiles but TEST: python_numpy_plot_data_plane_pickle fails when testing on my Mac. Comitting now to test on kiowa.

* Per #1319, small updated to write_tmp_dataplane.py script. Had a couple of if statements that should really be elif.

Co-authored-by: John Halley Gotway <johnhg@kiowa.rap.ucar.edu>
Co-authored-by: John Halley Gotway <johnhg@ucar.edu>

met/data/wrappers/Makefile.am

@@ -23,10 +23,11 @@ wrappersdir = $(pkgdatadir)/wrappers
 wrappers_DATA = \
      generic_python.py \
      generic_pickle.py \
-     read_pickle_dataplane.py \
-     write_pickle_dataplane.py \
+     read_tmp_dataplane.py \
+     write_tmp_dataplane.py \
      write_pickle_mpr.py \
-     write_pickle_point.py
+     read_tmp_ascii.py \
+     write_tmp_point.py
 
 EXTRA_DIST = ${wrappers_DATA}
 

met/data/wrappers/read_tmp_ascii.py

@@ -0,0 +1,49 @@
+"""
+Module Name: read_tmp_ascii.py
+
+Read MET Point Observations from a text file created by write_tmp_point.py script
+  or MET Matched Pairs from a text file created by write_tmp_mpr.py script
+
+Point observation format:
+    Message_Type, Station_ID, Valid_Time, Lat, Lon, Elevation,
+    GRIB_Code or Variable_Name, Level, Height, QC_String, Observation_Value
+
+Version  Date
+1.0.0    2021/02/18  David Fillmore  Initial version
+"""
+
+__author__ = 'David Fillmore'
+__version__ = '1.0.0'
+__email__ = 'met_help@ucar.edu'
+
+import argparse
+
+point_data = None
+
+def read_tmp_ascii(filename):
+    """
+    Arguments:
+        filename (string): temporary file created by write_tmp_point.py
+
+    Returns:
+        (list of lists): point data
+    """
+    print('read_tmp_ascii:' + filename)
+    f = open(filename, 'r')
+    lines = f.readlines()
+    f.close()
+
+    global point_data
+    point_data = [eval(line.strip('\n')) for line in lines]
+
+    return point_data
+
+if __name__ == '__main__':
+    """
+    Parse command line arguments
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--filename', type=str)
+    args = parser.parse_args()
+
+    data = read_tmp_ascii(args.filename)

met/data/wrappers/read_tmp_dataplane.py

@@ -0,0 +1,36 @@
+########################################################################
+#
+#    Reads temporary file into memory.
+#
+#    usage:  /path/to/python read_tmp_dataplane.py dataplane.tmp
+#
+########################################################################
+
+import sys
+import numpy as np
+import netCDF4 as nc
+
+print('Python Script:\t', sys.argv[0])
+met_info = {}
+
+netcdf_filename = sys.argv[1]
+print('Read NetCDF:\t',  netcdf_filename)
+
+# read NetCDF file
+ds = nc.Dataset(netcdf_filename, 'r')
+met_data = ds['met_data'][:]
+met_attrs = {}
+grid = {}
+for attr, attr_val in ds.__dict__.items():
+    if 'grid' in attr:
+        grid_attr = attr.split('.')[1]
+        grid[grid_attr] = attr_val
+    else:
+        met_attrs[attr] = attr_val
+grid['nx'], grid['ny'] = int(grid['nx']), int(grid['ny'])
+met_attrs['grid'] = grid
+met_attrs['name'] = met_attrs['name_str']
+del met_attrs['name_str']
+met_info['met_data'] = met_data
+met_info['attrs'] = met_attrs
+print(met_info)

met/data/wrappers/write_tmp_dataplane.py

@@ -0,0 +1,68 @@
+########################################################################
+#
+#    Adapted from a script provided by George McCabe
+#    Adapted by Randy Bullock
+#
+#    usage:  /path/to/python write_tmp_dataplane.py \
+#            tmp_output_filename <user_python_script>.py <args>
+#
+########################################################################
+
+import os
+import sys
+import importlib.util
+import netCDF4 as nc
+
+print('Python Script:\t', sys.argv[0])
+print('User Command:\t',  sys.argv[2:])
+
+netcdf_filename = sys.argv[1]
+
+print('Write NetCDF:\t',  netcdf_filename)
+
+pyembed_module_name = sys.argv[2]
+sys.argv = sys.argv[2:]
+
+# append user script dir to system path
+pyembed_dir, pyembed_file = os.path.split(pyembed_module_name)
+if pyembed_dir:
+    sys.path.insert(0, pyembed_dir)
+
+if not pyembed_module_name.endswith('.py'):
+    pyembed_module_name += '.py'
+
+user_base = os.path.basename(pyembed_module_name).replace('.py','')
+
+spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name)
+met_in = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(met_in)
+
+met_info = {'met_data': met_in.met_data}
+if hasattr(met_in.met_data, 'attrs') and met_in.met_data.attrs:
+    attrs = met_in.met_data.attrs
+else:
+    attrs = met_in.attrs
+met_info['attrs'] = attrs
+
+print('write_tmp_dataplane')
+print(met_info)
+
+# write NetCDF file
+ds = nc.Dataset(netcdf_filename, 'w')
+
+nx, ny = met_in.met_data.shape
+ds.createDimension('x', nx)
+ds.createDimension('y', ny)
+dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y'))
+dp[:] = met_in.met_data
+
+for attr, attr_val in met_info['attrs'].items():
+    print(attr, attr_val, type(attr_val))
+    if attr == 'name':
+        setattr(ds, 'name_str', attr_val)
+    elif type(attr_val) == str:
+        setattr(ds, attr, attr_val)
+    elif type(attr_val) == dict:
+        for key in attr_val:
+            setattr(ds, attr + '.' + key, attr_val[key])
+ds.close()

met/data/wrappers/write_pickle_point.py → met/data/wrappers/write_tmp_point.py

@@ -3,21 +3,20 @@
 #    Adapted from a script provided by George McCabe
 #    Adapted by Randy Bullock
 #
-#    usage:  /path/to/python write_pickle_point.py \
-#            pickle_output_filename <user_python_script>.py <args>
+#    usage:  /path/to/python write_tmp_point.py \
+#            tmp_ascii_output_filename <user_python_script>.py <args>
 #
 ########################################################################
 
 import os
 import sys
-import pickle
 import importlib.util
 
 print('Python Script:\t', sys.argv[0])
 print('User Command:\t',  sys.argv[2:])
-print('Write Pickle:\t',  sys.argv[1])
+print('Write Temporary Ascii:\t',  sys.argv[1])
 
-pickle_filename = sys.argv[1]
+tmp_filename = sys.argv[1]
 
 pyembed_module_name = sys.argv[2]
 sys.argv = sys.argv[2:]
@@ -28,4 +27,6 @@
 met_in = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(met_in)
 
-pickle.dump( met_in.point_data, open( pickle_filename, "wb" ) )
+f = open(tmp_filename, 'w')
+for line in met_in.point_data:
+    f.write(str(line) + '\n')