From f147aff975efdcc50dd62277f3a2f6cb61cf524d Mon Sep 17 00:00:00 2001 From: Teque5 Date: Wed, 19 Aug 2020 16:35:04 -0700 Subject: [PATCH] ordered metadata; improved setup; better use cases in readme This squashed commit is approved for public release by The Aerospace Corporation on 2020-08-19. Commits made by the Communication Software Implementation Department. This commit is an addendum to the prior PR#106 approved 2019-05-07. New Features * Ordered sigmf-meta so that the keys are in a more useful order upon manual inspection. * Use cases in `README.md`. Fixes * Improved version handling. --- README.md | 71 +++++++++++++++++++++++++++----------- setup.py | 8 ++--- sigmf/__init__.py | 6 +--- sigmf/sigmffile.py | 76 +++++++++++++++++++++++++++++------------ sigmf/version.py | 4 --- tests/test_sigmffile.py | 8 +++++ 6 files changed, 120 insertions(+), 53 deletions(-) delete mode 100644 sigmf/version.py diff --git a/README.md b/README.md index 478148e..5479be3 100644 --- a/README.md +++ b/README.md @@ -39,28 +39,22 @@ maintained for posterity. Anyone is welcome to get involved - indeed, the more people involved in the discussions, the more useful the standard is likely to be. -## Installation -After cloning, simply run the setup script for a static installation. +## Getting Started +This module can be installed the typical way: +```bash +pip install . ``` -python setup.py -``` - -Alternatively, install the module in developer mode if you plan to experiment -with your own changes. -``` -python setup.py develop -``` +## Use Cases +### Load a SigMF dataset; read its annotation, metadata, and samples -## Usage example -#### Load a SigMF dataset; read its annotation, metadata, and samples ```python from sigmf import SigMFFile, sigmffile # Load a dataset -sigmf_filename = 'datasets/my_dataset.sigmf-meta' # extension is optional -signal = sigmffile.fromfile(sigmf_filename) +filename = 'example.sigmf-meta' # extension is optional +signal = sigmffile.fromfile(filename) # Get some metadata and all annotations sample_rate = signal.get_global_field(SigMFFile.SAMPLE_RATE_KEY) @@ -69,11 +63,10 @@ signal_duration = sample_count / sample_rate annotations = signal.get_annotations() # Iterate over annotations -for annotation_idx, annotation in enumerate(annotations): +for adx, annotation in enumerate(annotations): annotation_start_idx = annotation[SigMFFile.START_INDEX_KEY] annotation_length = annotation[SigMFFile.LENGTH_INDEX_KEY] - annotation_comment = annotation.get(SigMFFile.COMMENT_KEY, - "[annotation {}]".format(annotation_idx)) + annotation_comment = annotation.get(SigMFFile.COMMENT_KEY, "[annotation {}]".format(adx)) # Get capture info associated with the start of annotation capture = signal.get_capture_info(annotation_start_idx) @@ -82,14 +75,54 @@ for annotation_idx, annotation in enumerate(annotations): freq_max = freq_center + 0.5*sample_rate # Get frequency edges of annotation (default to edges of capture) - freq_start = annotation.get(SigMFFile.FLO_KEY, f_min) - freq_stop = annotation.get(SigMFFile.FHI_KEY, f_max) + freq_start = annotation.get(SigMFFile.FLO_KEY) + freq_stop = annotation.get(SigMFFile.FHI_KEY) # Get the samples corresponding to annotation samples = signal.read_samples(annotation_start_idx, annotation_length) ``` +### Write a SigMF file from a numpy array +```python +import datetime as dt +from sigmf import SigMFFile + +# suppose we have an complex timeseries signal +data = np.zeros(1024, dtype=np.complex64) + +# write those samples to file in cf32_le +data.tofile('example.sigmf-data') + +# create the metadata +meta = SigMFFile( + data_file='example.sigmf-data', # extension is optional + global_info = { + SigMFFile.DATATYPE_KEY: 'cf32_le', + SigMFFile.SAMPLE_RATE_KEY: 48000, + SigMFFile.AUTHOR_KEY: 'jane.doe@domain.org', + SigMFFile.DESCRIPTION_KEY: 'All zero example file.', + SigMFFile.VERSION_KEY: sigmf.__version__, + } +) + +# create a capture key at time index 0 +meta.add_capture(0, metadata={ + SigMFFile.FREQUENCY_KEY: 915000000, + SigMFFile.DATETIME_KEY: dt.datetime.utcnow().isoformat()+'Z', +}) + +# add an annotation at sample 100 with length 200 & 10 KHz width +meta.add_annotation(100, 200, metadata = { + SigMFFile.FLO_KEY: 914995000.0, + SigMFFile.FHI_KEY: 915005000.0, + SigMFFile.COMMENT_KEY: 'example annotation', +}) + +# check for mistakes & write to disk +assert meta.validate() +meta.tofile('example.sigmf-meta') # extension is optional +``` ## Frequently Asked Questions diff --git a/setup.py b/setup.py index 7b84f96..b70352d 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ from setuptools import setup import os +import re shortdesc = "Signal Metadata Format Specification" longdesc = """ @@ -10,13 +11,12 @@ samples, and features of the signal itself. """ -# exec version.py to get __version__ (version.py is the single source of the version) -version_file = os.path.join(os.path.dirname(__file__), 'sigmf', 'version.py') -exec(open(version_file).read()) +with open(os.path.join('sigmf', '__init__.py')) as handle: + version = re.search(r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', handle.read()).group(1) setup( name='SigMF', - version=__version__, + version=version, description=shortdesc, long_description=longdesc, url='https://github.com/gnuradio/SigMF', diff --git a/sigmf/__init__.py b/sigmf/__init__.py index c31a9ba..d10d089 100644 --- a/sigmf/__init__.py +++ b/sigmf/__init__.py @@ -18,12 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -# Use version.py to get the version -# Never define in the __init__.py and import it in setup.py because you can't -# import sigmf in setup.py because you won't have the dependencies yet. -# https://packaging.python.org/guides/single-sourcing-package-version/ +__version__ = '0.0.2' -from .version import __version__ from .archive import SigMFArchive from .sigmffile import SigMFFile diff --git a/sigmf/sigmffile.py b/sigmf/sigmffile.py index e40b1ac..6166b75 100644 --- a/sigmf/sigmffile.py +++ b/sigmf/sigmffile.py @@ -17,7 +17,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. - +from collections import OrderedDict import codecs import json import tarfile @@ -75,6 +75,7 @@ def __init__( metadata=None, data_file=None, global_info=None, + skip_checksum=False, ): self.version = None self.schema = None @@ -89,7 +90,7 @@ def __init__( if global_info is not None: self.set_global_info(global_info) self.data_file = data_file - if self.data_file: + if self.data_file and not skip_checksum: self.calculate_hash() self._count_samples() @@ -293,32 +294,65 @@ def validate(self): self._metadata, schema.get_schema(schema_version), ) - - def dump(self, filep, pretty=False): - """ + def ordered_metadata(self): + ''' + Get a nicer representation of _metadata. Will sort keys, but put the + top-level fields 'global', 'captures', 'annotations' in front. + + Returns + ------- + ordered_meta : OrderedDict + Cleaner representation of _metadata with top-level keys correctly + ordered and the rest of the keys sorted. + ''' + ordered_meta = OrderedDict() + top_sort_order = ['global', 'captures', 'annotations'] + for top_key in top_sort_order: + assert top_key in self._metadata + ordered_meta[top_key] = json.loads(json.dumps(self._metadata[top_key], sort_keys=True)) + # If there are other top-level keys, they go later + # TODO: sort these `other` top-level keys + for oth_key, oth_val in self._metadata.items(): + if oth_key not in top_sort_order: + ordered_meta[oth_key] = json.loads(json.dumps(oth_val, sort_keys=True)) + return ordered_meta + + def dump(self, filep, pretty=True): + ''' Write metadata to a file. - Parameters: - filep -- File pointer or something that json.dump() can handle - pretty -- If true, output will be formatted extra nicely. - """ + Parameters + ---------- + filep : object + File pointer or something that json.dump() can handle + pretty : bool, optional + Is true by default. + ''' json.dump( - self._metadata, + self.ordered_metadata(), filep, - sort_keys=True if pretty else False, indent=4 if pretty else None, separators=(',', ': ') if pretty else None, ) - def dumps(self, pretty=False): - """ - Return a string representation of the metadata file. - - Parameters: - pretty -- If true, output will be formatted extra nicely. - """ + def dumps(self, pretty=True): + ''' + Get a string representation of the metadata. + + Parameters + ---------- + filep : object + File pointer or something that json.dump() can handle + pretty : bool, optional + Is true by default. + + Returns + ------- + string + String representation of the metadata using json formatter. + ''' return json.dumps( - self._metadata, + self.ordered_metadata(), indent=4 if pretty else None, separators=(',', ': ') if pretty else None, ) @@ -486,7 +520,7 @@ def fromarchive(archive_path, dir=None): return SigMFFile(metadata=metadata, data_file=data_file) -def fromfile(filename): +def fromfile(filename, skip_checksum=False): """ Creates and returns a returns a SigMFFile instance with metadata loaded from the specified file. The filename may be that of either a sigmf-meta file, a sigmf-data file, or a sigmf archive. @@ -509,7 +543,7 @@ def fromfile(filename): mdfile_reader = bytestream_reader(meta_fp) metadata = json.load(mdfile_reader) meta_fp.close() - return SigMFFile(metadata=metadata, data_file=data_fn) + return SigMFFile(metadata=metadata, data_file=data_fn, skip_checksum=skip_checksum) def get_sigmf_filenames(filename): """ diff --git a/sigmf/version.py b/sigmf/version.py deleted file mode 100644 index ffcdca5..0000000 --- a/sigmf/version.py +++ /dev/null @@ -1,4 +0,0 @@ -''' -This is the only place SigMF version is defined. -''' -__version__ = '0.0.2' diff --git a/tests/test_sigmffile.py b/tests/test_sigmffile.py index 58a6bf8..142af94 100644 --- a/tests/test_sigmffile.py +++ b/tests/test_sigmffile.py @@ -87,3 +87,11 @@ def test_add_multiple_captures_and_annotations(): sigf = SigMFFile() for idx in range(3): simulate_capture(sigf, idx, 1024) + +def test_ordered_metadata(): + '''check to make sure the metadata is sorted as expected''' + sigf = SigMFFile() + top_sort_order = ['global', 'captures', 'annotations'] + for kdx, key in enumerate(sigf.ordered_metadata()): + assert kdx == top_sort_order.index(key) +