diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 83d685b..0000000 --- a/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[report] -omit = - */instruments/templates/ diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..a7f6ad6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,29 @@ +--- +name: Bug report +about: Create a report to report a problem that needs to be fixed +labels: bug +title: "BUG: " + +--- + +# Description +A clear and concise description of what the bug is, including a description +of what you expected the outcome to be. + +# To Reproduce this bug: +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +Consider including images or test files to help others reproduce the bug and +solve the problem. + +## Test configuration + - OS: [e.g. Hal] + - Version [e.g. Python 3.47] + - Other details about your setup that could be relevant + +# Additional context +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d02da2e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,27 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "ENH: " +labels: enhancement + +--- + +# Description +A clear and concise description of the new feature or behaviour you would like. + +## Potential impact + +- Is the feature related to an existing problem? +- How critical is this feature to your workflow? +- How wide of an impact to you anticipate this enhancement having? +- Would this break any existing functionality? + +## Potential solution(s) +A clear and concise description of what you want to happen. + +# Alternatives +A clear description of any alternative solutions or features you've considered. + +# Additional context +Add any other context or screenshots about the feature request here, potentially +including your operational configuration. diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..2073086 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,19 @@ +--- +name: Question +about: A question about this project +title: "QUEST: " +labels: question + +--- + +# Description +A clear and concise summary of your query + +## Example code (optional) +If relevant, include sample code, images, or files so that others can understand +the full context of your question. + +## Configuration + - OS: [e.g. Hal] + - Version [e.g. Python 3.47] + - Other details about your setup that could be relevant diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..5fa8548 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,53 @@ +# Description + +Addresses # (issue) + +Please include a summary of the change and which issue is fixed. Please also +include relevant motivation and context. List any dependencies that are required +for this change. Please see ``CONTRIBUTING.md`` for more guidelines. + +# Type of change + +Please delete options that are not relevant. + +- Bug fix (non-breaking change which fixes an issue) +- New feature (non-breaking change which adds functionality or documentation) +- Breaking change (fix or feature that would cause existing functionality + to not work as expected) +- This change requires a documentation update + +# How Has This Been Tested? + +Please describe the tests that you ran to verify your changes. Provide +instructions so we can reproduce the problem and the solution. Including images +or test files is frequently very useful. Please also list any relevant details +for your test configuration. + +- Test A + +``` +Test B +``` + +## Test Configuration +* Operating system: Hal +* Version number: Python 3.X +* Any details about your local setup that are relevant + +# Checklist: + +- [ ] Make sure you are merging into the ``develop`` (not ``main``) branch +- [ ] My code follows the style guidelines of this project +- [ ] I have performed a self-review of my own code +- [ ] I have linted the files updated in this pull request +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes +- [ ] Any dependent changes have been merged and published in downstream modules +- [ ] Add a note to ``CHANGELOG.md``, summarizing the changes + +If this is a release PR, replace the first item of the above checklist with the +release checklist on the pysat wiki: +https://github.com/pysat/pysat/wiki/Checklist-for-Release diff --git a/.gitignore b/.gitignore index 6db464d..507fb9d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ var/ *.egg-info/ .installed.cfg *.egg +.build # PyInstaller # Usually these files are written by a python script from a template diff --git a/.travis.yml b/.travis.yml index aaebe06..f40fda5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,16 @@ language: python dist: xenial -matrix: +jobs: include: - - python: 3.6 - - python: 3.7 - - python: 3.8 + - name: '3.7 with flake8' + python: '3.7' + script: + - flake8 . --count --select=E,F,W + - pytest --cov=pysatMadrigal/ + - python: '3.8' + script: pytest --cov=pysatMadrigal/ + - python: '3.9' + script: pytest --cov=pysatMadrigal/ services: xvfb cache: pip @@ -15,42 +21,16 @@ addons: - gfortran - libncurses5-dev -install: - - sudo apt-get update - # We do this conditionally because it saves us some downloading if the - # version is the same. - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - - bash miniconda.sh -b -p $HOME/miniconda - - source "$HOME/miniconda/etc/profile.d/conda.sh" - - hash -r - - conda config --set always_yes True --set changeps1 False - - conda update -q conda - # Useful for debugging any issues with conda - - conda info -a - # Create conda test environment - - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy pandas xarray requests beautifulsoup4 lxml netCDF4 h5py pytest-cov pytest-ordering coveralls future - - conda activate test-environment - # Dependencies not available through conda, install through pip - - pip install pytest-flake8 - - pip install madrigalWeb - - pip install PyForecastTools - - pip install pysatCDF >/dev/null - # Custom pysat install - - cd .. - - git clone https://github.com/pysat/pysat.git - - cd pysat - - git checkout develop-3 - - python setup.py install - - export PYTHONPATH=$PYTHONPATH:$(pwd) +before_install: + - python -m pip install --upgrade pip + # Get the program and testing requirements + - pip install -r test_requirements.txt + - pip install -r requirements.txt # set up data directory - mkdir /home/travis/build/pysatData - - cd ../pysatMadrigal - # install pysatMadrigal + - python -c "import pysat; pysat.params['data_dirs'] = '/home/travis/build/pysatData'" +install: - python setup.py install -# command to run tests -script: - - pytest -vs --cov=pysatMadrigal/ --flake8 - after_success: - - coveralls + - coveralls --rcfile=setup.cfg diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 0000000..b70bcc6 --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,37 @@ +{ + "keywords": [ + "satellite", + "radar", + "Madrigal", + "DMSP", + "pysat", + "JRO", + "Jicamarca Radio Observatory", + "TEC", + "Total Electron Content", + "Ionosphere", + "Space Physics", + "Heliophysics" + ], + "creators": [ + { + "affiliation": "U.S. Naval Research Laboratory", + "name": "Burrell, Angeline G.", + "orcid": "0000-0001-8875-9326" + }, + { + "affiliation": "Goddard Space Flight Center", + "name": "Klenzing, Jeff", + "orcid": "0000-0001-8321-6074" + }, + { + "affiliation": "The University of Texas at Dallas", + "name": "Stoneback, Russell", + "orcid": "0000-0001-7216-4336" + }, + { + "affiliation": "Predictive Science", + "name": "Pembroke, Asher" + } + ] +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1e5d0df --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,36 @@ +# Change Log +All notable changes to this project will be documented in this file. +This project adheres to [Semantic Versioning](https://semver.org/). + +## [0.0.4] - 2021-06-11 +- Made changes to structure to comply with updates in pysat 3.0.0 +- Deprecations + - Restructed Instrument methods, moving `madrigal` to `general` and extracting + local methods from the instrument modules to platform-specific method files + - Cycled testing support to cover Python 3.7-3.9 +- Enhancements + - Added coords from pysat.utils + - Added Vertical TEC Instrument + - Added documentation + - Added load routine for simple formatted data + - Expanded feedback during data downloads + - Updated documentation configuration to improve maintainability + - Updated documentation style, displaying logo on sidebar in html format + - Changed zenodo author name format for better BibTeX compliance + - Updated CONTRIBUTING and README information +- Bug Fix + - Updated Madrigal methods to simplify compound data types and enable + creation of netCDF4 files using `Instrument.to_netcdf4()` + - Updated load for multiple files in pandas format + - Fixed remote listing routine to return filenames instead of experiments + - Fixed bug introduced by change in xarray requiring engine kwarg + - Fixed bug that would not list multiple types of files + +## [0.0.3] - 2020-06-15 +- pypi compatibility + +## [0.0.2] - 2020-05-13 +- zenodo link + +## [0.0.1] - 2020-05-13 +- Alpha release diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..483ea9a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,75 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, +body size, disability, ethnicity, gender identity and expression, level of +experience, nationality, personal appearance, race, religion, or sexual +identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an +appointed representative at an online or offline event. Representation of a +project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at pysat.developers@gmail.com. The +project team will review and investigate all complaints, and will respond in a +way that it deems appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an +incident. Further details of specific enforcement policies may be posted +separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 1.4, available at [https://contributor-covenant.org/version/1/4][version] + +[homepage]: https://contributor-covenant.org +[version]: https://contributor-covenant.org/version/1/4/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a81865e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,144 @@ +Contributing +============ + +Bug reports, feature suggestions and other contributions are greatly +appreciated! pysat and pysatMadrigal are community-driven projects that +welcome both feedback and contributions. + +Short version +------------- + +* Submit bug reports, feature requests, and questions at +`GitHub Issues `_ +* Make pull requests to the ``develop`` branch + +More about Issues +----------------- + +Bug reports, questions, and feature requests should all be made as GitHub +Issues. Templates are provided for each type of issue, to help you include +all the necessary information. + +Questions +^^^^^^^^^ + +Not sure how something works? Ask away! The more information you provide, the +easier the question will be to answer. You can also interact with the pysat +developers on our `slack channel `_. + +Bug reports +^^^^^^^^^^^ + +When reporting a bug please include: + +* Your operating system name and version +* Any details about your local setup that might be helpful in troubleshooting +* Detailed steps to reproduce the bug + +Feature requests +^^^^^^^^^^^^^^^^ + +If you are proposing a new feature or a change in something that already exists: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that code contributions + are welcome :) + +More about Development +---------------------- + +To set up `pysatMadrigal` for local development: + +1. Fork pysatMadrigal on + `GitHub `_. +2. Clone your fork locally:: + + git clone git@github.com:your_name_here/pysatMadrigal.git + +3. Create a branch for local development:: + + git checkout -b name-of-your-bugfix-or-feature + +4. Make your changes locally. Tests for new instruments are performed + automatically. Tests for custom functions should be added to the + appropriately named file in ``pysatMadrigal/tests``. For example, + Jicamarca methods containined in ``pysatMadrigal/instruments/methods/jro.py`` + should be named ``pysatMadrigal/tests/test_methods_jro.py``. If no test + file exists, then you should create one. This testing uses pytest, which + will run tests on any python file in the test directory that starts with + ``test``. Test classes must begin with ``Test``, and test methods must also + begin with ``test``. + +5. When you're done making changes, run all the checks to ensure that nothing + is broken on your local system:: + + pytest -vs pysatMadrigal + +6. Update/add documentation (in ``docs``). Even if you don't think it's + relevant, check to see if any existing examples have changed. + +7. Add your name to the .zenodo.json file as an author + +8. Commit your changes and push your branch to GitHub:: + + git add . + git commit -m "Brief description of your changes" + git push origin name-of-your-bugfix-or-feature + +9. Submit a pull request through the GitHub website. Pull requests should be + made to the ``develop`` branch. + +Pull Request Guidelines +^^^^^^^^^^^^^^^^^^^^^^^ + +If you need some code review or feedback while you're developing the code, just +make a pull request. Pull requests should be made to the ``develop`` branch. + +For merging, you should: + +1. Include an example for use +2. Add a note to ``CHANGELOG.md`` about the changes +3. Ensure that all checks passed (current checks include Travis-CI + and Coveralls) [1]_ + +.. [1] If you don't have all the necessary Python versions available locally or + have trouble building all the testing environments, you can rely on + Travis to run the tests for each change you add in the pull request. + Because testing here will delay tests by other developers, please ensure + that the code passes all tests on your local system first. + +Project Style Guidelines +^^^^^^^^^^^^^^^^^^^^^^^^ + +In general, pysat follows PEP8 and numpydoc guidelines. Pytest runs the unit +and integration tests, flake8 checks for style, and sphinx-build performs +documentation tests. However, there are certain additional style elements that +have been settled on to ensure the project maintains a consistent coding style. +These include: + +* Line breaks should occur before a binary operator (ignoring flake8 W503) +* Combine long strings using `join` +* Preferably break long lines on open parentheses rather than using `\` +* Use no more than 80 characters per line +* Avoid using Instrument class key attribute names as unrelated variable names: + `platform`, `name`, `tag`, and `inst_id` +* The pysat logger is imported into each sub-module and provides status updates + at the info and warning levels (as appropriate) +* Several dependent packages have common nicknames, including: + * `import datetime as dt` + * `import numpy as np` + * `import pandas as pds` + * `import xarray as xr` +* All classes should have `__repr__` and `__str__` functions +* Docstrings use `Note` instead of `Notes` +* Try to avoid creating a try/except statement where except passes +* Use setup and teardown in test classes +* Use pytest parametrize in test classes when appropriate +* Provide testing class methods with informative failure statements and + descriptive, one-line docstrings +* Block and inline comments should use proper English grammar and punctuation + with the exception of single sentences in a block, which may then omit the + final period +* When casting is necessary, use `np.int64` and `np.float64` to ensure operating + system agnosticism diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..d24537a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,13 @@ +include *.py +include *.f +include *.c +recursive-include pysatMadrigal *.py +include *.md +include *.txt +include LICENSE +include pysatMadrigal/version.txt +prune pysatMadrigal/tests +prune docs +prune demo +exclude *.pdf +exclude *.png diff --git a/README.md b/README.md index e19d85c..8f244e6 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,43 @@
- pysat + pysatMadrigal
# pysatMadrigal - - - +[![Documentation Status](https://readthedocs.org/projects/pysatMadrigal/badge/?version=latest)](http://pysatMadrigal.readthedocs.io/en/latest/?badge=latest) [![Build Status](https://travis-ci.org/pysat/pysatMadrigal.svg?branch=main)](https://travis-ci.com/pysat/pysatMadrigal) [![Coverage Status](https://coveralls.io/repos/github/pysat/pysatMadrigal/badge.svg?branch=main)](https://coveralls.io/github/pysat/pysatMadrigal?branch=main) [![DOI](https://zenodo.org/badge/258384773.svg)](https://zenodo.org/badge/latestdoi/258384773) +[![PyPI version](https://badge.fury.io/py/pysatMadrigal.svg)](https://badge.fury.io/py/pysatMadrigal) +pysatMadrigal allows users to import data from the Madrigal database into +pysat ([pysat documentation](http://pysat.readthedocs.io/en/latest/)). - -pysatMadrigal allows users to import data from the Madrigal database into pysat. It currently supports the Ion Velocity Meter on the Defense Meteorological Satellite (`dmsp_ivm`) and the Jicamarca Radio Observatory Incoherent Scatter Radar (`jro_isr`). It also includes templates and an interface for interacting with madrigalWeb. +# Installation +The following instructions provide a guide for installing pysatMadrigal and +give some examples on how to use the routines. -Documentation ---------------------- -[Full Documentation for main package](http://pysat.readthedocs.io/en/latest/) +## Prerequisites +pysatMadrigal uses common Python modules, as well as modules developed by and +for the Space Physics community. This module officially supports Python 3.7+. -# Installation +| Common modules | Community modules | +| -------------- | ----------------- | +| h5py | madrigalWeb | +| numpy | pysat >= 3.0.0 | +| pandas | | +| xarray | | -Currently, the main way to get pysatMadrigal is through github. +## PyPi Installation +``` +pip install pysatMadrigal +``` + +## GitHub Installation ``` git clone https://github.com/pysat/pysatMadrigal.git ``` @@ -38,24 +50,27 @@ cd pysatMadrigal/ python setup.py install ``` -Note: pre-1.0.0 version ------------------- -pysatMadrigal is currently in an initial development phase. Much of the API is being built off of the upcoming pysat 3.0.0 software in order to streamline the usage and test coverage. This version of pysat is planned for release later this year. Currently, you can access the develop version of this through github: -``` -git clone https://github.com/pysat/pysat.git -cd pysat -git checkout develop-3 -python setup.py install -``` -It should be noted that this is a working branch and is subject to change. +Note: pre-0.1.0 version +----------------------- +pysatMadrigal is currently provided as an alpha pre-release. Feedback and +contributions are appreciated. -# Using with pysat +# Examples -The instrument modules are portable and designed to be run like any pysat instrument. +The instrument modules are portable and designed to be run like any pysat +instrument. ``` import pysat from pysatMadrigal.instruments import dmsp_ivm +ivm = pysat.Instrument(inst_module=dmsp_ivm, tag='utd', inst_id='f15') +``` + +Another way to use the instruments in an external repository is to register the +instruments. This only needs to be done the first time you load an instrument. +Afterward, pysat will identify them using the `platform` and `name` keywords. -ivm = pysat.Instrument(inst_module=dmsp_ivm) +``` +pysat.utils.registry.register('pysatMadrigal.instruments.dmsp_ivm') +dst = pysat.Instrument('dmsp', 'ivm', tag='utd', inst_id='f15') ``` diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..3b0bc03 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = pysatMadrigal +SOURCEDIR = . +BUILDDIR = .build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/citing.rst b/docs/citing.rst new file mode 100644 index 0000000..3e97310 --- /dev/null +++ b/docs/citing.rst @@ -0,0 +1,40 @@ +Citation Guidelines +=================== + +When publishing work that uses pysatMadrigal, please cite the package and +any package it depends on that plays an important role in your analysis. +Specifying which version of pysatMadrigal used will also improve the +reproducibility of your presented results. + +pysatMadrigal +------------- + +* Burrell, A. G., et al. (2020). + pysat/pysatMadrigal (Version 0.0.3). Zenodo. + doi:10.5281/zenodo.3824979. + +.. code-block:: latex + + @Misc{pysatMadrigal, + author = {Burrell, A. G. and Klenzing, J. H. and Stoneback, R. + and Pembroke, A.}, + title = {pysat/pysatMadrigal}, + year = {2020}, + date = {2020-05-13}, + url = {https://github.com/pysat/pysatMadrigal}, + doi = {10.5281/zenodo.3824979}, + publisher = {Zenodo}, + version = {v0.0.3}, + } + +MadrigalWeb +----------- + +pysatMadrigal uses `MadrigalWeb `_ to +access the Madrigal database. This package is described in the following +journal article. + +* Burrell A. G., et al. (2018). Snakes on a Spaceship - An Overview of Python in + Heliophysics. Journal of Geophysical Research: Space Physics, 123, + doi:10.1029/2018ja025877. + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..9c581aa --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- +# +# pysatMadrigal documentation build configuration file, created by +# sphinx-quickstart on Wed Jul 5 16:25:26 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import json +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'numpydoc', + 'm2r2'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['.templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The main toctree document (using required variable name). +master_doc = 'index' + +# General information about the project. +project = 'pysatMadrigal' +title = '{:s} Documentation'.format(project) +zenodo = json.loads(open('../.zenodo.json').read()) +author = ', '.join([creator['name'] for creator in zenodo['creators']]) +description = ''.join(['Tools for accessing and analyzing data from the ', + 'Madrigal database']) +copyright = ', '.join(['2021', author]) + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +doc_dir = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(doc_dir, "..", project, "version.txt"), "r") as fin: + version = fin.read().strip() +release = '{:s}-alpha'.format(version) # Include alpha/beta/rc tags. + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['.build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' +html_theme_path = ["_themes", ] + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +html_logo = os.path.join(os.path.abspath('.'), 'figures', 'pysatMadrigal.png') + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = {'logo_only': True} + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = '{:s}doc'.format(project) + +# -- Options for LaTeX output --------------------------------------------- +# +# The paper size ('letterpaper' or 'a4paper'). +# 'papersize': 'letterpaper', +# +# The font size ('10pt', '11pt' or '12pt'). +# 'pointsize': '10pt', +# +# Additional stuff for the LaTeX preamble. +# 'preamble': '', +# +# Latex figure (float) alignment +# 'figure_align': 'htbp', +latex_elements = {} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [(master_doc, '{:s}.tex'.format(project), title, author, + 'manual')] + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, project, title, [author], 1)] + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [(master_doc, project, title, author, project, + description, 'Space Physics')] + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/docs/develop_guide.rst b/docs/develop_guide.rst new file mode 100644 index 0000000..b5ff78f --- /dev/null +++ b/docs/develop_guide.rst @@ -0,0 +1,6 @@ +Guide for Developers +==================== + +.. toctree:: + develop_guide/code_of_conduct.rst + develop_guide/contributing.rst diff --git a/docs/develop_guide/code_of_conduct.rst b/docs/develop_guide/code_of_conduct.rst new file mode 100644 index 0000000..f4ab40b --- /dev/null +++ b/docs/develop_guide/code_of_conduct.rst @@ -0,0 +1 @@ +.. mdinclude:: ../../CODE_OF_CONDUCT.md diff --git a/docs/develop_guide/contributing.rst b/docs/develop_guide/contributing.rst new file mode 100644 index 0000000..f2c987c --- /dev/null +++ b/docs/develop_guide/contributing.rst @@ -0,0 +1 @@ +.. mdinclude:: ../../CONTRIBUTING.md diff --git a/docs/examples.rst b/docs/examples.rst new file mode 100644 index 0000000..a6e1174 --- /dev/null +++ b/docs/examples.rst @@ -0,0 +1,8 @@ +Examples +======== + +Here are some examples that demonstrate how to use various pysatMadrigal +tools + +.. toctree:: + examples/ex_init.rst diff --git a/docs/examples/ex_init.rst b/docs/examples/ex_init.rst new file mode 100644 index 0000000..e97d283 --- /dev/null +++ b/docs/examples/ex_init.rst @@ -0,0 +1,71 @@ +Loading DMSP IVM +================ + +pysatMadrigal uses `pysat `_ to download, load, +and provide an analysis framework for data sets archived at the Madrigal +database. As specified in the +`pysat tutorial `_, +data may be loaded using the following commands. Defense Meteorological +Satellite Program (DMSP) Ion Velocity Meter (IVM) data is used as an example. + +:: + + + import datetime as dt + import pysat + import pysatMadrigal as py_mad + + stime = dt.datetime(2012, 5, 14) + ivm = pysat.Instrument(inst_module=py_mad.instruments.dmsp_ivm, + tag='utd', inst_id='f15', update_files=True) + ivm.download(start=stime, user="Name+Surname", password="email@org.inst") + ivm.load(date=stime) + print(ivm) + + +The output includes a day of data with UTDallas quality flags from the F15 +spacecraft (as implied by the `tag` and `inst_id`), for the specified date. +At the time of publication this produces the output shown below. + +:: + + pysat Instrument object + ----------------------- + Platform: 'dmsp' + Name: 'ivm' + Tag: 'utd' + Instrument id: 'f15' + + Data Processing + --------------- + Cleaning Level: 'clean' + Data Padding: None + Keyword Arguments Passed to load: {'xarray_coords': [], 'file_type': 'hdf5'} + Keyword Arguments Passed to list_remote_files: {'user': None, 'password': None, 'url': 'http://cedar.openmadrigal.org', 'two_digit_year_break': None} + Custom Functions: 0 applied + + Local File Statistics + --------------------- + Number of files: 1 + Date Range: 31 December 2014 --- 1 January 2015 + + Loaded Data Statistics + ---------------------- + Date: 31 December 2014 + DOY: 365 + Time range: 31 December 2014 00:00:04 --- 31 December 2014 23:18:20 + Number of Times: 4811 + Number of variables: 30 + + Variable Names: + year month day + ... + rms_x sigma_vy sigma_vz + + pysat Meta object + ----------------- + Tracking 7 metadata values + Metadata for 30 standard variables + Metadata for 0 ND variables + + diff --git a/docs/figures/gnss_tec_vtec_example.png b/docs/figures/gnss_tec_vtec_example.png new file mode 100644 index 0000000..8edb00f Binary files /dev/null and b/docs/figures/gnss_tec_vtec_example.png differ diff --git a/poweredbypysat.png b/docs/figures/poweredbypysat.png similarity index 100% rename from poweredbypysat.png rename to docs/figures/poweredbypysat.png diff --git a/docs/figures/pysatMadrigal.png b/docs/figures/pysatMadrigal.png new file mode 100644 index 0000000..e0df670 Binary files /dev/null and b/docs/figures/pysatMadrigal.png differ diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..7f216f6 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1,2 @@ + +.. mdinclude:: ../CHANGELOG.md diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..4787575 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,29 @@ +.. pysatMadrigal documentation main file. Remember that it should at least + contain the root `toctree` directive. + +Welcome to the pysatMadrigal documentation +========================================== + +This documentation describes the pysatMadrigal module, which contains routines +to download, load, and support analysis for data sets available at the Madrigal +data base as pysat.Instrument objects. + +.. toctree:: + :maxdepth: -1 + + overview.rst + installation.rst + citing.rst + supported_instruments.rst + methods.rst + examples.rst + develop_guide.rst + history.rst + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..4ecc22a --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,53 @@ +Installation +============ + +The following instructions will allow you to install pysatMadrigal. + +Prerequisites +------------- + +.. image:: figures/poweredbypysat.png + :width: 150px + :align: right + :alt: powered by pysat Logo, blue planet with orbiting python + + +pysatMadrigal uses common Python modules, as well as modules developed by +and for the Space Physics community. This module officially supports +Python 3.6+. + + ============== ================= + Common modules Community modules + ============== ================= + h5py madrigalWeb + numpy pysat + pandas + xarray + ============== ================= + + +Installation Options +-------------------- + +You may either install pysatMadrigal via pip or by cloning the git repository + +1. Install from pip +:: + + pip install pysatMadrigal + + +2. Clone the git repository and use the ``setup.py`` file to install +:: + + + git clone https://github.com/pysat/pysatMadrigal.git + + # Install on the system (root privileges required) + sudo python3 setup.py install + + # Install at the user level + python3 setup.py install --user + + # Install at the user level with the intent to develop locally + python3 setup.py develop --user diff --git a/docs/methods.rst b/docs/methods.rst new file mode 100644 index 0000000..df0529c --- /dev/null +++ b/docs/methods.rst @@ -0,0 +1,44 @@ +Methods +======= + +Several methods exist to help combine multiple data sets and convert between +equivalent indices. + + +DMSP +---- + +Supports the Defense Meteorological Satellite Program instruments by providing +common custom routines alongside reference and acknowledgement information. + + +.. automodule:: pysatMadrigal.instruments.methods.dmsp + :members: + +GNSS +---- + +Supports the Global Navigation Satellite System instruments by providing +reference and acknowledgement information. + + +.. automodule:: pysatMadrigal.instruments.methods.gnss + :members: + +JRO +--- + +Supports the Jicamarca Radio Observatory instrumnets by providing common custom +routines alongside reference and acknowledgement information. + + +.. automodule:: pysatMadrigal.instruments.methods.jro + :members: + +General +------- +Supports the Madrigal data access. + + +.. automodule:: pysatMadrigal.instruments.methods.general + :members: diff --git a/docs/overview.rst b/docs/overview.rst new file mode 100644 index 0000000..97fb1f9 --- /dev/null +++ b/docs/overview.rst @@ -0,0 +1,11 @@ +Overview +======== + +The `CEDAR Madrigal database `_ houses several +ground- and space-based data sets for instruments that provide upper +atmospheric, scientific observations. + +.. image:: figures/pysatMadrigal.png + :width: 400px + :align: center + :alt: PysatMadrigal Logo, A globe centered on the western hemisphere with an orbiting python and the module name superimposed. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..fc12cc5 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +h5py +madrigalWeb +m2r2 +numpydoc +pandas +xarray +pysat diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst new file mode 100644 index 0000000..afe5f1c --- /dev/null +++ b/docs/supported_instruments.rst @@ -0,0 +1,33 @@ +Supported Instruments +===================== + +DMSP_IVM +-------- + +Supports the Defense Meteorological Satelitte Program (DMSP) Ion Velocity +Meter (IVM) Madrigal data. + + +.. automodule:: pysatMadrigal.instruments.dmsp_ivm + :members: + +GNSS_TEC +-------- + +The Global Navigation Satellite System (GNSS) Total Electron Content (TEC) +provides a measure of column plasma density over the globle. The Madrigal +TEC is provided by MIT Haystack. + +.. automodule:: pysatMadrigal.instruments.gnss_tec + :members: + +JRO_ISR +---------------- + +The incoherent scatter radar (ISR) at the +`Jicamarca Radio Observatory `_ regularly +measures the velocity, density, and other ionospheric characteristics near the +magnetic equator over Peru. + +.. automodule:: pysatMadrigal.instruments.jro_isr + :members: diff --git a/pysatMadrigal/__init__.py b/pysatMadrigal/__init__.py index 0b73a26..c1a628f 100644 --- a/pysatMadrigal/__init__.py +++ b/pysatMadrigal/__init__.py @@ -1 +1,2 @@ from pysatMadrigal import instruments # noqa F401 +from pysatMadrigal import utils # noqa F401 diff --git a/pysatMadrigal/instruments/__init__.py b/pysatMadrigal/instruments/__init__.py index 2eb5b01..0ad832e 100644 --- a/pysatMadrigal/instruments/__init__.py +++ b/pysatMadrigal/instruments/__init__.py @@ -1,4 +1,8 @@ -from pysatMadrigal.instruments import dmsp_ivm, jro_isr +# Import Madrigal instruments +from pysatMadrigal.instruments import dmsp_ivm, gnss_tec, jro_isr + +# Import Madrigal methods from pysatMadrigal.instruments import methods # noqa F401 -__all__ = ['dmsp_ivm', 'jro_isr'] +# Define variable name with all available instruments +__all__ = ['dmsp_ivm', 'gnss_tec', 'jro_isr'] diff --git a/pysatMadrigal/instruments/dmsp_ivm.py b/pysatMadrigal/instruments/dmsp_ivm.py index 2d7581e..5f24ac4 100644 --- a/pysatMadrigal/instruments/dmsp_ivm.py +++ b/pysatMadrigal/instruments/dmsp_ivm.py @@ -1,3 +1,8 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- # -*- coding: utf-8 -*- """Supports the Ion Velocity Meter (IVM) onboard the Defense Meteorological Satellite Program (DMSP). @@ -9,95 +14,100 @@ composition, plasma temperature, and plasma motion may be determined. The DM directly measures the arrival angle of plasma. Using the reported motion of the satellite the angle is converted into ion motion along -two orthogonal directions, perpendicular to the satellite track. +two orthogonal directions, perpendicular to the satellite track. The IVM is +part of the Special Sensor for Ions, Electrons, and Scintillations (SSIES) +instrument suite on DMSP. Downloads data from the National Science Foundation Madrigal Database. The routine is configured to utilize data files with instrument performance flags generated at the Center for Space Sciences at the University of Texas at Dallas. -Parameters +Properties ---------- -platform : string +platform 'dmsp' -name : string +name 'ivm' -tag : string - 'utd', None -sat_id : string +tag + 'utd', '' +inst_id ['f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18'] Example ------- +:: + import pysat dmsp = pysat.Instrument('dmsp', 'ivm', 'utd', 'f15', clean_level='clean') dmsp.download(dt.datetime(2017, 12, 30), dt.datetime(2017, 12, 31), user='Firstname+Lastname', password='email@address.com') - dmsp.load(2017,363) + dmsp.load(2017, 363) Note ---- - Please provide name and email when downloading data with this routine. +Please provide name and email when downloading data with this routine. Code development supported by NSF grant 1259508 """ -from __future__ import print_function -from __future__ import absolute_import - import datetime as dt import functools import numpy as np -import pandas as pds -from pysatMadrigal.instruments.methods import madrigal as mad_meth -from pysat.instruments.methods import general as mm_gen +from pysat import logger + +from pysatMadrigal.instruments.methods import general, dmsp -import logging -logger = logging.getLogger(__name__) +# ---------------------------------------------------------------------------- +# Instrument attributes platform = 'dmsp' name = 'ivm' tags = {'utd': 'UTDallas DMSP data processing', '': 'Level 2 data processing'} -sat_ids = {'f11': ['utd', ''], 'f12': ['utd', ''], 'f13': ['utd', ''], - 'f14': ['utd', ''], 'f15': ['utd', ''], 'f16': [''], 'f17': [''], - 'f18': ['']} -_test_dates = {'f11': {'utd': dt.datetime(1998, 1, 2)}, - 'f12': {'utd': dt.datetime(1998, 1, 2)}, - 'f13': {'utd': dt.datetime(1998, 1, 2)}, - 'f14': {'utd': dt.datetime(1998, 1, 2)}, - 'f15': {'utd': dt.datetime(2017, 12, 30)}} +inst_ids = {'f11': ['utd', ''], 'f12': ['utd', ''], 'f13': ['utd', ''], + 'f14': ['utd', ''], 'f15': ['utd', ''], 'f16': [''], 'f17': [''], + 'f18': ['']} + pandas_format = True -# support list files routine -# use the default CDAWeb method -dmsp_fname1 = {'utd': 'dms_ut_{year:4d}{month:02d}{day:02d}_', - '': 'dms_{year:4d}{month:02d}{day:02d}_'} -dmsp_fname2 = {'utd': '.{version:03d}.hdf5', '': 's?.{version:03d}.hdf5'} +# Local attributes +dmsp_fname1 = {'utd': 'dms_ut_{{year:4d}}{{month:02d}}{{day:02d}}_', + '': 'dms_{{year:4d}}{{month:02d}}{{day:02d}}_'} +dmsp_fname2 = {'utd': '.{{version:03d}}.{file_type}', + '': 's?.{{version:03d}}.{file_type}'} supported_tags = {ss: {kk: dmsp_fname1[kk] + ss[1:] + dmsp_fname2[kk] - for kk in sat_ids[ss]} for ss in sat_ids.keys()} -list_files = functools.partial(mm_gen.list_files, - supported_tags=supported_tags) + for kk in inst_ids[ss]} for ss in inst_ids.keys()} +remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5') + for kk in inst_ids[ss]} for ss in inst_ids.keys()} -# madrigal tags +# Madrigal tags madrigal_inst_code = 8100 -madrigal_tag = {'f11': {'utd': 10241, '': 10111}, - 'f12': {'utd': 10242, '': 10112}, - 'f13': {'utd': 10243, '': 10113}, - 'f14': {'utd': 10244, '': 10114}, - 'f15': {'utd': 10245, '': 10115}, - 'f16': {'': 10116}, - 'f17': {'': 10117}, - 'f18': {'': 10118}, } - -# support listing files currently available on remote server (Madrigal) -list_remote_files = functools.partial(mad_meth.list_remote_files, - supported_tags=supported_tags, - inst_code=madrigal_inst_code) - -# support load routine -load = mad_meth.load +madrigal_tag = {'f11': {'utd': '10241', '': '10111'}, + 'f12': {'utd': '10242', '': '10112'}, + 'f13': {'utd': '10243', '': '10113'}, + 'f14': {'utd': '10244', '': '10114'}, + 'f15': {'utd': '10245', '': '10115'}, + 'f16': {'': '10116'}, + 'f17': {'': '10117'}, + 'f18': {'': '10118'}, } + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = { + 'f11': {tag: dt.datetime(1998, 1, 2) for tag in inst_ids['f11']}, + 'f12': {tag: dt.datetime(1998, 1, 2) for tag in inst_ids['f12']}, + 'f13': {tag: dt.datetime(1998, 1, 2) for tag in inst_ids['f13']}, + 'f14': {tag: dt.datetime(1998, 1, 2) for tag in inst_ids['f14']}, + 'f15': {tag: dt.datetime(2017, 12, 30) for tag in inst_ids['f15']}, + 'f16': {tag: dt.datetime(2009, 1, 1) for tag in inst_ids['f16']}, + 'f17': {tag: dt.datetime(2009, 1, 1) for tag in inst_ids['f17']}, + 'f18': {tag: dt.datetime(2017, 12, 30) for tag in inst_ids['f18']}} + +# ---------------------------------------------------------------------------- +# Instrument methods def init(self): @@ -110,285 +120,111 @@ def init(self): self : pysat.Instrument This object - Returns - -------- - Void : (NoneType) - Object modified in place. - - """ - logger.info(mad_meth.cedar_rules()) + logger.info(general.cedar_rules()) + self.acknowledgements = general.cedar_rules() + self.references = dmsp.references(self.name) return -def download(date_array, tag='', sat_id='', data_path=None, user=None, - password=None): - """Downloads data from Madrigal. - - Parameters - ---------- - date_array : array-like - list of datetimes to download data for. The sequence of dates need not - be contiguous. - tag : string ('') - Tag identifier used for particular dataset. This input is provided by - pysat. - sat_id : string ('') - Satellite ID string identifier used for particular dataset. This input - is provided by pysat. - data_path : string (None) - Path to directory to download data to. - user : string (None) - User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - - Returns - -------- - Void : (NoneType) - Downloads data to disk. - - Notes - ----- - The user's names should be provided in field user. Ritu Karidhal should - be entered as Ritu+Karidhal - - The password field should be the user's email address. These parameters - are passed to Madrigal when downloading. - - The affiliation field is set to pysat to enable tracking of pysat - downloads. - - """ - mad_meth.download(date_array, inst_code=str(madrigal_inst_code), - kindat=str(madrigal_tag[sat_id][tag]), - data_path=data_path, user=user, password=password) - - -def default(inst): - pass - - -def clean(inst): +def clean(self): """Routine to return DMSP IVM data cleaned to the specified level - 'Clean' enforces that both RPA and DM flags are <= 1 - 'Dusty' <= 2 - 'Dirty' <= 3 - 'None' None + Note + ---- + Supports 'clean', 'dusty', 'dirty' - Routine is called by pysat, and not by the end user directly. + 'clean' enforces that both RPA and DM flags are <= 1 + 'dusty' <= 2 + 'dirty' <= 3 + 'none' Causes pysat to skip this routine - Parameters - ----------- - inst : (pysat.Instrument) - Instrument class object, whose attribute clean_level is used to return - the desired level of data selectivity. - - Returns - -------- - Void : (NoneType) - data in inst is modified in-place. - - Notes - -------- - Supports 'clean', 'dusty', 'dirty' + Routine is called by pysat, and not by the end user directly. """ - if inst.tag == 'utd': - if inst.clean_level == 'clean': - idx, = np.where((inst['rpa_flag_ut'] <= 1) - & (inst['idm_flag_ut'] <= 1)) - elif inst.clean_level == 'dusty': - idx, = np.where((inst['rpa_flag_ut'] <= 2) - & (inst['idm_flag_ut'] <= 2)) - elif inst.clean_level == 'dirty': - idx, = np.where((inst['rpa_flag_ut'] <= 3) - & (inst['idm_flag_ut'] <= 3)) + if self.tag == 'utd': + if self.clean_level == 'clean': + idx, = np.where((self['rpa_flag_ut'] <= 1) + & (self['idm_flag_ut'] <= 1)) + elif self.clean_level == 'dusty': + idx, = np.where((self['rpa_flag_ut'] <= 2) + & (self['idm_flag_ut'] <= 2)) + elif self.clean_level == 'dirty': + idx, = np.where((self['rpa_flag_ut'] <= 3) + & (self['idm_flag_ut'] <= 3)) else: - idx = slice(0, inst.index.shape[0]) + idx = slice(0, self.index.shape[0]) else: - if inst.clean_level in ['clean', 'dusty', 'dirty']: + if self.clean_level in ['clean', 'dusty', 'dirty']: logger.warning('this level 1 data has no quality flags') - idx = slice(0, inst.index.shape[0]) + idx = slice(0, self.index.shape[0]) - # downselect data based upon cleaning conditions above - inst.data = inst[idx] + # Downselect data based upon cleaning conditions above + self.data = self[idx] return -def smooth_ram_drifts(inst, rpa_flag_key=None, rpa_vel_key='ion_v_sat_for'): - """ Smooth the ram drifts using a rolling mean +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal and pysat methods - Parameters - ----------- - rpa_flag_key : string or NoneType - RPA flag key, if None will not select any data. The UTD RPA flag key - is 'rpa_flag_ut' (default=None) - rpa_vel_key : string - RPA velocity data key (default='ion_v_sat_for') - - Returns - --------- - RPA data in instrument object - - """ +# Support listing the local files +list_files = functools.partial(general.list_files, + supported_tags=supported_tags) - if rpa_flag_key in list(inst.data.keys()): - rpa_idx, = np.where(inst[rpa_flag_key] == 1) - else: - rpa_idx = list() +# Set the list_remote_files routine +list_remote_files = functools.partial(general.list_remote_files, + inst_code=madrigal_inst_code, + kindats=madrigal_tag, + supported_tags=remote_tags) - inst[rpa_idx, rpa_vel_key] = \ - inst[rpa_idx, rpa_vel_key].rolling(15, 5).mean() - return +# Set the load routine +load = general.load -def update_DMSP_ephemeris(inst, ephem=None): - """Updates DMSP instrument data with DMSP ephemeris +def download(date_array, tag='', inst_id='', data_path=None, user=None, + password=None, file_type='hdf5'): + """Downloads data from Madrigal. Parameters ---------- - ephem : pysat.Instrument or NoneType - dmsp_ivm_ephem instrument object - - Returns - --------- - Updates 'mlt' and 'mlat' - - """ - - # Ensure the right ephemera is loaded - if ephem is None: - logger.info('No ephemera provided for {:}'.format(inst.date)) - inst.data = pds.DataFrame(None) - return - - if ephem.sat_id != inst.sat_id: - raise ValueError('ephemera provided for the wrong satellite') - - if ephem.date != inst.date: - ephem.load(date=inst.date, verifyPad=True) - - if ephem.data.empty: - logger.info('unable to load ephemera for {:}'.format(inst.date)) - inst.data = pds.DataFrame(None) - return - - # Reindex the ephemeris data - ephem.data = ephem.data.reindex(index=inst.data.index, method='pad') - ephem.data = ephem.data.interpolate('time') - - # Update the DMSP instrument - inst['mlt'] = ephem['SC_AACGM_LTIME'] - inst['mlat'] = ephem['SC_AACGM_LAT'] - - return - - -def add_drift_unit_vectors(inst): - """ Add unit vectors for the satellite velocity - - Returns - --------- - Adds unit vectors in cartesian and polar coordinates for RAM and - cross-track directions - - 'unit_ram_x', 'unit_ram_y', 'unit_ram_r', 'unit_ram_theta' - - 'unit_cross_x', 'unit_cross_y', 'unit_cross_r', 'unit_cross_theta' - - Notes - --------- - Assumes that the RAM vector is pointed perfectly forward - - """ - # Calculate theta and R in radians from MLT and MLat, respectively - theta = inst['mlt'] * (np.pi / 12.0) - np.pi * 0.5 - r = np.radians(90.0 - inst['mlat'].abs()) - - # Determine the positions in cartesian coordinates - pos_x = r * np.cos(theta) - pos_y = r * np.sin(theta) - diff_x = pos_x.diff() - diff_y = pos_y.diff() - norm = np.sqrt(diff_x**2 + diff_y**2) - - # Calculate the RAM and cross-track unit vectors in cartesian and polar - # coordinates. - # x points along MLT = 6, y points along MLT = 12 - inst['unit_ram_x'] = diff_x / norm - inst['unit_ram_y'] = diff_y / norm - inst['unit_cross_x'] = -diff_y / norm - inst['unit_cross_y'] = diff_x / norm - idx, = np.where(inst['mlat'] < 0) - inst.data.loc[inst.index[idx], 'unit_cross_x'] *= -1.0 - inst.data.loc[inst.index[idx], 'unit_cross_y'] *= -1.0 - - inst['unit_ram_r'] = inst['unit_ram_x'] * np.cos(theta) + \ - inst['unit_ram_y'] * np.sin(theta) - inst['unit_ram_theta'] = -inst['unit_ram_x'] * np.sin(theta) + \ - inst['unit_ram_y'] * np.cos(theta) - - inst['unit_cross_r'] = inst['unit_cross_x'] * np.cos(theta) + \ - inst['unit_cross_y'] * np.sin(theta) - inst['unit_cross_theta'] = -inst['unit_cross_x'] * np.sin(theta) + \ - inst['unit_cross_y'] * np.cos(theta) - return - + date_array : array-like + list of datetimes to download data for. The sequence of dates need not + be contiguous. + tag : string + Tag identifier used for particular dataset. This input is provided by + pysat. (default='') + inst_id : string + Satellite ID string identifier used for particular dataset. This input + is provided by pysat. (default='') + data_path : string + Path to directory to download data to. (default=None) + user : string + User string input used for download. Provided by user and passed via + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) + file_type : string + File format for Madrigal data. (default='hdf5') + + Note + ---- + The user's names should be provided in field user. Ritu Karidhal should + be entered as Ritu+Karidhal -def add_drifts_polar_cap_x_y(inst, rpa_flag_key=None, - rpa_vel_key='ion_v_sat_for', - cross_vel_key='ion_v_sat_left'): - """ Add polar cap drifts in cartesian coordinates + The password field should be the user's email address. These parameters + are passed to Madrigal when downloading. - Parameters - ------------ - rpa_flag_key : string or NoneType - RPA flag key, if None will not select any data. The UTD RPA flag key - is 'rpa_flag_ut' (default=None) - rpa_vel_key : string - RPA velocity data key (default='ion_v_sat_for') - cross_vel_key : string - Cross-track velocity data key (default='ion_v_sat_left') - - Returns - ---------- - Adds 'ion_vel_pc_x', 'ion_vel_pc_y', and 'partial'. The last data key - indicates whether RPA data was available (False) or not (True). + The affiliation field is set to pysat to enable tracking of pysat + downloads. - Notes - ------- - Polar cap drifts assume there is no vertical component to the X-Y - velocities """ - - # Get the good RPA data, if available - if rpa_flag_key in list(inst.data.keys()): - rpa_idx, = np.where(inst[rpa_flag_key] != 1) - else: - rpa_idx = list() - - # Use the cartesian unit vectors to calculate the desired velocities - iv_x = inst[rpa_vel_key].copy() - iv_x[rpa_idx] = 0.0 - - # Check to see if unit vectors have been created - if 'unit_ram_y' not in list(inst.data.keys()): - add_drift_unit_vectors(inst) - - # Calculate the velocities - inst['ion_vel_pc_x'] = iv_x * inst['unit_ram_x'] + \ - inst[cross_vel_key] * inst['unit_cross_x'] - inst['ion_vel_pc_y'] = iv_x * inst['unit_ram_y'] + \ - inst[cross_vel_key] * inst['unit_cross_y'] - - # Flag the velocities as full (False) or partial (True) - inst['partial'] = False - inst[rpa_idx, 'partial'] = True - + general.download(date_array, inst_code=str(madrigal_inst_code), + kindat=madrigal_tag[inst_id][tag], data_path=data_path, + user=user, password=password, file_type=file_type) return diff --git a/pysatMadrigal/instruments/gnss_tec.py b/pysatMadrigal/instruments/gnss_tec.py new file mode 100644 index 0000000..f1a8d9a --- /dev/null +++ b/pysatMadrigal/instruments/gnss_tec.py @@ -0,0 +1,220 @@ +# -*- coding: utf-8 -*-. +"""Supports the MIT Haystack GNSS TEC data products + +The Global Navigation Satellite System (GNSS) is used in conjunction with a +world-wide receiver network to produce total electron content (TEC) data +products, including vertical and line-of-sight TEC. + +Downloads data from the MIT Haystack Madrigal Database. + +Properties +---------- +platform + 'gnss' +name + 'tec' +tag + 'vtec' + +Examples +-------- +:: + + import datetime as dt + import pysat + import pysatMadrigal as pymad + + vtec = pysat.Instrument(inst_module=pymad.instruments.gnss_tec, tag='vtec') + vtec.download(dt.datetime(2017, 11, 19), dt.datetime(2017, 11, 20), + user='Firstname+Lastname', password='email@address.com') + vtec.load(date=dt.datetime(2017, 11, 19)) + + +Note +---- +Please provide name and email when downloading data with this routine. + +""" + +import datetime as dt +import functools +import numpy as np + +from pysat import logger + +from pysatMadrigal.instruments.methods import general, gnss + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'gnss' +name = 'tec' +tags = {'vtec': 'vertical TEC'} +inst_ids = {'': [tag for tag in tags.keys()]} + +pandas_format = False + +# Local attributes +dname = '{{year:02d}}{{month:02d}}{{day:02d}}' +vname = '.{{version:03d}}' +supported_tags = {ss: {'vtec': ''.join(['gps', dname, 'g', vname, + ".{file_type}"])} + for ss in inst_ids.keys()} +remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5') + for kk in inst_ids[ss]} for ss in inst_ids.keys()} + +# Madrigal tags +madrigal_inst_code = 8000 +madrigal_tag = {'': {'vtec': '3500'}} # , 'los': '3505'}} <- Issue #12 + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'vtec': dt.datetime(2017, 11, 19)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + + +def init(self): + """Initializes the Instrument object with values specific to GNSS TEC + + Runs once upon instantiation. + + """ + + ackn_str = '\n'.join([gnss.acknowledgements(self.name), + general.cedar_rules()]) + + logger.info(ackn_str) + self.acknowledgements = ackn_str + self.references = gnss.references(self.name, self.tag) + + return + + +def clean(self): + """Routine to return GNSS TEC data at a specific level + + Note + ---- + Supports 'clean', 'dusty', 'dirty', or 'None'. + Routine is called by pysat, and not by the end user directly. + + """ + if self.tag == "vtec": + logger.info("".join(["Data provided at a clean level, further ", + "cleaning may be performed using the ", + "measurement error 'dtec'"])) + + return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal methods + +# Support listing the local files +list_files = functools.partial(general.list_files, + supported_tags=supported_tags, + two_digit_year_break=99) + +# Support listing files currently available on remote server (Madrigal) +list_remote_files = functools.partial(general.list_remote_files, + supported_tags=remote_tags, + inst_code=madrigal_inst_code, + kindats=madrigal_tag) + + +def download(date_array, tag='', inst_id='', data_path=None, user=None, + password=None, url='http://cedar.openmadrigal.org', + file_type='netCDF4'): + """Downloads data from Madrigal. + + Parameters + ---------- + date_array : array-like + list of datetimes to download data for. The sequence of dates need not + be contiguous. + tag : str + Tag identifier used for particular dataset. This input is provided by + pysat. (default='') + inst_id : str + Instrument ID string identifier used for particular dataset. This input + is provided by pysat. (default='') + data_path : str + Path to directory to download data to. (default=None) + user : str + User string input used for download. Provided by user and passed via + pysat. (default=None) + password : str + Password for data download. (default=None) + url : str + URL for Madrigal site (default='http://cedar.openmadrigal.org') + file_type : str + File format for Madrigal data. (default='netCDF4') + + Note + ---- + The user's names should be provided in field user. Anthea Coster should + be entered as Anthea+Coster + + The password field should be the user's email address. These parameters + are passed to Madrigal when downloading. + + The affiliation field is set to pysat to enable tracking of pysat + downloads. + + """ + general.download(date_array, inst_code=str(madrigal_inst_code), + kindat=madrigal_tag[inst_id][tag], data_path=data_path, + user=user, password=password, file_type=file_type, url=url) + + return + + +def load(fnames, tag=None, inst_id=None): + """ Routine to load the GNSS TEC data + + Parameters + ---------- + fnames : list + List of filenames + tag : str or NoneType + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default=None) + inst_id : str or NoneType + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default=None) + + Returns + ------- + data : xarray.Dataset + Object containing satellite data + meta : pysat.Meta + Object containing metadata such as column names and units + + """ + # Define the xarray coordinate dimensions (apart from time) + # Not needed for netCDF + xcoords = {'vtec': {('time', 'gdlat', 'glon', 'kindat', 'kinst'): + ['gdalt', 'tec', 'dtec'], + ('time', ): ['year', 'month', 'day', 'hour', 'min', + 'sec', 'ut1_unix', 'ut2_unix', 'recno']}} + + # Load the specified data + data, meta = general.load(fnames, tag, inst_id, xarray_coords=xcoords[tag]) + + # Squeeze the kindat and kinst 'coordinates', but keep them as floats + squeeze_dims = np.array(['kindat', 'kinst']) + squeeze_mask = [sdim in data.coords for sdim in squeeze_dims] + if np.any(squeeze_mask): + data = data.squeeze(dim=squeeze_dims[squeeze_mask]) + + # Fix the units for tec and dtec + if tag == 'vtec': + meta['tec'] = {meta.labels.units: 'TECU'} + meta['dtec'] = {meta.labels.units: 'TECU'} + + return data, meta diff --git a/pysatMadrigal/instruments/jro_isr.py b/pysatMadrigal/instruments/jro_isr.py index b92d13a..93dc70a 100644 --- a/pysatMadrigal/instruments/jro_isr.py +++ b/pysatMadrigal/instruments/jro_isr.py @@ -8,22 +8,25 @@ Downloads data from the JRO Madrigal Database. -Parameters +Properties ---------- -platform : string +platform 'jro' -name : string +name 'isr' -tag : string +tag 'drifts', 'drifts_ave', 'oblique_stan', 'oblique_rand', 'oblique_long' -Example -------- +Examples +-------- +:: + import pysat - dmsp = pysat.Instrument('jro', 'isr', 'drifts', clean_level='clean') - dmsp.download(dt.datetime(2017, 12, 30), dt.datetime(2017, 12, 31), - user='Firstname+Lastname', password='email@address.com') - dmsp.load(2017,363) + jro = pysat.Instrument('jro', 'isr', 'drifts', clean_level='clean') + jro.download(pysat.datetime(2017, 12, 30), pysat.datetime(2017, 12, 31), + user='Firstname+Lastname', password='email@address.com') + jro.load(2017, 363) + Note ---- @@ -31,19 +34,16 @@ """ -from __future__ import print_function -from __future__ import absolute_import import datetime as dt import functools import numpy as np -from pysatMadrigal.instruments.methods import madrigal as mad_meth -from pysat.instruments.methods import general as mm_gen -from pysat.utils import coords +from pysat import logger -import logging -logger = logging.getLogger(__name__) +from pysatMadrigal.instruments.methods import general, jro +# ---------------------------------------------------------------------------- +# Instrument attributes platform = 'jro' name = 'isr' @@ -51,39 +51,39 @@ 'oblique_stan': 'Standard Faraday rotation double-pulse', 'oblique_rand': 'Randomized Faraday rotation double-pulse', 'oblique_long': 'Long pulse Faraday rotation'} -sat_ids = {'': list(tags.keys())} -_test_dates = {'': {'drifts': dt.datetime(2010, 1, 19), - 'drifts_ave': dt.datetime(2010, 1, 19), - 'oblique_stan': dt.datetime(2010, 4, 19), - 'oblique_rand': dt.datetime(2000, 11, 9), - 'oblique_long': dt.datetime(2010, 4, 12)}} +inst_ids = {'': list(tags.keys())} + pandas_format = False -# support list files routine -# use the default CDAWeb method -jro_fname1 = 'jro{year:4d}{month:02d}{day:02d}' -jro_fname2 = '.{version:03d}.hdf5' +# Local attributes +jro_fname1 = 'jro{{year:4d}}{{month:02d}}{{day:02d}}' +jro_fname2 = '.{{version:03d}}.{file_type}' supported_tags = {ss: {'drifts': jro_fname1 + "drifts" + jro_fname2, 'drifts_ave': jro_fname1 + "drifts_avg" + jro_fname2, 'oblique_stan': jro_fname1 + jro_fname2, 'oblique_rand': jro_fname1 + "?" + jro_fname2, 'oblique_long': jro_fname1 + "?" + jro_fname2} - for ss in sat_ids.keys()} -list_files = functools.partial(mm_gen.list_files, - supported_tags=supported_tags) + for ss in inst_ids.keys()} +remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5') + for kk in inst_ids[ss]} for ss in inst_ids.keys()} -# madrigal tags +# Madrigal tags madrigal_inst_code = 10 -madrigal_tag = {'': {'drifts': 1910, 'drifts_ave': 1911, 'oblique_stan': 1800, - 'oblique_rand': 1801, 'oblique_long': 1802}, } +madrigal_tag = {'': {'drifts': "1910", 'drifts_ave': "1911", + 'oblique_stan': "1800", 'oblique_rand': "1801", + 'oblique_long': "1802"}, } -# support listing files currently available on remote server (Madrigal) -list_remote_files = functools.partial(mad_meth.list_remote_files, - supported_tags=supported_tags, - inst_code=madrigal_inst_code) +# ---------------------------------------------------------------------------- +# Instrument test attributes -# support load routine -load = functools.partial(mad_meth.load, xarray_coords=['gdalt']) +_test_dates = {'': {'drifts': dt.datetime(2010, 1, 19), + 'drifts_ave': dt.datetime(2010, 1, 19), + 'oblique_stan': dt.datetime(2010, 4, 19), + 'oblique_rand': dt.datetime(2000, 11, 9), + 'oblique_long': dt.datetime(2010, 4, 12)}} + +# ---------------------------------------------------------------------------- +# Instrument methods # Madrigal will sometimes include multiple days within a file # labeled with a single date. @@ -91,96 +91,31 @@ # To ensure this function is always applied first, we set the filter # function as the default function for (JRO). # Default function is run first by the nanokernel on every load call. -default = mad_meth.filter_data_single_date +preprocess = general.filter_data_single_date def init(self): """Initializes the Instrument object with values specific to JRO ISR - - Runs once upon instantiation. - - Parameters - ---------- - self : pysat.Instrument - This object - - Returns - -------- - Void : (NoneType) - Object modified in place. - - """ - logger.info(' '.join(["The Jicamarca Radio Observatory is operated by", - "the Instituto Geofisico del Peru, Ministry of", - "Education, with support from the National Science", - "Foundation as contracted through Cornell", - "University. ", mad_meth.cedar_rules()])) - return - + ackn_str = '\n'.join([jro.acknowledgements(), general.cedar_rules()]) -def download(date_array, tag='', sat_id='', data_path=None, user=None, - password=None): - """Downloads data from Madrigal. + logger.info(ackn_str) + self.acknowledgements = ackn_str + self.references = jro.references() - Parameters - ---------- - date_array : array-like - list of datetimes to download data for. The sequence of dates need not - be contiguous. - tag : string ('') - Tag identifier used for particular dataset. This input is provided by - pysat. - sat_id : string ('') - Satellite ID string identifier used for particular dataset. This input - is provided by pysat. - data_path : string (None) - Path to directory to download data to. - user : string (None) - User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - - Returns - -------- - Void : (NoneType) - Downloads data to disk. - - Notes - ----- - The user's names should be provided in field user. Ruby Payne-Scott should - be entered as Ruby+Payne-Scott - - The password field should be the user's email address. These parameters - are passed to Madrigal when downloading. - - The affiliation field is set to pysat to enable tracking of pysat - downloads. - - """ - mad_meth.download(date_array, inst_code=str(madrigal_inst_code), - kindat=str(madrigal_tag[sat_id][tag]), - data_path=data_path, user=user, password=password) + return def clean(self): """Routine to return JRO ISR data cleaned to the specified level - Returns - -------- - Void : (NoneType) - data in inst is modified in-place. - - Notes - -------- - Supports 'clean', 'dusty', 'dirty' - 'Clean' is unknown for oblique modes, over 200 km for drifts - 'Dusty' is unknown for oblique modes, over 200 km for drifts - 'Dirty' is unknown for oblique modes, over 200 km for drifts + Note + ---- + Supports 'clean' + 'clean' is unknown for oblique modes, over 200 km for drifts + 'dusty' is the same as clean + 'Dirty' is the same as clean 'None' None Routine is called by pysat, and not by the end user directly. @@ -220,71 +155,135 @@ def clean(self): return -def calc_measurement_loc(self): - """ Calculate the instrument measurement location in geographic coordinates +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal and pysat methods + +# Support listing the local files +list_files = functools.partial(general.list_files, + supported_tags=supported_tags) + +# Set list_remote_files routine +list_remote_files = functools.partial(general.list_remote_files, + supported_tags=remote_tags, + inst_code=madrigal_inst_code, + kindats=madrigal_tag) - Returns - ------- - Void : adds 'gdlat#', 'gdlon#' to the instrument, for all directions that - have azimuth and elevation keys that match the format 'eldir#' and 'azdir#' + +def download(date_array, tag='', inst_id='', data_path=None, user=None, + password=None, file_type='hdf5'): + """Downloads data from Madrigal. + + Parameters + ---------- + date_array : array-like + list of datetimes to download data for. The sequence of dates need not + be contiguous. + tag : str + Tag identifier used for particular dataset. This input is provided by + pysat. (default='') + inst_id : str + Satellite ID string identifier used for particular dataset. This input + is provided by pysat. (default='') + data_path : str + Path to directory to download data to. (default=None) + user : str + User string input used for download. Provided by user and passed via + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : str + Password for data download. (default=None) + file_type : str + File format for Madrigal data. (default='hdf5') + + Notes + ----- + The user's names should be provided in field user. Ruby Payne-Scott should + be entered as Ruby+Payne-Scott + + The password field should be the user's email address. These parameters + are passed to Madrigal when downloading. + + The affiliation field is set to pysat to enable tracking of pysat + downloads. """ + general.download(date_array, inst_code=str(madrigal_inst_code), + kindat=madrigal_tag[inst_id][tag], data_path=data_path, + user=user, password=password, file_type=file_type) - az_keys = [kk[5:] for kk in list(self.data.keys()) - if kk.find('azdir') == 0] - el_keys = [kk[5:] for kk in list(self.data.keys()) - if kk.find('eldir') == 0] - good_dir = list() - - for i, kk in enumerate(az_keys): - if kk in el_keys: - try: - good_dir.append(int(kk)) - except ValueError: - logger.warning("unknown direction number [{:}]".format(kk)) - - # Calculate the geodetic latitude and longitude for each direction - if len(good_dir) == 0: - raise ValueError("No matching azimuth and elevation data included") - - for dd in good_dir: - # Format the direction location keys - az_key = 'azdir{:d}'.format(dd) - el_key = 'eldir{:d}'.format(dd) - lat_key = 'gdlat{:d}'.format(dd) - lon_key = 'gdlon{:d}'.format(dd) - # JRO is located 520 m above sea level (jro.igp.gob.pe./english/) - # Also, altitude has already been calculated - gdaltr = np.ones(shape=self['gdlonr'].shape) * 0.52 - gdlat, gdlon, _ = coords.local_horizontal_to_global_geo(self[az_key], - self[el_key], - self['range'], - self['gdlatr'], - self['gdlonr'], - gdaltr, - geodetic=True) - - # Assigning as data, to ensure that the number of coordinates match - # the number of data dimensions - self.data = self.data.assign({lat_key: gdlat, lon_key: gdlon}) - - # Add metadata for the new data values - bm_label = "Beam {:d} ".format(dd) - self.meta[lat_key] = {self.meta.units_label: 'degrees', - self.meta.name_label: bm_label + 'latitude', - self.meta.desc_label: bm_label + 'latitude', - self.meta.plot_label: bm_label + 'Latitude', - self.meta.axis_label: bm_label + 'Latitude', - self.meta.scale_label: 'linear', - self.meta.min_label: -90.0, - self.meta.max_label: 90.0, - self.meta.fill_label: np.nan} - self.meta[lon_key] = {self.meta.units_label: 'degrees', - self.meta.name_label: bm_label + 'longitude', - self.meta.desc_label: bm_label + 'longitude', - self.meta.plot_label: bm_label + 'Longitude', - self.meta.axis_label: bm_label + 'Longitude', - self.meta.scale_label: 'linear', - self.meta.fill_label: np.nan} - return +def load(fnames, tag=None, inst_id=None): + """ Routine to load the JRO ISR data + + Parameters + ----------- + fnames : list + List of filenames + tag : str or NoneType + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default=None) + inst_id : str or NoneType + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default=None) + + Returns + -------- + data : xarray.Dataset + Object containing satellite data + meta : pysat.Meta + Object containing metadata such as column names and units + + """ + # Define the xarray coordinate dimensions (apart from time) + xcoords = {'drifts': {('time', 'gdalt', 'gdlatr', 'gdlonr', 'kindat', + 'kinst'): ['nwlos', 'range', 'vipn2', 'dvipn2', + 'vipe1', 'dvipe1', 'vi72', 'dvi72', + 'vi82', 'dvi82', 'paiwl', 'pacwl', + 'pbiwl', 'pbcwl', 'pciel', 'pccel', + 'pdiel', 'pdcel', 'jro10', 'jro11'], + ('time', ): ['year', 'month', 'day', 'hour', 'min', + 'sec', 'spcst', 'pl', 'cbadn', 'inttms', + 'azdir7', 'eldir7', 'azdir8', 'eldir8', + 'jro14', 'jro15', 'jro16', 'ut1_unix', + 'ut2_unix', 'recno']}, + 'drifts_ave': {('time', 'gdalt', 'gdlatr', 'gdlonr', 'kindat', + 'kinst'): ['altav', 'range', 'vipn2', 'dvipn2', + 'vipe1', 'dvipe1'], + ('time', ): ['year', 'month', 'day', 'hour', + 'min', 'sec', 'spcst', 'pl', + 'cbadn', 'inttms', 'ut1_unix', + 'ut2_unix', 'recno']}, + 'oblique_stan': {('time', 'gdalt', 'gdlatr', 'gdlonr', 'kindat', + 'kinst'): ['rgate', 'ne', 'dne', 'te', 'dte', + 'ti', 'dti', 'ph+', 'dph+', 'phe+', + 'dphe+'], + ('time', ): ['year', 'month', 'day', 'hour', + 'min', 'sec', 'azm', 'elm', + 'pl', 'inttms', 'tfreq', + 'ut1_unix', 'ut2_unix', 'recno']}, + 'oblique_rand': {('time', 'gdalt', 'gdlatr', 'gdlonr', 'kindat', + 'kinst'): ['rgate', 'pop', 'dpop', 'te', 'dte', + 'ti', 'dti', 'ph+', 'dph+', 'phe+', + 'dphe+'], + ('time', ): ['year', 'month', 'day', 'hour', + 'min', 'sec', 'azm', 'elm', + 'pl', 'inttms', 'tfreq', + 'ut1_unix', 'ut2_unix', 'recno']}, + 'oblique_long': {('time', 'gdalt', 'gdlatr', 'gdlonr', 'kindat', + 'kinst'): ['rgate', 'pop', 'dpop', 'te', 'dte', + 'ti', 'dti', 'ph+', 'dph+', 'phe+', + 'dphe+'], + ('time', ): ['year', 'month', 'day', 'hour', + 'min', 'sec', 'azm', 'elm', + 'pl', 'inttms', 'tfreq', + 'ut1_unix', 'ut2_unix', 'recno']}} + + # Load the specified data + data, meta = general.load(fnames, tag, inst_id, xarray_coords=xcoords[tag]) + + # Squeeze the kindat and kinst 'coordinates', but keep them as floats + data = data.squeeze(dim=['kindat', 'kinst', 'gdlatr', 'gdlonr']) + + return data, meta diff --git a/pysatMadrigal/instruments/methods/__init__.py b/pysatMadrigal/instruments/methods/__init__.py index f9df482..9554193 100644 --- a/pysatMadrigal/instruments/methods/__init__.py +++ b/pysatMadrigal/instruments/methods/__init__.py @@ -1 +1,2 @@ -from pysatMadrigal.instruments.methods import madrigal # noqa F401 +from pysatMadrigal.instruments.methods import dmsp, general # noqa F401 +from pysatMadrigal.instruments.methods import gnss, jro # noqa F401 diff --git a/pysatMadrigal/instruments/methods/dmsp.py b/pysatMadrigal/instruments/methods/dmsp.py new file mode 100644 index 0000000..ee15793 --- /dev/null +++ b/pysatMadrigal/instruments/methods/dmsp.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- +# -*- coding: utf-8 -*- +"""Methods supporting the Defense Meteorological Satellite Program (DMSP) +platform + +""" + +import numpy as np +import pandas as pds + +from pysat import logger + + +def references(name): + """Provides references for the DMSP instruments and experiments + + Parameters + ---------- + name : str + Instrument name + + Returns + ------- + refs : str + String providing reference guidenance for the DMSP data + + """ + + refs = {'ivm': ' '.join(('F. J. Rich, Users Guide for the Topside', + 'Ionospheric Plasma Monitor (SSIES, SSIES-2 and', + 'SSIES-3) on Spacecraft of the Defense', + 'Meteorological Satellite Program (Air Force', + 'Phillips Laboratory, Hanscom AFB, MA, 1994),', + 'Vol. 1, p. 25.'))} + + return refs[name] + + +def smooth_ram_drifts(inst, rpa_flag_key=None, rpa_vel_key='ion_v_sat_for'): + """ Smooth the ram drifts using a rolling mean + + Parameters + ---------- + inst : pysat.Instrument + DMSP IVM Instrument object + rpa_flag_key : string or NoneType + RPA flag key, if None will not select any data. The UTD RPA flag key + is 'rpa_flag_ut' (default=None) + rpa_vel_key : string + RPA velocity data key (default='ion_v_sat_for') + + """ + + if rpa_flag_key in list(inst.data.keys()): + rpa_idx, = np.where(inst[rpa_flag_key] == 1) + else: + rpa_idx = list() + + inst[rpa_idx, rpa_vel_key] = inst[rpa_idx, + rpa_vel_key].rolling(15, 5).mean() + return + + +def update_DMSP_ephemeris(inst, ephem=None): + """Updates DMSP instrument data with DMSP ephemeris + + Parameters + ---------- + inst : pysat.Instrument + DMSP IVM Instrumet object + ephem : pysat.Instrument or NoneType + DMSP IVM_EPHEM instrument object + + """ + + # Ensure the right ephemera is loaded + if ephem is None: + logger.info('No ephemera provided for {:}'.format(inst.date)) + inst.data = pds.DataFrame(None) + return + + if ephem.inst_id != inst.inst_id: + raise ValueError('ephemera provided for the wrong satellite') + + if ephem.date != inst.date: + ephem.load(date=inst.date, verifyPad=True) + + if ephem.data.empty: + logger.info('unable to load ephemera for {:}'.format(inst.date)) + inst.data = pds.DataFrame(None) + return + + # Reindex the ephemeris data + ephem.data = ephem.data.reindex(index=inst.data.index, method='pad') + ephem.data = ephem.data.interpolate('time') + + # Update the DMSP instrument + inst['mlt'] = ephem['SC_AACGM_LTIME'] + inst['mlat'] = ephem['SC_AACGM_LAT'] + + return + + +def add_drift_unit_vectors(inst): + """ Add unit vectors for the satellite velocity + + Parameters + ---------- + inst : pysat.Instrument + DMSP IVM Instrument object + + Note + ---- + Assumes that the RAM vector is pointed perfectly forward + + """ + # Calculate theta and R in radians from MLT and MLat, respectively + theta = inst['mlt'] * (np.pi / 12.0) - np.pi * 0.5 + r = np.radians(90.0 - inst['mlat'].abs()) + + # Determine the positions in cartesian coordinates + pos_x = r * np.cos(theta) + pos_y = r * np.sin(theta) + diff_x = pos_x.diff() + diff_y = pos_y.diff() + norm = np.sqrt(diff_x**2 + diff_y**2) + + # Calculate the RAM and cross-track unit vectors in cartesian and polar + # coordinates. + # x points along MLT = 6, y points along MLT = 12 + inst['unit_ram_x'] = diff_x / norm + inst['unit_ram_y'] = diff_y / norm + inst['unit_cross_x'] = -diff_y / norm + inst['unit_cross_y'] = diff_x / norm + idx, = np.where(inst['mlat'] < 0) + inst.data.loc[inst.index[idx], 'unit_cross_x'] *= -1.0 + inst.data.loc[inst.index[idx], 'unit_cross_y'] *= -1.0 + + inst['unit_ram_r'] = (inst['unit_ram_x'] * np.cos(theta) + + inst['unit_ram_y'] * np.sin(theta)) + inst['unit_ram_theta'] = (-inst['unit_ram_x'] * np.sin(theta) + + inst['unit_ram_y'] * np.cos(theta)) + + inst['unit_cross_r'] = (inst['unit_cross_x'] * np.cos(theta) + + inst['unit_cross_y'] * np.sin(theta)) + inst['unit_cross_theta'] = (-inst['unit_cross_x'] * np.sin(theta) + + inst['unit_cross_y'] * np.cos(theta)) + return + + +def add_drifts_polar_cap_x_y(inst, rpa_flag_key=None, + rpa_vel_key='ion_v_sat_for', + cross_vel_key='ion_v_sat_left'): + """ Add polar cap drifts in cartesian coordinates + + Parameters + ---------- + inst : pysat.Instrument + DMSP IVM Instrument object + rpa_flag_key : string or NoneType + RPA flag key, if None will not select any data. The UTD RPA flag key + is 'rpa_flag_ut' (default=None) + rpa_vel_key : string + RPA velocity data key (default='ion_v_sat_for') + cross_vel_key : string + Cross-track velocity data key (default='ion_v_sat_left') + + Note + ---- + Polar cap drifts assume there is no vertical component to the X-Y + velocities. + + Adds 'ion_vel_pc_x', 'ion_vel_pc_y', and 'partial'. The last data key + indicates whether RPA data was available (False) or not (True). + + """ + + # Get the good RPA data, if available + if rpa_flag_key in list(inst.data.keys()): + rpa_idx, = np.where(inst[rpa_flag_key] != 1) + else: + rpa_idx = list() + + # Use the cartesian unit vectors to calculate the desired velocities + iv_x = inst[rpa_vel_key].copy() + iv_x[rpa_idx] = 0.0 + + # Check to see if unit vectors have been created + if 'unit_ram_y' not in list(inst.data.keys()): + add_drift_unit_vectors(inst) + + # Calculate the velocities + inst['ion_vel_pc_x'] = (iv_x * inst['unit_ram_x'] + + inst[cross_vel_key] * inst['unit_cross_x']) + inst['ion_vel_pc_y'] = (iv_x * inst['unit_ram_y'] + + inst[cross_vel_key] * inst['unit_cross_y']) + + # Flag the velocities as full (False) or partial (True) + inst['partial'] = False + inst[rpa_idx, 'partial'] = True + + return diff --git a/pysatMadrigal/instruments/methods/general.py b/pysatMadrigal/instruments/methods/general.py new file mode 100644 index 0000000..1fd081e --- /dev/null +++ b/pysatMadrigal/instruments/methods/general.py @@ -0,0 +1,835 @@ +# -*- coding: utf-8 -*-. +"""General routines for integrating CEDAR Madrigal instruments into pysat. + +""" + +import datetime as dt +import gzip +import numpy as np +import os +import pandas as pds +import xarray as xr + +import h5py +import pysat + +from madrigalWeb import madrigalWeb + + +logger = pysat.logger +file_types = {'hdf5': 'hdf5', 'netCDF4': 'netCDF4', 'simple': 'simple.gz'} + + +def cedar_rules(): + """General acknowledgement statement for Madrigal data. + + Returns + ------- + ackn : str + String with general acknowledgement for all CEDAR Madrigal data + + """ + ackn = "".join(["Contact the PI when using this data, in accordance ", + "with the CEDAR 'Rules of the Road'"]) + return ackn + + +def load(fnames, tag=None, inst_id=None, xarray_coords=None): + """Loads data from Madrigal into Pandas or XArray + + Parameters + ---------- + fnames : array-like + iterable of filename strings, full path, to data files to be loaded. + This input is nominally provided by pysat itself. + tag : str + tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. While + tag defaults to None here, pysat provides '' as the default + tag unless specified by user at Instrument instantiation. (default='') + inst_id : str + Satellite ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + xarray_coords : list or NoneType + List of keywords to use as coordinates if xarray output is desired + instead of a Pandas DataFrame. Can build an xarray Dataset + that have different coordinate dimensions by providing a dict + inside the list instead of coordinate variable name strings. Each dict + will have a tuple of coordinates as the key and a list of variable + strings as the value. Empty list if None. For example, + xarray_coords=[{('time',): ['year', 'doy'], + ('time', 'gdalt'): ['data1', 'data2']}]. (default=None) + + Returns + ------- + data : pds.DataFrame or xr.Dataset + A pandas DataFrame or xarray Dataset holding the data from the file + meta : pysat.Meta + Metadata from the file, as well as default values from pysat + + Note + ---- + Currently HDF5 reading breaks if a different file type was used previously + + This routine is called as needed by pysat. It is not intended + for direct user interaction. + + """ + # Test the file formats + load_file_types = {ftype: [] for ftype in file_types.keys()} + for fname in fnames: + for ftype in file_types.keys(): + if fname.find(ftype) > 0: + load_file_types[ftype].append(fname) + break + + # Initialize xarray coordinates, if needed + if xarray_coords is None: + xarray_coords = [] + + # Initialize the output + meta = pysat.Meta() + labels = [] + data = None + + # Load the file data for netCDF4 files + if len(load_file_types["netCDF4"]) == 1: + # Xarray natively opens netCDF data into a Dataset + file_data = xr.open_dataset(load_file_types["netCDF4"][0], + engine="netcdf4") + elif len(load_file_types["netCDF4"]) > 1: + file_data = xr.open_mfdataset(load_file_types["netCDF4"], + combine='by_coords', engine="netcdf4") + + if len(load_file_types["netCDF4"]) > 0: + # Currently not saving file header data, as all metadata is at + # the data variable level + for item in file_data.data_vars.keys(): + name_string = item + unit_string = file_data[item].attrs['units'] + desc_string = file_data[item].attrs['description'] + meta[name_string.lower()] = {meta.labels.name: name_string, + meta.labels.units: unit_string, + meta.labels.desc: desc_string} + + # Remove any metadata from xarray + file_data[item].attrs = {} + + # Reset UNIX timestamp as datetime and set it as an index + file_data = file_data.rename({'timestamps': 'time'}) + time_data = pds.to_datetime(file_data['time'].values, unit='s') + data = file_data.assign_coords({'time': ('time', time_data)}) + + # Load the file data for HDF5 files + if len(load_file_types["hdf5"]) > 0 or len(load_file_types["simple"]) > 0: + # Ensure we don't try to create an xarray object with only time as + # the coordinate + coord_len = len(xarray_coords) + if 'time' in xarray_coords: + coord_len -= 1 + + # Cycle through all the filenames + fdata = [] + fnames = list(load_file_types["hdf5"]) + fnames.extend(load_file_types["simple"]) + for fname in fnames: + # Open the specified file + if fname in load_file_types["simple"]: + # Get the gzipped text data + with gzip.open(fname, 'rb') as fin: + file_data = fin.readlines() + + # Load available info into pysat.Meta if this is the first file + header = [item.decode('UTF-8') + for item in file_data.pop(0).split()] + if len(labels) == 0: + for item in header: + labels.append(item) + + # Only update metadata if necessary + if item.lower() not in meta: + meta[item.lower()] = {meta.labels.name: item} + + # Construct a dict of the output + file_dict = {item.lower(): list() for item in header} + for line in file_data: + for i, val in enumerate(line.split()): + file_dict[header[i].lower()].append(float(val)) + + # Load data into frame, with labels from metadata + ldata = pds.DataFrame.from_dict(file_dict) + else: + # Open the specified file and get the data and metadata + filed = h5py.File(fname, 'r') + file_data = filed['Data']['Table Layout'] + file_meta = filed['Metadata']['Data Parameters'] + + # Load available info into pysat.Meta if this is the first file + if len(labels) == 0: + for item in file_meta: + name_string = item[0].decode('UTF-8') + unit_string = item[3].decode('UTF-8') + desc_string = item[1].decode('UTF-8') + labels.append(name_string) + + # Only update metadata if necessary + if name_string.lower() not in meta: + meta[name_string.lower()] = { + meta.labels.name: name_string, + meta.labels.units: unit_string, + meta.labels.desc: desc_string} + + # Add additional metadata notes. Custom attributes attached to + # meta are attached to corresponding Instrument object when + # pysat receives data and meta from this routine + for key in filed['Metadata']: + if key != 'Data Parameters': + setattr(meta, key.replace(' ', '_'), + filed['Metadata'][key][:]) + + # Load data into frame, with labels from metadata + ldata = pds.DataFrame.from_records(file_data, columns=labels) + + # Enforce lowercase variable names + ldata.columns = [item.lower() for item in ldata.columns] + + # Extended processing is the same for simple and HDF5 files + # + # Construct datetime index from times + time_keys = np.array(['year', 'month', 'day', 'hour', 'min', 'sec']) + if not np.all([key in ldata.columns for key in time_keys]): + time_keys = [key for key in time_keys + if key not in ldata.columns] + raise ValueError(' '.join(["unable to construct time index, ", + "missing {:}".format(time_keys)])) + + uts = 3600.0 * ldata.loc[:, 'hour'] + 60.0 * ldata.loc[:, 'min'] \ + + ldata.loc[:, 'sec'] + time = pysat.utils.time.create_datetime_index( + year=ldata.loc[:, 'year'], month=ldata.loc[:, 'month'], + day=ldata.loc[:, 'day'], uts=uts) + + # Declare index or recast as xarray + if coord_len > 0: + # If a list was provided, recast as a dict and grab the data + # columns + if not isinstance(xarray_coords, dict): + xarray_coords = {tuple(xarray_coords): + [col for col in ldata.columns + if col not in xarray_coords]} + + # Determine the order in which the keys should be processed: + # Greatest to least number of dimensions + len_dict = {len(xcoords): xcoords + for xcoords in xarray_coords.keys()} + coord_order = [len_dict[xkey] for xkey in sorted( + [lkey for lkey in len_dict.keys()], reverse=True)] + + # Append time to the data frame + ldata = ldata.assign(time=pds.Series(time, index=ldata.index)) + + # Cycle through each of the coordinate dimensions + xdatasets = list() + for xcoords in coord_order: + if not np.all([xkey.lower() in ldata.columns + for xkey in xcoords]): + raise ValueError(''.join(['unknown coordinate key ', + 'in [{:}'.format(xcoords), + '], use only: {:}'.format( + ldata.columns)])) + if not np.all([xkey.lower() in ldata.columns + for xkey in xarray_coords[xcoords]]): + data_mask = [xkey.lower() in ldata.columns + for xkey in xarray_coords[xcoords]] + if np.all(~np.array(data_mask)): + raise ValueError(''.join([ + 'all provided data variables [', + '{:}] are unk'.format(xarray_coords[xcoords]), + 'nown, use only: {:}'.format(ldata.columns)])) + else: + logger.warning(''.join([ + 'unknown data variable in [', + '{:}], use'.format(xarray_coords[xcoords]), + ' only: {:}'.format(ldata.columns)])) + + # Remove the coordinates that aren't present + temp = np.array(xarray_coords[xcoords])[data_mask] + xarray_coords[xcoords] = list(temp) + + # Select the desired data values + sel_data = ldata[list(xcoords) + xarray_coords[xcoords]] + + # Remove duplicates before indexing, to ensure data with + # the same values at different locations are kept + sel_data = sel_data.drop_duplicates() + + # Set the indices + sel_data = sel_data.set_index(list(xcoords)) + + # Recast as an xarray + xdatasets.append(sel_data.to_xarray()) + + # Get the necessary information to test the data + lcols = ldata.columns + len_data = len(lcols) + + # Merge all of the datasets + ldata = xr.merge(xdatasets) + test_variables = [xkey for xkey in ldata.variables.keys()] + ltest = len(test_variables) + + # Test to see that all data was retrieved + if ltest != len_data: + if ltest < len_data: + estr = 'missing: {:}'.format( + ' '.join([dvar for dvar in lcols + if dvar not in test_variables])) + else: + estr = 'have extra: {:}'.format( + ' '.join([tvar for tvar in test_variables + if tvar not in lcols])) + raise ValueError(''.join([ + 'coordinates not supplied for all data columns', + ': {:d} != {:d}; '.format(ltest, len_data), estr])) + else: + # Set the index to time + ldata.index = time + + # Raise a logging warning if there are duplicate times. This + # means the data should be stored as an xarray Dataset + if np.any(time.duplicated()): + logger.warning(''.join(["duplicated time indices, ", + "consider specifing additional", + " coordinates and storing the ", + "data as an xarray Dataset"])) + + # Compile a list of the data objects + fdata.append(ldata) + + # Merge the data together, accounting for potential netCDF output + if data is None and len(fdata) == 1: + data = fdata[0] + else: + if coord_len > 0: + if data is None: + data = xr.merge(fdata) + else: + data = xr.combine_by_coords([data, xr.merge(fdata)]) + else: + if data is None: + data = pds.concat(fdata) + data = data.sort_index() + else: + ldata = pds.concat(fdata).sort_index().to_xarray() + ldata = ldata.rename({'index': 'time'}) + data = xr.combine_by_coords([data, ldata]).to_pandas() + + # Ensure that data is at least an empty Dataset + if data is None: + if len(xarray_coords) > 0: + data = xr.Dataset() + else: + data = pds.DataFrame(dtype=np.float64) + + return data, meta + + +def download(date_array, inst_code=None, kindat=None, data_path=None, + user=None, password=None, url="http://cedar.openmadrigal.org", + file_type='hdf5'): + """Downloads data from Madrigal. + + Parameters + ---------- + date_array : array-like + list of datetimes to download data for. The sequence of dates need not + be contiguous. + inst_code : string + Madrigal instrument code(s), cast as a string. If multiple are used, + separate them with commas. (default=None) + kindat : string + Experiment instrument code(s), cast as a string. If multiple are used, + separate them with commas. (default=None) + data_path : string + Path to directory to download data to. (default=None) + user : string + User string input used for download. Provided by user and passed via + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) + url : string + URL for Madrigal site (default='http://cedar.openmadrigal.org') + file_type : string + File format for Madrigal data. Load routines currently only accepts + 'hdf5' and 'netCDF4', but any of the Madrigal options may be used + here. (default='hdf5') + + Note + ---- + The user's names should be provided in field user. Ruby Payne-Scott should + be entered as Ruby+Payne-Scott + + The password field should be the user's email address. These parameters + are passed to Madrigal when downloading. + + The affiliation field is set to pysat to enable tracking of pysat + downloads. + + """ + + if file_type not in file_types.keys(): + raise ValueError("Unknown file format {:}, accepts {:}".format( + file_type, file_types.keys())) + + _check_madrigal_params(inst_code=inst_code, user=user, password=password) + + if kindat is None: + raise ValueError("Must supply Madrigal experiment code") + + # Get the list of desired remote files + start = date_array.min() + stop = date_array.max() + if start == stop: + stop += dt.timedelta(days=1) + + # Initialize the connection to Madrigal + logger.info('Connecting to Madrigal') + web_data = madrigalWeb.MadrigalData(url) + logger.info('Connection established.') + + files = get_remote_filenames(inst_code=inst_code, kindat=kindat, + user=user, password=password, + web_data=web_data, url=url, + start=start, stop=stop) + + for mad_file in files: + # Build the local filename + local_file = os.path.join(data_path, + os.path.basename(mad_file.name)) + if local_file.find(file_type) <= 0: + split_file = local_file.split(".") + split_file[-1] = file_type + local_file = ".".join(split_file) + + if not os.path.isfile(local_file): + fstr = ''.join(('Downloading data for ', local_file)) + logger.info(fstr) + web_data.downloadFile(mad_file.name, local_file, user, password, + "pysat", format=file_type) + else: + estr = ''.join((local_file, ' already exists. Skipping.')) + logger.info(estr) + + return + + +def get_remote_filenames(inst_code=None, kindat=None, user=None, + password=None, web_data=None, + url="http://cedar.openmadrigal.org", + start=dt.datetime(1900, 1, 1), stop=dt.datetime.now(), + date_array=None): + """Retrieve the remote filenames for a specified Madrigal experiment + + Parameters + ---------- + inst_code : string + Madrigal instrument code(s), cast as a string. If multiple are used, + separate them with commas. (default=None) + kindat : string + Madrigal experiment code(s), cast as a string. If multiple are used, + separate them with commas. If not supplied, all will be returned. + (default=None) + data_path : string + Path to directory to download data to. (default=None) + user : string + User string input used for download. Provided by user and passed via + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) + web_data : MadrigalData + Open connection to Madrigal database or None (will initiate using url) + (default=None) + url : string + URL for Madrigal site (default='http://cedar.openmadrigal.org') + start : dt.datetime + Starting time for file list (defaults to 01-01-1900) + stop : dt.datetime + Ending time for the file list (defaults to time of run) + date_array : dt.datetime + Array of datetimes to download data for. The sequence of dates need not + be contiguous and will be used instead of start and stop if supplied. + (default=None) + + Returns + ------- + files : madrigalWeb.madrigalWeb.MadrigalExperimentFile + Madrigal file object that contains remote experiment file data + + Note + ---- + The user's names should be provided in field user. Ruby Payne-Scott should + be entered as Ruby+Payne-Scott + + The password field should be the user's email address. These parameters + are passed to Madrigal when downloading. + + The affiliation field is set to pysat to enable tracking of pysat + downloads. + + + """ + + _check_madrigal_params(inst_code=inst_code, user=user, password=password) + + if kindat is None: + kindat = [] + else: + kindat = [int(kk) for kk in kindat.split(",")] + + # If date_array supplied, overwrite start and stop + if date_array is not None: + if len(date_array) == 0: + raise ValueError('unknown date_array supplied: {:}'.format( + date_array)) + start = date_array.min() + stop = date_array.max() + # If start and stop are identical, increment + if start == stop: + stop += dt.timedelta(days=1) + # Open connection to Madrigal + if web_data is None: + web_data = madrigalWeb.MadrigalData(url) + + # Get list of experiments for instrument from in desired range + exp_list = web_data.getExperiments(inst_code, start.year, start.month, + start.day, start.hour, start.minute, + start.second, stop.year, stop.month, + stop.day, stop.hour, stop.minute, + stop.second) + + # Iterate over experiments to grab files for each one + files = list() + logger.info("Found {:d} Madrigal experiments".format(len(exp_list))) + for exp in exp_list: + if good_exp(exp, date_array=date_array): + file_list = web_data.getExperimentFiles(exp.id) + + if len(kindat) == 0: + files.extend(file_list) + else: + for file_obj in file_list: + if file_obj.kindat in kindat: + files.append(file_obj) + + return files + + +def good_exp(exp, date_array=None): + """ Determine if a Madrigal experiment has good data for specified dates + + Parameters + ---------- + exp : MadrigalExperimentFile + MadrigalExperimentFile object + date_array : array-like + list of datetimes to download data for. The sequence of dates need not + be contiguous. + + Returns + ------- + gflag : boolean + True if good, False if bad + + """ + + gflag = False + + if exp.id != -1: + if date_array is None: + gflag = True + else: + exp_start = dt.date(exp.startyear, exp.startmonth, + exp.startday) + exp_end = (dt.date(exp.endyear, exp.endmonth, exp.endday) + + dt.timedelta(days=1)) + + for date_val in date_array: + if date_val.date() >= exp_start and date_val.date() <= exp_end: + gflag = True + break + + return gflag + + +def list_remote_files(tag, inst_id, inst_code=None, kindats=None, user=None, + password=None, supported_tags=None, + url="http://cedar.openmadrigal.org", + two_digit_year_break=None, start=dt.datetime(1900, 1, 1), + stop=dt.datetime.utcnow()): + """List files available from Madrigal. + + Parameters + ---------- + tag : string or NoneType + Denotes type of file to load. Accepted types are . + (default=None) + inst_id : string or NoneType + Specifies the satellite ID for a constellation. Not used. + (default=None) + inst_code : string + Madrigal instrument code(s), cast as a string. If multiple are used, + separate them with commas. (default=None) + kindats : dict + Madrigal experiment codes, in a dict of dicts with inst_ids as top level + keys and tags as second level keys with Madrigal experiment code(s) + as values. These should be strings, with multiple codes separated by + commas. (default=None) + data_path : string + Path to directory to download data to. (default=None) + user : string + User string input used for download. Provided by user and passed via + pysat. If an account is required for dowloads this routine here must + error if user not supplied. (default=None) + password : string + Password for data download. (default=None) + supported_tags : dict or NoneType + keys are inst_id, each containing a dict keyed by tag + where the values file format template strings. (default=None) + url : string + URL for Madrigal site (default='http://cedar.openmadrigal.org') + two_digit_year_break : int + If filenames only store two digits for the year, then + '1900' will be added for years >= two_digit_year_break + and '2000' will be added for years < two_digit_year_break. + start : dt.datetime + Starting time for file list (defaults to 01-01-1900) + stop : dt.datetime + Ending time for the file list (defaults to time of run) + + Returns + ------- + pds.Series + A series of filenames, see `pysat.utils.files.process_parsed_filenames` + for more information. + + Raises + ------ + ValueError + For missing kwarg input + KeyError + For dictionary input missing requested tag/inst_id + + Note + ---- + The user's names should be provided in field user. Ruby Payne-Scott should + be entered as Ruby+Payne-Scott + + The password field should be the user's email address. These parameters + are passed to Madrigal when downloading. + + The affiliation field is set to pysat to enable tracking of pysat + downloads. + + Examples + -------- + This method is intended to be set in an instrument support file at the + top level using functools.partial + :: + + list_remote_files = functools.partial(mad_meth.list_remote_files, + supported_tags=supported_tags, + inst_code=madrigal_inst_code, + kindats=madrigal_tag) + + """ + + _check_madrigal_params(inst_code=inst_code, user=user, password=password) + + # Test input + if supported_tags is None or kindats is None: + raise ValueError('Must supply supported_tags and kindats dicts') + + # Raise KeyError if input dictionaries don't match the input + format_str = supported_tags[inst_id][tag] + kindat = kindats[inst_id][tag] + + # Retrieve remote file experiment list + files = get_remote_filenames(inst_code=inst_code, kindat=kindat, user=user, + password=password, url=url, start=start, + stop=stop) + + filenames = [os.path.basename(file_exp.name) for file_exp in files] + + # Parse these filenames to grab out the ones we want + logger.info("Parsing filenames") + stored = pysat.utils.files.parse_fixed_width_filenames(filenames, + format_str) + + # Process the parsed filenames and return a properly formatted Series + logger.info("Processing filenames") + return pysat.utils.files.process_parsed_filenames(stored, + two_digit_year_break) + + +def list_files(tag=None, inst_id=None, data_path=None, format_str=None, + supported_tags=None, file_cadence=dt.timedelta(days=1), + two_digit_year_break=None, delimiter=None, file_type=None): + """Return a Pandas Series of every file for chosen Instrument data. + + Parameters + ---------- + tag : string or NoneType + Denotes type of file to load. Accepted types are . + (default=None) + inst_id : string or NoneType + Specifies the satellite ID for a constellation. Not used. + (default=None) + data_path : string or NoneType + Path to data directory. If None is specified, the value previously + set in Instrument.files.data_path is used. (default=None) + format_str : string or NoneType + User specified file format. If None is specified, the default + formats associated with the supplied tags are used. (default=None) + supported_tags : dict or NoneType + keys are inst_id, each containing a dict keyed by tag + where the values file format template strings. (default=None) + file_cadence : dt.timedelta or pds.DateOffset + pysat assumes a daily file cadence, but some instrument data file + contain longer periods of time. This parameter allows the specification + of regular file cadences greater than or equal to a day (e.g., weekly, + monthly, or yearly). (default=dt.timedelta(days=1)) + two_digit_year_break : int or NoneType + If filenames only store two digits for the year, then '1900' will be + added for years >= two_digit_year_break and '2000' will be added for + years < two_digit_year_break. If None, then four-digit years are + assumed. (default=None) + delimiter : string or NoneType + Delimiter string upon which files will be split (e.g., '.'). If None, + filenames will be parsed presuming a fixed width format. (default=None) + file_type : str or NoneType + File format for Madrigal data. Load routines currently accepts 'hdf5', + 'simple', and 'netCDF4', but any of the Madrigal options may be used + here. If None, will look for all known file types. (default=None) + + Returns + ------- + out : pds.Series + A pandas Series containing the verified available files + + """ + # Initialize the transitional variables + list_file_types = file_types.keys() if file_type is None else [file_type] + sup_tags = {inst_id: {tag: supported_tags[inst_id][tag]}} + out_series = list() + + # Cycle through each requested file type, loading the requested files + for ftype in list_file_types: + if supported_tags[inst_id][tag].find('{file_type}') > 0: + sup_tags[inst_id][tag] = supported_tags[inst_id][tag].format( + file_type=file_types[ftype]) + + out_series.append(pysat.instruments.methods.general.list_files( + tag=tag, inst_id=inst_id, data_path=data_path, + format_str=format_str, supported_tags=sup_tags, + file_cadence=file_cadence, + two_digit_year_break=two_digit_year_break, delimiter=delimiter)) + + # Combine the file lists, ensuring the files are correctly ordered + if len(out_series) == 0: + out = pds.Series(dtype=str) + elif len(out_series) == 1: + out = out_series[0] + else: + out = pds.concat(out_series).sort_index() + + return out + + +def filter_data_single_date(inst): + """Filters data to a single date. + + Parameters + ---------- + inst : pysat.Instrument + Instrument object to which this routine should be attached + + Note + ---- + Madrigal serves multiple days within a single JRO file + to counter this, we will filter each loaded day so that it only + contains the relevant day of data. This is only applied if loading + by date. It is not applied when supplying pysat with a specific + filename to load, nor when data padding is enabled. Note that when + data padding is enabled the final data available within the instrument + will be downselected by pysat to only include the date specified. + + This routine is intended to be added to the Instrument + nanokernel processing queue via + :: + + inst = pysat.Instrument() + inst.custom_attach(filter_data_single_date) + + This function will then be automatically applied to the + Instrument object data on every load by the pysat nanokernel. + + Warnings + -------- + For the best performance, this function should be added first in the queue. + This may be ensured by setting the default function in a pysat instrument + file to this one. + + To do this, within platform_name.py set `preprocess` at the top level. + :: + + preprocess = pysat.instruments.methods.madrigal.filter_data_single_date + + """ + + # Only do this if loading by date! + if inst._load_by_date and inst.pad is None: + # Identify times for the loaded date + idx, = np.where((inst.index >= inst.date) + & (inst.index < (inst.date + pds.DateOffset(days=1)))) + + # Downselect from all data + inst.data = inst[idx] + + return + + +def _check_madrigal_params(inst_code, user, password): + """Checks that parameters requried by Madrigal database are passed through. + + Parameters + ---------- + inst_code : str + Madrigal instrument code(s), cast as a string. If multiple are used, + separate them with commas. + user : str + The user's names should be provided in field user. Ruby Payne-Scott + should be entered as Ruby+Payne-Scott + password : str + The password field should be the user's email address. These parameters + are passed to Madrigal when downloading. + + Raises + ------ + ValueError + Default values of None will raise an error. + + """ + + if inst_code is None: + raise ValueError("Must supply Madrigal instrument code") + + if not (isinstance(user, str) and isinstance(password, str)): + raise ValueError(' '.join(("The madrigal database requries a username", + "and password. Please input these as", + "user='firstname+lastname' and", + "password='myname@email.address' in this", + "function."))) + + return diff --git a/pysatMadrigal/instruments/methods/gnss.py b/pysatMadrigal/instruments/methods/gnss.py new file mode 100644 index 0000000..584ba89 --- /dev/null +++ b/pysatMadrigal/instruments/methods/gnss.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- +# -*- coding: utf-8 -*- +"""Methods supporting the Global Navigation Satellite System platform +""" + + +def acknowledgements(name): + """Provide the acknowledgements for different GNSS instruments + + Parameters + ---------- + name : str + Instrument name + + Returns + ------- + ackn : str + Acknowledgement information to provide in studies using this data + + """ + + ackn = {'tec': ''.join(["GPS TEC data products and access through the ", + "Madrigal distributed data system are provided to ", + "the community by the Massachusetts Institute of ", + "Technology under support from U.S. National ", + "Science Foundation grant AGS-1242204. Data for ", + "the TEC processing is provided by the following ", + "organizations: UNAVCO, Scripps Orbit and ", + "Permanent Array Center, Institut Geographique ", + "National, France, International GNSS Service, The", + " Crustal Dynamics Data Information System ", + "(CDDIS), National Geodetic Survey, Instituto ", + "Brasileiro de Geografiae Estatística, RAMSAC ", + "CORS of Instituto Geográfico Nacional del la ", + "República Agentina, Arecibo Observatory, ", + "Low-Latitude Ionospheric Sensor Network (LISN), ", + "Topcon Positioning Systems, Inc., Canadian High ", + "Arctic Ionospheric Network, Institute of Geology", + " and Geophysics, Chinese Academy of Sciences, ", + "China Meterorology Administration, Centro di ", + "Niveau des Eaux Littorales Ricerche Sismogiche, ", + "Système d’Observation du (SONEL), RENAG : ", + "REseau NAtional GPS permanent, and GeoNet—the ", + "official source of geological hazard information ", + "for New Zealand."])} + + return ackn[name] + + +def references(name, tag): + """Provides suggested references for the specified data set + + Parameters + ---------- + name : str + Instrument name + tag : str + Instrument tag + + Returns + ------- + refs : str + Suggested Instrument reference(s) + + """ + + refs = {'tec': + {'vtec': "Rideout and Coster (2006) doi:10.1007/s10291-006-0029-5"}} + + return refs[name][tag] diff --git a/pysatMadrigal/instruments/methods/jro.py b/pysatMadrigal/instruments/methods/jro.py new file mode 100644 index 0000000..98802df --- /dev/null +++ b/pysatMadrigal/instruments/methods/jro.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3824979 +# ---------------------------------------------------------------------------- +# -*- coding: utf-8 -*- +"""Methods supporting the Jicamarca Radio Observatory (JRO) platform +""" + +import numpy as np + +from pysat import logger + +from pysatMadrigal.utils import coords + + +def acknowledgements(): + """Provides acknowlegements for the JRO instruments and experiments + + Returns + ------- + ackn : str + String providing acknowledgement text for studies using JRO data + + """ + ackn = ' '.join(["The Jicamarca Radio Observatory is a facility of the", + "Instituto Geofisico del Peru operated with support from", + "the NSF AGS-1433968 through Cornell University."]) + + return ackn + + +def references(): + """Provides references for the JRO instruments and experiments + + Returns + ------- + refs : str + String providing reference guidenance for the JRO experiments + + """ + + refs = "Depends on the radar experiment; contact PI" + return refs + + +def calc_measurement_loc(inst): + """ Calculate the instrument measurement location in geographic coordinates + + Parameters + ---------- + inst : pysat.Instrument + JRO ISR Instrument object + + Note + ---- + Adds 'gdlat#', 'gdlon#' to the instrument, for all directions that + have azimuth and elevation keys that match the format 'eldir#' and 'azdir#' + + """ + + az_keys = [kk[5:] for kk in list(inst.data.keys()) + if kk.find('azdir') == 0] + el_keys = [kk[5:] for kk in list(inst.data.keys()) + if kk.find('eldir') == 0] + good_dir = list() + + for i, kk in enumerate(az_keys): + if kk in el_keys: + try: + good_dir.append(int(kk)) + except ValueError: + logger.warning("unknown direction number [{:}]".format(kk)) + + # Calculate the geodetic latitude and longitude for each direction + if len(good_dir) == 0: + raise ValueError("No matching azimuth and elevation data included") + + for dd in good_dir: + # Format the direction location keys + az_key = 'azdir{:d}'.format(dd) + el_key = 'eldir{:d}'.format(dd) + lat_key = 'gdlat{:d}'.format(dd) + lon_key = 'gdlon{:d}'.format(dd) + # JRO is located 520 m above sea level (jro.igp.gob.pe./english/) + # Also, altitude has already been calculated + gdaltr = np.ones(shape=inst['gdlonr'].shape) * 0.52 + gdlat, gdlon, _ = coords.local_horizontal_to_global_geo(inst[az_key], + inst[el_key], + inst['range'], + inst['gdlatr'], + inst['gdlonr'], + gdaltr, + geodetic=True) + + # Assigning as data, to ensure that the number of coordinates match + # the number of data dimensions + inst.data = inst.data.assign({lat_key: gdlat, lon_key: gdlon}) + + # Add metadata for the new data values + bm_label = "Beam {:d} ".format(dd) + inst.meta[lat_key] = {inst.meta.labels.units: 'degrees', + inst.meta.labels.name: bm_label + 'latitude', + inst.meta.labels.desc: bm_label + 'latitude', + inst.meta.labels.min_val: -90.0, + inst.meta.labels.max_val: 90.0, + inst.meta.labels.fill_val: np.nan} + inst.meta[lon_key] = {inst.meta.labels.units: 'degrees', + inst.meta.labels.name: bm_label + 'longitude', + inst.meta.labels.desc: bm_label + 'longitude', + inst.meta.labels.fill_val: np.nan} + + return diff --git a/pysatMadrigal/instruments/methods/madrigal.py b/pysatMadrigal/instruments/methods/madrigal.py deleted file mode 100644 index f812d75..0000000 --- a/pysatMadrigal/instruments/methods/madrigal.py +++ /dev/null @@ -1,549 +0,0 @@ -# -*- coding: utf-8 -*-. -"""Provides default routines for integrating CEDAR Madrigal instruments into -pysat, reducing the amount of user intervention. - - """ - -from __future__ import absolute_import -from __future__ import print_function - -import datetime as dt -import logging -import numpy as np -import os -import pandas as pds -import sys - -import h5py -from madrigalWeb import madrigalWeb - -import pysat - -logger = logging.getLogger(__name__) - - -def cedar_rules(): - """ General acknowledgement statement for Madrigal data. - - Returns - ------- - ackn : string - String with general acknowledgement for all CEDAR Madrigal data - - """ - ackn = "Contact the PI when using this data, in accordance with the CEDAR" - ackn += " 'Rules of the Road'" - return ackn - - -# support load routine -def load(fnames, tag=None, sat_id=None, xarray_coords=[]): - """Loads data from Madrigal into Pandas. - - This routine is called as needed by pysat. It is not intended - for direct user interaction. - - Parameters - ---------- - fnames : array-like - iterable of filename strings, full path, to data files to be loaded. - This input is nominally provided by pysat itself. - tag : string ('') - tag name used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. While - tag defaults to None here, pysat provides '' as the default - tag unless specified by user at Instrument instantiation. - sat_id : string ('') - Satellite ID used to identify particular data set to be loaded. - This input is nominally provided by pysat itself. - xarray_coords : list - List of keywords to use as coordinates if xarray output is desired - instead of a Pandas DataFrame (default=[]) - - Returns - ------- - data : pds.DataFrame or xr.DataSet - A pandas DataFrame or xarray DataSet holding the data from the HDF5 - file - metadata : pysat.Meta - Metadata from the HDF5 file, as well as default values from pysat - - Examples - -------- - :: - inst = pysat.Instrument('jro', 'isr', 'drifts') - inst.load(2010,18) - - """ - - # Ensure 'time' wasn't included as a coordinate, since it is the default - if 'time' in xarray_coords: - xarray_coords.pop(xarray_coords.index('time')) - - # Open the specified file - filed = h5py.File(fnames[0], 'r') - # data - file_data = filed['Data']['Table Layout'] - # metadata - file_meta = filed['Metadata']['Data Parameters'] - # load up what is offered into pysat.Meta - meta = pysat.Meta() - meta.info = {'acknowledgements': - ' '.join(["See 'meta.Experiment_Notes' for instrument", - "specific acknowledgements\n", cedar_rules()]), - 'references': "See 'meta.Experiment_Notes' for references"} - labels = [] - for item in file_meta: - # handle difference in string output between python 2 and 3 - name_string = item[0] - unit_string = item[3] - desc_string = item[1] - if sys.version_info[0] >= 3: - name_string = name_string.decode('UTF-8') - unit_string = unit_string.decode('UTF-8') - desc_string = desc_string.decode('UTF-8') - labels.append(name_string) - meta[name_string.lower()] = {'long_name': name_string, - 'units': unit_string, - 'desc': desc_string} - - # add additional metadata notes - # custom attributes attached to meta are attached to - # corresponding Instrument object when pysat receives - # data and meta from this routine - for key in filed['Metadata']: - if key != 'Data Parameters': - setattr(meta, key.replace(' ', '_'), filed['Metadata'][key][:]) - # data into frame, with labels from metadata - data = pds.DataFrame.from_records(file_data, columns=labels) - # lowercase variable names - data.columns = [item.lower() for item in data.columns] - # datetime index from times - time_keys = np.array(['year', 'month', 'day', 'hour', 'min', 'sec']) - if not np.all([key in data.columns for key in time_keys]): - time_keys = [key for key in time_keys if key not in data.columns] - raise ValueError(' '.join(["unable to construct time index, missing", - "{:}".format(time_keys)])) - - uts = 3600.0 * data.loc[:, 'hour'] + 60.0 * data.loc[:, 'min'] \ - + data.loc[:, 'sec'] - time = pysat.utils.time.create_datetime_index(year=data.loc[:, 'year'], - month=data.loc[:, 'month'], - day=data.loc[:, 'day'], - uts=uts) - # Declare index or recast as xarray - if len(xarray_coords) > 0: - if not np.all([xkey.lower() in data.columns - for xkey in xarray_coords]): - estr = 'unknown coordinate key in {:}, '.format(xarray_coords) - estr += 'use only {:}'.format(data.columns) - raise ValueError(estr) - - # Append time to the data frame and add as the first coordinate - data = data.assign(time=pds.Series(time, index=data.index)) - xarray_coords.insert(0, 'time') - - # Set the indices - data = data.set_index(xarray_coords) - - # Recast the data as an xarray - data = data.to_xarray() - else: - # Set the index to time, and put up a warning if there are duplicate - # times. This could mean the data should be stored as an xarray - # DataSet - data.index = time - - if np.any(time.duplicated()): - logger.warning(' '.join(["duplicated time indices, consider", - "specifing additional coordinates and", - "storing the data as an xarray DataSet"])) - - return data, meta - - -def download(date_array, inst_code=None, kindat=None, data_path=None, - user=None, password=None, url="http://cedar.openmadrigal.org", - file_format='hdf5'): - """Downloads data from Madrigal. - - Parameters - ---------- - date_array : array-like - list of datetimes to download data for. The sequence of dates need not - be contiguous. - inst_code : string (None) - Madrigal instrument code(s), cast as a string. If multiple are used, - separate them with commas. - kindat : string (None) - Experiment instrument code(s), cast as a string. If multiple are used, - separate them with commas. - data_path : string (None) - Path to directory to download data to. - user : string (None) - User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - url : string ('http://cedar.openmadrigal.org') - URL for Madrigal site - file_format : string ('hdf5') - File format for Madrigal data. Load routines currently only accept - 'hdf5', but any of the Madrigal options may be used here. - - Returns - -------- - Void : (NoneType) - Downloads data to disk. - - Notes - ----- - The user's names should be provided in field user. Ruby Payne-Scott should - be entered as Ruby+Payne-Scott - - The password field should be the user's email address. These parameters - are passed to Madrigal when downloading. - - The affiliation field is set to pysat to enable tracking of pysat - downloads. - - """ - - if inst_code is None: - raise ValueError("Must supply Madrigal instrument code") - - if kindat is None: - raise ValueError("Must supply Madrigal experiment code") - - # currently passes things along if no user and password supplied - # need to do this for testing - # TODO, implement user and password values in test code - # specific to each instrument - if user is None: - logger.info('No user information supplied for download.') - user = 'pysat_testing' - if password is None: - logger.info('Please provide email address in password field.') - password = 'pysat_testing@not_real_email.org' - - # Initialize the connection to Madrigal - web_data = madrigalWeb.MadrigalData(url) - - # Get the list of desired remote files - start = date_array.min() - stop = date_array.max() - if start == stop: - stop += dt.timedelta(days=1) - files = get_remote_filenames(inst_code=inst_code, kindat=kindat, user=user, - password=password, web_data=web_data, url=url, - start=start, stop=stop) - - for mad_file in files: - local_file = os.path.join(data_path, os.path.basename(mad_file.name)) - - if not os.path.isfile(local_file): - web_data.downloadFile(mad_file.name, local_file, user, password, - "pysat", format=file_format) - - -def get_remote_filenames(inst_code=None, kindat=None, user=None, - password=None, web_data=None, - url="http://cedar.openmadrigal.org", - start=dt.datetime(1900, 1, 1), stop=dt.datetime.now(), - date_array=None): - """Retrieve the remote filenames for a specified Madrigal instrument - (and experiment) - - Parameters - ---------- - inst_code : string (None) - Madrigal instrument code(s), cast as a string. If multiple are used, - separate them with commas. - kindat : string (None) - Madrigal experiment code(s), cast as a string. If multiple are used, - separate them with commas. If not supplied, all will be returned. - data_path : string (None) - Path to directory to download data to. - user : string (None) - User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - web_data : MadrigalData (None) - Open connection to Madrigal database or None (will initiate using url) - url : string ('http://cedar.openmadrigal.org') - URL for Madrigal site - start : dt.datetime - Starting time for file list (defaults to 01-01-1900) - stop : dt.datetime - Ending time for the file list (defaults to time of run) - date_array : dt.datetime (None) - Array of datetimes to download data for. The sequence of dates need not - be contiguous and will be used instead of start and stop if supplied. - - Returns - ------- - Void : (NoneType) - Downloads data to disk. - - Notes - ----- - The user's names should be provided in field user. Ruby Payne-Scott should - be entered as Ruby+Payne-Scott - - The password field should be the user's email address. These parameters - are passed to Madrigal when downloading. - - The affiliation field is set to pysat to enable tracking of pysat - downloads. - - - """ - - if inst_code is None: - raise ValueError("Must supply Madrigal instrument code") - - if kindat is None: - kindat = [] - else: - kindat = [int(kk) for kk in kindat.split(",")] - - # currently passes things along if no user and password supplied - # need to do this for testing - # TODO, implement user and password values in test code - # specific to each instrument - if user is None: - print('No user information supplied for download.') - user = 'pysat_testing' - if password is None: - print('Please provide email address in password field.') - password = 'pysat_testing@not_real_email.org' - - # If date_array supplied, overwrite start and stop - if date_array is not None: - if len(date_array) == 0: - raise ValueError('unknown date_array supplied: {:}'.format( - date_array)) - start = date_array.min() - stop = date_array.max() - if start == stop: - stop += dt.timedelta(days=1) - - # open connection to Madrigal - if web_data is None: - web_data = madrigalWeb.MadrigalData(url) - - # get list of experiments for instrument from 1900 till now - exp_list = web_data.getExperiments(inst_code, start.year, start.month, - start.day, start.hour, start.minute, - start.second, stop.year, stop.month, - stop.day, stop.hour, stop.minute, - stop.second) - - # iterate over experiments to grab files for each one - files = list() - logger.info("Found {:d} Madrigal experiments".format(len(exp_list))) - for exp in exp_list: - if good_exp(exp, date_array=date_array): - file_list = web_data.getExperimentFiles(exp.id) - - if len(kindat) == 0: - files.extend(file_list) - else: - for file_obj in file_list: - if file_obj.kindat in kindat: - files.append(file_obj) - - return files - - -def good_exp(exp, date_array=None): - """ Determine if a Madrigal experiment has good data for specified dates - - Parameters - ---------- - exp : MadrigalExperimentFile - MadrigalExperimentFile object - date_array : array-like - list of datetimes to download data for. The sequence of dates need not - be contiguous. - - Returns - ------- - gflag : boolean - True if good, False if bad - - """ - - gflag = False - - if exp.id != -1: - if date_array is None: - gflag = True - else: - exp_start = dt.datetime(exp.startyear, exp.startmonth, - exp.startday, exp.starthour, - exp.startmin, exp.startsec) - exp_end = dt.datetime(exp.endyear, exp.endmonth, exp.endday, - exp.endhour, exp.endmin, exp.endsec) - - for date_val in date_array: - if date_val >= exp_start and date_val < exp_end: - gflag = True - break - - return gflag - - -def list_remote_files(tag, sat_id, inst_code=None, kindat=None, user=None, - password=None, supported_tags=None, - url="http://cedar.openmadrigal.org", - two_digit_year_break=None, start=dt.datetime(1900, 1, 1), - stop=dt.datetime.now()): - """List files available from Madrigal. - - Parameters - ---------- - tag : (string or NoneType) - Denotes type of file to load. Accepted types are . - (default=None) - sat_id : (string or NoneType) - Specifies the satellite ID for a constellation. Not used. - (default=None) - inst_code : string (None) - Madrigal instrument code(s), cast as a string. If multiple are used, - separate them with commas. - kindat : string (None) - Madrigal experiment code(s), cast as a string. If multiple are used, - separate them with commas. If not supplied, all will be returned. - data_path : string (None) - Path to directory to download data to. - user : string (None) - User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - supported_tags : (dict or NoneType) - keys are sat_id, each containing a dict keyed by tag - where the values file format template strings. (default=None) - url : string ('http://cedar.openmadrigal.org') - URL for Madrigal site - two_digit_year_break : int - If filenames only store two digits for the year, then - '1900' will be added for years >= two_digit_year_break - and '2000' will be added for years < two_digit_year_break. - start : (dt.datetime) - Starting time for file list (defaults to 01-01-1900) - stop : (dt.datetime) - Ending time for the file list (defaults to time of run) - - Returns - ------- - Void : (NoneType) - Downloads data to disk. - - Notes - ----- - The user's names should be provided in field user. Ruby Payne-Scott should - be entered as Ruby+Payne-Scott - - The password field should be the user's email address. These parameters - are passed to Madrigal when downloading. - - The affiliation field is set to pysat to enable tracking of pysat - downloads. - - Examples - -------- - This method is intended to be set in an instrument support file at the - top level using functools.partial - :: - list_remote_files = functools.partial(mad_meth.list_remote_files, - supported_tags=supported_tags, - inst_code=madrigal_inst_code) - - """ - if inst_code is None: - raise ValueError("Must supply Madrigal instrument code") - - # currently passes things along if no user and password supplied - # need to do this for testing - # TODO, implement user and password values in test code - # specific to each instrument - if user is None: - logger.info('No user information supplied for download.') - user = 'pysat_testing' - if password is None: - logger.info('Please provide email address in password field.') - password = 'pysat_testing@not_real_email.org' - - # Test input - try: - format_str = supported_tags[sat_id][tag] - except KeyError: - raise ValueError('Problem parsing supported_tags') - - # Retrieve remote file list - files = get_remote_filenames(inst_code=inst_code, kindat=kindat, user=user, - password=password, url=url, start=start, - stop=stop) - - # parse these filenames to grab out the ones we want - logger.info("Parsing filenames") - stored = pysat._files.parse_fixed_width_filenames(files, format_str) - - # process the parsed filenames and return a properly formatted Series - logger.info("Processing filenames") - return pysat._files.process_parsed_filenames(stored, two_digit_year_break) - - -def filter_data_single_date(self): - """Filters data to a single date. - - Parameters - ---------- - self : pysat.Instrument - This object - - Note - ---- - Madrigal serves multiple days within a single JRO file - to counter this, we will filter each loaded day so that it only - contains the relevant day of data. This is only applied if loading - by date. It is not applied when supplying pysat with a specific - filename to load, nor when data padding is enabled. Note that when - data padding is enabled the final data available within the instrument - will be downselected by pysat to only include the date specified. - - This routine is intended to be added to the Instrument - nanokernel processing queue via - inst = pysat.Instrument() - inst.custom.attach(filter_data_single_date, 'modify') - This function will then be automatically applied to the - Instrument object data on every load by the pysat nanokernel. - - Warnings - -------- - For the best performance, this function should be added first in the queue. - This may be ensured by setting the default function in a - pysat instrument file to this one. - - within platform_name.py set - default = pysat.instruments.methods.madrigal.filter_data_single_date - at the top level - - """ - - # only do this if loading by date! - if self._load_by_date and self.pad is None: - # identify times for the loaded date - idx, = np.where((self.index >= self.date) - & (self.index < (self.date + pds.DateOffset(days=1)))) - # downselect from all data - self.data = self[idx] diff --git a/pysatMadrigal/instruments/templates/madrigal_pandas.py b/pysatMadrigal/instruments/templates/madrigal_pandas.py index 7fb8141..a24e0a8 100644 --- a/pysatMadrigal/instruments/templates/madrigal_pandas.py +++ b/pysatMadrigal/instruments/templates/madrigal_pandas.py @@ -10,39 +10,41 @@ Warnings -------- - All data downloaded under this general support is placed in the same - directory, pysat_data_dir/madrigal/pandas/. For technical reasons, - the file search algorithm for pysat's Madrigal support is set to permissive - defaults. Thus, all instrument files downloaded via this interface will be - picked up by the madrigal pandas pysat Instrument object unless the - file_format keyword is used at instantiation. +All data downloaded under this general support is placed in the same directory, +pysat_data_dir/madrigal/pandas/. For technical reasons, the file search +algorithm for pysat's Madrigal support is set to permissive defaults. Thus, all +instrument files downloaded via this interface will be picked up by the madrigal +pandas pysat Instrument object unless the file_format keyword is used at +instantiation. - Files can be safely downloaded without knowing the file_format keyword, - or equivalently, how Madrigal names the files. See `Examples` for more. +Files can be safely downloaded without knowing the file_format keyword, +or equivalently, how Madrigal names the files. See `Examples` for more. -Parameters +Properties ---------- -platform : string +platform 'madrigal' -name : string +name 'pandas' -tag : string - '' +tag + madrigal instrument code as an integer +inst_id + madrigal kindat as a string Examples -------- :: - # for isolated use of a madrigal data set + + # For isolated use of a madrigal data set import pysat - # download DMSP data from Madrigal - dmsp = pysat.Instrument('madrigal', 'pandas', - madrigal_code=8100, - madrigal_tag=10241) + # Download DMSP data from Madrigal + dmsp = pysat.Instrument('madrigal', 'pandas', inst_code=8100, + kindat='10241') dmsp.download(dt.datetime(2017, 12, 30), dt.datetime(2017, 12, 31), user='Firstname+Lastname', password='email@address.com') - dmsp.load(2017,363) + dmsp.load(2017, 363) - # for users that plan on using multiple Madrigal datasets + # For users that plan on using multiple Madrigal datasets # using this general interface then an additional parameter # should be supplied upon instrument instantiation (file_format) @@ -63,64 +65,53 @@ # python standards for string templates/Formatters # https://docs.python.org/2/library/string.html - # the complete instantiation for this instrument is - dmsp = pysat.Instrument('madrigal', 'pandas', - madrigal_code=8100, - madrigal_tag=10241, - file_format='dms_ut_{year:4d}{month:02d}{day:02d}_11.002.hdf5') + # The complete instantiation for this instrument is + file_fmt = 'dms_ut_{year:4d}{month:02d}{day:02d}_11.002.hdf5' + dmsp = pysat.Instrument('madrigal', 'pandas', inst_code=8100, + kindat='10241', file_format=file_fmt) Note ---- - Please provide name and email when downloading data with this routine. +Please provide name and email when downloading data with this routine. """ -from __future__ import print_function -from __future__ import absolute_import - -import datetime as dt import functools -from pysat.instruments.methods import madrigal as mad_meth -from pysat.instruments.methods import nasa_cdaweb as cdw +from pysat.instruments.methods import general as ps_gen +from pysat import logger -import logging -logger = logging.getLogger(__name__) +from pysatMadrigal.instruments.methods import general + +# ---------------------------------------------------------------------------- +# Instrument attributes platform = 'madrigal' name = 'pandas' -tags = {'': 'General Madrigal data access loaded into pysat via pandas.'} -sat_ids = {'': list(tags.keys())} -# need to sort out test day setting for unit testing -_test_dates = {'': {'': dt.datetime(2010, 1, 19)}} - -# support list files routine -# use the default CDAWeb method -######### -# need a way to get the filename strings for a particular instrument -# I've put in wildcards for now.... -######### -jro_fname1 = '*{year:4d}{month:02d}{day:02d}' -jro_fname2 = '.{version:03d}.hdf5' -supported_tags = {ss: {'': jro_fname1 + "*" + jro_fname2} - for ss in sat_ids.keys()} -list_files = functools.partial(cdw.list_files, - supported_tags=supported_tags) +tags = {'': 'General pysat Madrigal data access.'} +inst_ids = {'': list(tags.keys())} -# let pysat know that data is spread across more than one file -# multi_file_day=True - -# Set to False to specify using xarray (not using pandas) -# Set to True if data will be returned via a pandas DataFrame pandas_format = True -# support load routine -load = mad_meth.load +# Local attributes +# +# Need a way to get the filename strings for a particular instrument unless +# wildcards start working +fname = '*{year:4d}{month:02d}{day:02d}*.{version:03d}.hdf5' +supported_tags = {ss: {tt: fname for tt in inst_ids[ss]} + for ss in inst_ids.keys()} +remote_tags = {ss: {kk: supported_tags[ss][kk].format(file_type='hdf5') + for kk in inst_ids[ss]} for ss in inst_ids.keys()} + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +# Need to sort out test day setting for unit testing, maybe through a remote +# function +# _test_dates = {'': {'': dt.datetime(2010, 1, 19)}} -# support download routine -# real download attached during init -# however, pysat requires a method before we get there -download = mad_meth.download +# ---------------------------------------------------------------------------- +# Instrument methods def init(self): @@ -133,90 +124,23 @@ def init(self): self : pysat.Instrument This object - Returns - -------- - Void : (NoneType) - Object modified in place. - - """ - logger.info(mad_meth.cedar_rules()) - - code = self.kwargs['madrigal_code'] - tag = self.kwargs['madrigal_tag'] - self._download_rtn = functools.partial(_general_download, - inst_code=str(code), - kindat=str(tag)) - return + logger.info(general.cedar_rules()) + self.acknowledgements = general.cedar_rules() + self.references = 'Please remember to cite the instrument articles.' + self.inst_code = self.kwargs['inst_code'] + self.kindat = self.kwargs['kindat'] -def _general_download(date_array, tag='', sat_id='', data_path=None, user=None, - password=None, inst_code=None, kindat=None): - """Downloads data from Madrigal. - - Method will be partially set using functools.partial. Intended to - have the same call structure as normal instrument download routine. - Upon Instrument instantiation this routine will be set to - parameters specific to a Madrigal data set. It will then work like - a standard download call. - - Parameters - ---------- - date_array : array-like - list of datetimes to download data for. The sequence of dates need not - be contiguous. - tag : string ('') - Tag identifier used for particular dataset. This input is provided by - pysat. - sat_id : string ('') - Satellite ID string identifier used for particular dataset. This input - is provided by pysat. - data_path : string (None) - Path to directory to download data to. - user : string (None) - User string input used for download. Provided by user and passed via - pysat. If an account - is required for dowloads this routine here must error if user not - supplied. - password : string (None) - Password for data download. - inst_code : int - Madrigal integer code used to identify platform - kindat : int - Madrigal integer code used to identify data set - - Returns - -------- - Void : (NoneType) - Downloads data to disk. - - Notes - ----- - The user's names should be provided in field user. Ruby Payne-Scott should - be entered as Ruby+Payne-Scott - - The password field should be the user's email address. These parameters - are passed to Madrigal when downloading. - - The affiliation field is set to pysat to enable tracking of pysat - downloads. - - """ - mad_meth.download(date_array, inst_code=inst_code, kindat=kindat, - data_path=data_path, user=user, password=password) + return def clean(self): """Placeholder routine that would normally return cleaned data - Returns - -------- - Void : (NoneType) - data in inst is modified in-place. - - Notes - -------- + Note + ---- Supports 'clean', 'dusty', 'dirty' in the sense that it prints a message noting there is no cleaning. 'None' is also supported as it signifies no cleaning. @@ -224,7 +148,34 @@ def clean(self): Routine is called by pysat, and not by the end user directly. """ + if self.clean_level in ['clean', 'dusty', 'dirty']: logger.warning('Generalized Madrigal data support has no cleaning.') return + + +# ---------------------------------------------------------------------------- +# Instrument functions +# +# Use the default Madrigal and pysat methods + +# Set the list_remote_files routine +# Need to fix this +# list_remote_files = functools.partial(general.list_remote_files, +# inst_code=self.kwargs['inst_code'], +# kindats=self.kwargs['kindat'], +# supported_tags=remote_tags) + +# Set the load routine +load = general.load + +# Set the list routine +list_files = functools.partial(ps_gen.list_files, + supported_tags=supported_tags) + +# Set up the download routine +# Needs to be fixed +# download = functools.partial(general.download, +# inst_code=str(self.kwargs['inst_code']), +# kindat=self.kwargs['kindat']) diff --git a/pysatMadrigal/tests/test_instruments.py b/pysatMadrigal/tests/test_instruments.py index 428a645..0de7a57 100644 --- a/pysatMadrigal/tests/test_instruments.py +++ b/pysatMadrigal/tests/test_instruments.py @@ -1,38 +1,71 @@ +import tempfile import pytest -import pysatMadrigal -from pysat.tests.instrument_test_class import generate_instrument_list +import pysat +from pysat.utils import generate_instrument_list from pysat.tests.instrument_test_class import InstTestClass -instruments = generate_instrument_list(package=pysatMadrigal.instruments) +import pysatMadrigal + +# Optional code to pass through user and password info to test instruments +# dict, keyed by pysat instrument, with a list of usernames and passwords +# user_info = {'platform_name': {'user': 'pysat_user', +# 'password': 'None'}} +user_info = {module: {'user': 'pysat+CI_tests', + 'password': 'pysat.developers@gmail.com'} + for module in pysatMadrigal.instruments.__all__} +# Developers for instrument libraries should update the following line to +# point to their own subpackage location +# e.g., +# instruments = generate_instrument_list(inst_loc=mypackage.inst) +# If user and password info supplied, use the following instead +# instruments = generate_instrument_list(inst_loc=mypackage.inst, +# user_info=user_info) +instruments = generate_instrument_list(inst_loc=pysatMadrigal.instruments, + user_info=user_info) method_list = [func for func in dir(InstTestClass) if callable(getattr(InstTestClass, func))] + # Search tests for iteration via pytestmark, update instrument list for method in method_list: if hasattr(getattr(InstTestClass, method), 'pytestmark'): # Get list of names of pytestmarks - Nargs = len(getattr(InstTestClass, method).pytestmark) - names = [getattr(InstTestClass, method).pytestmark[j].name - for j in range(0, Nargs)] + mark_name = [mod_mark.name for mod_mark + in getattr(InstTestClass, method).pytestmark] + # Add instruments from your library - if 'all_inst' in names: - mark = pytest.mark.parametrize("name", instruments['names']) + if 'all_inst' in mark_name: + mark = pytest.mark.parametrize("inst_name", instruments['names']) getattr(InstTestClass, method).pytestmark.append(mark) - elif 'download' in names: - mark = pytest.mark.parametrize("inst", instruments['download']) + elif 'download' in mark_name: + mark = pytest.mark.parametrize("inst_dict", + instruments['download']) getattr(InstTestClass, method).pytestmark.append(mark) - elif 'no_download' in names: - mark = pytest.mark.parametrize("inst", instruments['no_download']) + elif 'no_download' in mark_name: + mark = pytest.mark.parametrize("inst_dict", + instruments['no_download']) getattr(InstTestClass, method).pytestmark.append(mark) class TestInstruments(InstTestClass): + def setup_class(self): + """Runs once before the tests to initialize the testing setup + """ + # Make sure to use a temporary directory so that the user's setup is + # not altered + self.tempdir = tempfile.TemporaryDirectory() + self.saved_path = pysat.params['data_dirs'] + pysat.params.data['data_dirs'] = [self.tempdir.name] - def setup(self): - """Runs before every method to create a clean testing setup.""" - self.package = pysatMadrigal.instruments + # Developers for instrument libraries should update the following line + # to point to their own subpackage location, e.g., + # self.inst_loc = mypackage.instruments + self.inst_loc = pysatMadrigal.instruments - def teardown(self): - """Runs after every method to clean up previous testing.""" - del self.package + def teardown_class(self): + """Runs after every method to clean up previous testing + """ + pysat.params.data['data_dirs'] = self.saved_path + self.tempdir.cleanup() + del self.inst_loc, self.saved_path, self.tempdir diff --git a/pysatMadrigal/tests/test_methods_general.py b/pysatMadrigal/tests/test_methods_general.py new file mode 100644 index 0000000..88fde64 --- /dev/null +++ b/pysatMadrigal/tests/test_methods_general.py @@ -0,0 +1,34 @@ +import pytest + +from pysatMadrigal.instruments.methods import general + + +class TestBasic(): + + def setup(self): + """Runs before every method to create a clean testing setup.""" + self.kwargs = {'inst_code': 'inst_code', + 'user': 'username', + 'password': 'password'} + + def cleanup(self): + """Runs after every method to clean up previous testing.""" + del self.kwargs + + @pytest.mark.parametrize("del_val", + ['inst_code', 'user', 'password']) + def test_check_madrigal_params_no_input(self, del_val): + """Test that an error is thrown if None is passed through""" + self.kwargs[del_val] = None + + with pytest.raises(ValueError): + general._check_madrigal_params(**self.kwargs) + + @pytest.mark.parametrize("del_val", + ['user', 'password']) + def test_check_madrigal_params_bad_input(self, del_val): + """Test that an error is thrown if non-string is passed through""" + self.kwargs[del_val] = 17 + + with pytest.raises(ValueError): + general._check_madrigal_params(**self.kwargs) diff --git a/pysatMadrigal/tests/test_utils_coords.py b/pysatMadrigal/tests/test_utils_coords.py new file mode 100644 index 0000000..4b2549e --- /dev/null +++ b/pysatMadrigal/tests/test_utils_coords.py @@ -0,0 +1,304 @@ +""" +tests the pysat coords area +""" +import numpy as np + +import pytest + +from pysatMadrigal.utils import coords + + +class TestGeodeticGeocentric(): + + def setup(self): + """Runs before every method to create a clean testing setup.""" + self.val = {'lat': 45.0, 'lon': 8.0, 'az': 52.0, 'el': 63.0} + self.out = None + self.loc = None + + def teardown(self): + """Runs after every method to clean up previous testing.""" + del self.val, self.out, self.loc + + @pytest.mark.parametrize("kwargs,target", + [({}, [44.8075768, 8.0, 6367.48954386]), + ({'inverse': False}, + [44.8075768, 8.0, 6367.48954386]), + ({'inverse': True}, + [45.1924232, 8.0, 6367.3459085])]) + def test_geodetic_to_geocentric(self, kwargs, target): + """Test conversion from geodetic to geocentric coordinates""" + + self.out = coords.geodetic_to_geocentric(self.val['lat'], + lon_in=self.val['lon'], + **kwargs) + + for i, self.loc in enumerate(self.out): + assert np.all(abs(self.loc - target[i]) < 1.0e-6) + if isinstance(self.loc, np.ndarray): + assert self.loc.shape == self.val['lat'].shape + + def test_geodetic_to_geocentric_and_back(self): + """Tests the reversibility of geodetic to geocentric conversions""" + + self.out = coords.geodetic_to_geocentric(self.val['lat'], + lon_in=self.val['lon'], + inverse=False) + self.loc = coords.geodetic_to_geocentric(self.out[0], + lon_in=self.out[1], + inverse=True) + assert np.all(abs(self.val['lat'] - self.loc[0]) < 1.0e-6) + assert np.all(abs(self.val['lon'] - self.loc[1]) < 1.0e-6) + + @pytest.mark.parametrize("kwargs,target", + [({}, [44.8075768, 8.0, 6367.48954386, + 51.7037677, 62.8811403]), + ({'inverse': False}, + [44.8075768, 8.0, 6367.48954386, + 51.7037677, 62.8811403]), + ({'inverse': True}, + [45.1924232, 8.0, 6367.3459085, + 52.2989610, 63.1180720])]) + def test_geodetic_to_geocentric_horizontal(self, kwargs, target): + """Test conversion from geodetic to geocentric coordinates""" + + self.out = coords.geodetic_to_geocentric_horizontal(self.val['lat'], + self.val['lon'], + self.val['az'], + self.val['el'], + **kwargs) + + for i, self.loc in enumerate(self.out): + assert np.all(abs(self.loc - target[i]) < 1.0e-6) + if isinstance(self.loc, np.ndarray): + assert self.loc.shape == self.val['lat'].shape + + def test_geodetic_to_geocentric_horizontal_and_back(self): + """Tests the reversibility of geodetic to geocentric horiz conversions + + Note: inverse of az and el angles currently non-functional + + """ + + self.out = coords.geodetic_to_geocentric_horizontal(self.val['lat'], + self.val['lon'], + self.val['az'], + self.val['el'], + inverse=False) + self.loc = coords.geodetic_to_geocentric_horizontal(self.out[0], + self.out[1], + self.out[3], + self.out[4], + inverse=True) + + assert np.all(abs(self.val['lat'] - self.loc[0]) < 1.0e-6) + assert np.all(abs(self.val['lon'] - self.loc[1]) < 1.0e-6) + assert np.all(abs(self.val['az'] - self.loc[3]) < 1.0e-6) + assert np.all(abs(self.val['el'] - self.loc[4]) < 1.0e-6) + + +class TestGeodeticGeocentricArray(TestGeodeticGeocentric): + + def setup(self): + """Runs before every method to create a clean testing setup.""" + arr = np.ones(shape=(10,), dtype=float) + self.val = {'lat': 45.0 * arr, + 'lon': 8.0 * arr, + 'az': 52.0 * arr, + 'el': 63.0 * arr} + self.out = None + self.loc = None + + def teardown(self): + """Runs after every method to clean up previous testing.""" + del self.val, self.out, self.loc + + +class TestSphereCartesian(): + + def setup(self): + """Runs before every method to create a clean testing setup.""" + self.val = {'az': 45.0, 'el': 30.0, 'r': 1.0, + 'x': 0.6123724356957946, + 'y': 0.6123724356957946, + 'z': 0.5} + self.out = None + self.loc = None + + def teardown(self): + """Runs after every method to clean up previous testing.""" + del self.val, self.out, self.loc + + @pytest.mark.parametrize("kwargs,input,target", + [({}, ['az', 'el', 'r'], + ['x', 'y', 'z']), + ({'inverse': False}, ['az', 'el', 'r'], + ['x', 'y', 'z']), + ({'inverse': True}, ['x', 'y', 'z'], + ['az', 'el', 'r'])]) + def test_spherical_to_cartesian(self, kwargs, input, target): + """Test conversion from spherical to cartesian coordinates""" + + self.out = coords.spherical_to_cartesian(self.val[input[0]], + self.val[input[1]], + self.val[input[2]], + **kwargs) + + for i, self.loc in enumerate(self.out): + assert np.all(abs(self.loc - self.val[target[i]]) < 1.0e-6) + if isinstance(self.loc, np.ndarray): + assert self.loc.shape == self.val[input[0]].shape + + def test_spherical_to_cartesian_and_back(self): + """Tests the reversibility of spherical to cartesian conversions""" + + self.out = coords.spherical_to_cartesian(self.val['x'], self.val['y'], + self.val['z'], inverse=True) + self.out = coords.spherical_to_cartesian(self.out[0], self.out[1], + self.out[2], inverse=False) + + assert np.all(abs(self.val['x'] - self.out[0]) < 1.0e-6) + assert np.all(abs(self.val['y'] - self.out[1]) < 1.0e-6) + assert np.all(abs(self.val['z'] - self.out[2]) < 1.0e-6) + + +class TestSphereCartesianArray(TestSphereCartesian): + + def setup(self): + """Runs before every method to create a clean testing setup.""" + arr = np.ones(shape=(10,), dtype=float) + self.val = {'az': 45.0 * arr, 'el': 30.0 * arr, 'r': 1.0 * arr, + 'x': 0.6123724356957946 * arr, + 'y': 0.6123724356957946 * arr, + 'z': 0.5 * arr} + self.out = None + self.loc = None + + def teardown(self): + """Runs after every method to clean up previous testing.""" + del self.val, self.out, self.loc + + +class TestGlobalLocal(): + + def setup(self): + """Runs before every method to create a clean testing setup.""" + self.val = {'x': 7000.0, 'y': 8000.0, 'z': 9000.0, + 'lat': 37.5, 'lon': 289.0, 'rad': 6380.0} + self.out = None + self.loc = None + + def teardown(self): + """Runs after every method to clean up previous testing.""" + del self.val, self.out, self.loc + + @pytest.mark.parametrize("kwargs,target", + [({}, + [-9223.1752649, -2239.8352784, 11323.1268511]), + ({'inverse': False}, + [-9223.1752649, -2239.8352784, 11323.1268511]), + ({'inverse': True}, + [-5709.804677, -4918.397556, 15709.5775005])]) + def test_global_to_local_cartesian(self, kwargs, target): + """Test conversion from global to local cartesian coordinates""" + + self.out = coords.global_to_local_cartesian(self.val['x'], + self.val['y'], + self.val['z'], + self.val['lat'], + self.val['lon'], + self.val['rad'], + **kwargs) + + for i, self.loc in enumerate(self.out): + assert np.all(abs(self.loc - target[i]) < 1.0e-6) + if isinstance(self.loc, np.ndarray): + assert self.loc.shape == self.val['x'].shape + + def test_global_to_local_cartesian_and_back(self): + """Tests the reversibility of the global to loc cartesian transform""" + + self.out = coords.global_to_local_cartesian(self.val['x'], + self.val['y'], + self.val['z'], + self.val['lat'], + self.val['lon'], + self.val['rad'], + inverse=False) + self.out = coords.global_to_local_cartesian(self.out[0], self.out[1], + self.out[2], + self.val['lat'], + self.val['lon'], + self.val['rad'], + inverse=True) + assert np.all(abs(self.val['x'] - self.out[0]) < 1.0e-6) + assert np.all(abs(self.val['y'] - self.out[1]) < 1.0e-6) + assert np.all(abs(self.val['z'] - self.out[2]) < 1.0e-6) + + +class TestGlobalLocalArray(TestGlobalLocal): + + def setup(self): + """Runs before every method to create a clean testing setup.""" + arr = np.ones(shape=(10,), dtype=float) + self.val = {'x': 7000.0 * arr, 'y': 8000.0 * arr, 'z': 9000.0 * arr, + 'lat': 37.5 * arr, 'lon': 289.0 * arr, 'rad': 6380.0 * arr} + self.out = None + self.loc = None + + def teardown(self): + """Runs after every method to clean up previous testing.""" + del self.val, self.out, self.loc + + +class TestLocalHorizontalGlobal(): + """Tests for local horizontal to global geo and back """ + + def setup(self): + """Runs before every method to create a clean testing setup.""" + self.val = {'az': 30.0, 'el': 45.0, 'dist': 1000.0, + 'lat': 45.0, 'lon': 0.0, 'alt': 400.0} + self.out = None + self.loc = None + + def teardown(self): + """Runs after every method to clean up previous testing.""" + del self.val, self.out, self.loc + + @pytest.mark.parametrize("kwargs,target", + [({}, [50.4190376, -7.6940084, 7172.1548652]), + ({'geodetic': True}, + [50.4190376, -7.6940084, 7172.1548652]), + ({'geodetic': False}, + [50.4143159, -7.6855552, 7185.6983666])]) + def test_local_horizontal_to_global_geo(self, kwargs, target): + """Tests the conversion of the local horizontal to global geo""" + + self.out = coords.local_horizontal_to_global_geo(self.val['az'], + self.val['el'], + self.val['dist'], + self.val['lat'], + self.val['lon'], + self.val['alt'], + **kwargs) + + for i, self.loc in enumerate(self.out): + assert np.all(abs(self.loc - target[i]) < 1.0e-6) + if isinstance(self.loc, np.ndarray): + assert self.loc.shape == self.val['lat'].shape + + +class TestLocalHorizontalGlobalArray(TestLocalHorizontalGlobal): + """Tests for local horizontal to global geo and back """ + + def setup(self): + """Runs before every method to create a clean testing setup.""" + arr = np.ones(shape=(10,), dtype=float) + self.val = {'az': 30.0 * arr, 'el': 45.0 * arr, 'dist': 1000.0 * arr, + 'lat': 45.0 * arr, 'lon': 0.0 * arr, 'alt': 400.0 * arr} + self.out = None + self.loc = None + + def teardown(self): + """Runs after every method to clean up previous testing.""" + del self.val, self.out, self.loc diff --git a/pysatMadrigal/utils/__init__.py b/pysatMadrigal/utils/__init__.py new file mode 100644 index 0000000..0854b8d --- /dev/null +++ b/pysatMadrigal/utils/__init__.py @@ -0,0 +1 @@ +from pysatMadrigal.utils import coords # noqa F401 diff --git a/pysatMadrigal/utils/coords.py b/pysatMadrigal/utils/coords.py new file mode 100644 index 0000000..b2488e6 --- /dev/null +++ b/pysatMadrigal/utils/coords.py @@ -0,0 +1,349 @@ +""" +Coordinate transformation functions + +""" + +import numpy as np + + +def geodetic_to_geocentric(lat_in, lon_in=None, inverse=False): + """Converts position from geodetic to geocentric or vice-versa. + + Parameters + ---------- + lat_in : float + latitude in degrees. + lon_in : float or NoneType + longitude in degrees. Remains unchanged, so does not need to be + included. (default=None) + inverse : bool + False for geodetic to geocentric, True for geocentric to geodetic. + (default=False) + + Returns + ------- + lat_out : float + latitude [degree] (geocentric/detic if inverse=False/True) + lon_out : float or NoneType + longitude [degree] (geocentric/detic if inverse=False/True) + rad_earth : float + Earth radius [km] (geocentric/detic if inverse=False/True) + + Note + ----- + Uses WGS-84 values + + References + ---------- + Based on J.M. Ruohoniemi's geopack and R.J. Barnes radar.pro + + """ + rad_eq = 6378.1370 # WGS-84 semi-major axis + flat = 1.0 / 298.257223563 # WGS-84 flattening + rad_pol = rad_eq * (1.0 - flat) # WGS-84 semi-minor axis + + # The ratio between the semi-major and minor axis is used several times + rad_ratio_sq = (rad_eq / rad_pol)**2 + + # Calculate the square of the second eccentricity (e') + eprime_sq = rad_ratio_sq - 1.0 + + # Calculate the tangent of the input latitude + tan_in = np.tan(np.radians(lat_in)) + + # If converting from geodetic to geocentric, take the inverse of the + # radius ratio + if not inverse: + rad_ratio_sq = 1.0 / rad_ratio_sq + + # Calculate the output latitude + lat_out = np.degrees(np.arctan(rad_ratio_sq * tan_in)) + + # Calculate the Earth radius at this latitude + rad_earth = rad_eq / np.sqrt(1.0 + eprime_sq + * np.sin(np.radians(lat_out))**2) + + # longitude remains unchanged + lon_out = lon_in + + return lat_out, lon_out, rad_earth + + +def geodetic_to_geocentric_horizontal(lat_in, lon_in, az_in, el_in, + inverse=False): + """Converts from local horizontal coordinates in a geodetic system to local + horizontal coordinates in a geocentric system + + Parameters + ---------- + lat_in : float + latitude in degrees of the local horizontal coordinate system center + lon_in : float + longitude in degrees of the local horizontal coordinate system center + az_in : float + azimuth in degrees within the local horizontal coordinate system + el_in : float + elevation in degrees within the local horizontal coordinate system + inverse : bool + False for geodetic to geocentric, True for inverse (default=False) + + Returns + ------- + lat_out : float + latitude in degrees of the converted horizontal coordinate system + center + lon_out : float + longitude in degrees of the converted horizontal coordinate system + center + rad_earth : float + Earth radius in km at the geocentric/detic (False/True) location + az_out : float + azimuth in degrees of the converted horizontal coordinate system + el_out : float + elevation in degrees of the converted horizontal coordinate system + + References + ---------- + Based on J.M. Ruohoniemi's geopack and R.J. Barnes radar.pro + + """ + + az = np.radians(az_in) + el = np.radians(el_in) + + # Transform the location of the local horizontal coordinate system center + lat_out, lon_out, rad_earth = geodetic_to_geocentric(lat_in, lon_in, + inverse=inverse) + + # Calcualte the deviation from vertical in radians + dev_vert = np.radians(lat_in - lat_out) + + # Calculate cartesian coordinated in local system + x_local = np.cos(el) * np.sin(az) + y_local = np.cos(el) * np.cos(az) + z_local = np.sin(el) + + # Now rotate system about the x axis to align local vertical vector + # with Earth radial vector + x_out = x_local + y_out = y_local * np.cos(dev_vert) + z_local * np.sin(dev_vert) + z_out = -y_local * np.sin(dev_vert) + z_local * np.cos(dev_vert) + + # Transform the azimuth and elevation angles + az_out = np.degrees(np.arctan2(x_out, y_out)) + el_out = np.degrees(np.arctan(z_out / np.sqrt(x_out**2 + y_out**2))) + + return lat_out, lon_out, rad_earth, az_out, el_out + + +def spherical_to_cartesian(az_in, el_in, r_in, inverse=False): + """Convert a position from spherical to cartesian, or vice-versa + + Parameters + ---------- + az_in : float + azimuth/longitude in degrees or cartesian x in km (inverse=False/True) + el_in : float + elevation/latitude in degrees or cartesian y in km (inverse=False/True) + r_in : float + distance from origin in km or cartesian z in km (inverse=False/True) + inverse : boolian + False to go from spherical to cartesian and True for the inverse + + Returns + ------- + x_out : float + cartesian x in km or azimuth/longitude in degrees (inverse=False/True) + y_out : float + cartesian y in km or elevation/latitude in degrees (inverse=False/True) + z_out : float + cartesian z in km or distance from origin in km (inverse=False/True) + + Note + ----- + This transform is the same for local or global spherical/cartesian + transformations. + + Returns elevation angle (angle from the xy plane) rather than zenith angle + (angle from the z-axis) + + """ + + if inverse: + # Cartesian to Spherical + xy_sq = az_in**2 + el_in**2 + z_out = np.sqrt(xy_sq + r_in**2) # This is r + y_out = np.degrees(np.arctan2(np.sqrt(xy_sq), r_in)) # This is zenith + y_out = 90.0 - y_out # This is the elevation + x_out = np.degrees(np.arctan2(el_in, az_in)) # This is azimuth + else: + # Spherical coordinate system uses zenith angle (degrees from the + # z-axis) and not the elevation angle (degrees from the x-y plane) + zen_in = np.radians(90.0 - el_in) + + # Spherical to Cartesian + x_out = r_in * np.sin(zen_in) * np.cos(np.radians(az_in)) + y_out = r_in * np.sin(zen_in) * np.sin(np.radians(az_in)) + z_out = r_in * np.cos(zen_in) + + return x_out, y_out, z_out + + +def global_to_local_cartesian(x_in, y_in, z_in, lat_cent, lon_cent, rad_cent, + inverse=False): + """Converts a position from global to local cartesian or vice-versa + + Parameters + ---------- + x_in : float + global or local cartesian x in km (inverse=False/True) + y_in : float + global or local cartesian y in km (inverse=False/True) + z_in : float + global or local cartesian z in km (inverse=False/True) + lat_cent : float + geocentric latitude in degrees of local cartesian system origin + lon_cent : float + geocentric longitude in degrees of local cartesian system origin + rad_cent : float + distance from center of the Earth in km of local cartesian system + origin + inverse : bool + False to convert from global to local cartesian coodiantes, and True + for the inverse (default=False) + + Returns + ------- + x_out : float + local or global cartesian x in km (inverse=False/True) + y_out : float + local or global cartesian y in km (inverse=False/True) + z_out : float + local or global cartesian z in km (inverse=False/True) + + Note + ----- + The global cartesian coordinate system has its origin at the center of the + Earth, while the local system has its origin specified by the input + latitude, longitude, and radius. The global system has x intersecting + the equatorial plane and the prime meridian, z pointing North along the + rotational axis, and y completing the right-handed coodinate system. + The local system has z pointing up, y pointing North, and x pointing East. + + """ + + # Get the global cartesian coordinates of local origin + x_cent, y_cent, z_cent = spherical_to_cartesian(lon_cent, lat_cent, + rad_cent) + + # Get the amount of rotation needed to align the x-axis with the + # Earth's rotational axis + ax_rot = np.radians(90.0 - lat_cent) + + # Get the amount of rotation needed to align the global x-axis with the + # prime meridian + mer_rot = np.radians(lon_cent - 90.0) + + if inverse: + # Rotate about the x-axis to align the z-axis with the Earth's + # rotational axis + xrot = x_in + yrot = y_in * np.cos(ax_rot) - z_in * np.sin(ax_rot) + zrot = y_in * np.sin(ax_rot) + z_in * np.cos(ax_rot) + + # Rotate about the global z-axis to get the global x-axis aligned + # with the prime meridian and translate the local center to the + # global origin + x_out = xrot * np.cos(mer_rot) - yrot * np.sin(mer_rot) + x_cent + y_out = xrot * np.sin(mer_rot) + yrot * np.cos(mer_rot) + y_cent + z_out = zrot + z_cent + else: + # Translate global origin to the local origin + xtrans = x_in - x_cent + ytrans = y_in - y_cent + ztrans = z_in - z_cent + + # Rotate about the global z-axis to get the local x-axis pointing East + xrot = xtrans * np.cos(-mer_rot) - ytrans * np.sin(-mer_rot) + yrot = xtrans * np.sin(-mer_rot) + ytrans * np.cos(-mer_rot) + zrot = ztrans + + # Rotate about the x-axis to get the z-axis pointing up + x_out = xrot + y_out = yrot * np.cos(-ax_rot) - zrot * np.sin(-ax_rot) + z_out = yrot * np.sin(-ax_rot) + zrot * np.cos(-ax_rot) + + return x_out, y_out, z_out + + +def local_horizontal_to_global_geo(az, el, dist, lat_orig, lon_orig, alt_orig, + geodetic=True): + """ Convert from local horizontal coordinates to geodetic or geocentric + coordinates + + Parameters + ---------- + az : float + Azimuth (angle from North) of point in degrees + el : float + Elevation (angle from ground) of point in degrees + dist : float + Distance from origin to point in km + lat_orig : float + Latitude of origin in degrees + lon_orig : float + Longitude of origin in degrees + alt_orig : float + Altitude of origin in km from the surface of the Earth + geodetic : bool + True if origin coordinates are geodetic, False if they are geocentric. + Will return coordinates in the same system as the origin input. + (default=True) + + Returns + ------- + lat_pnt : float + Latitude of point in degrees + lon_pnt : float + Longitude of point in degrees + rad_pnt : float + Distance to the point from the centre of the Earth in km + + References + ---------- + Based on J.M. Ruohoniemi's geopack and R.J. Barnes radar.pro + + """ + + # If the data are in geodetic coordiantes, convert to geocentric + if geodetic: + (glat, glon, rearth, gaz, gel) = \ + geodetic_to_geocentric_horizontal(lat_orig, lon_orig, az, el, + inverse=False) + grad = rearth + alt_orig + else: + glat = lat_orig + glon = lon_orig + grad = alt_orig + 6371.0 # Add the mean earth radius in km + gaz = az + gel = el + + # Convert from local horizontal to local cartesian coordiantes + x_loc, y_loc, z_loc = spherical_to_cartesian(gaz, gel, dist, inverse=False) + + # Convert from local to global cartesian coordiantes + x_glob, y_glob, z_glob = global_to_local_cartesian(x_loc, y_loc, z_loc, + glat, glon, grad, + inverse=True) + + # Convert from global cartesian to geocentric coordinates + lon_pnt, lat_pnt, rad_pnt = spherical_to_cartesian(x_glob, y_glob, z_glob, + inverse=True) + + # Convert from geocentric to geodetic, if desired + if geodetic: + lat_pnt, lon_pnt, rearth = geodetic_to_geocentric(lat_pnt, lon_pnt, + inverse=True) + rad_pnt = rearth + rad_pnt - 6371.0 + + return lat_pnt, lon_pnt, rad_pnt diff --git a/pysatMadrigal/version.txt b/pysatMadrigal/version.txt index bcab45a..81340c7 100644 --- a/pysatMadrigal/version.txt +++ b/pysatMadrigal/version.txt @@ -1 +1 @@ -0.0.3 +0.0.4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..27c3023 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +h5py +madrigalWeb +numpy +pandas +pysat>=3.0 +xarray diff --git a/setup.cfg b/setup.cfg index 22d9e74..1fbaebd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,60 @@ +[metadata] +name = pysatMadrigal +version = file: pysatMadrigal/version.txt +url = https://github.com/pysat/pysatMadrigal +author = Angeline G. Burrell, et al. +author_email = pysat.developers@gmail.com +description = 'Madrigal instrument support for the pysat ecosystem' +keywords = + pysat + ionosphere + Madrigal + CEDAR + thermosphere + GPS + GNSS + TEC + Jicamarca + DMSP + ISR + Incoherent scatter radar +classifiers = + Development Status :: 3 - Alpha + Topic :: Scientific/Engineering :: Physics + Topic :: Scientific/Engineering :: Atmospheric Science + Intended Audience :: Science/Research + License :: OSI Approved :: BSD License + Natural Language :: English + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Operating System :: MacOS :: MacOS X + Operating System :: POSIX :: Linux +license_file = LICENSE +long_description = file: README.md +long_description_content_type = text/markdown + +[options] +python_requires = >= 3.5 +setup_requires = setuptools >= 38.6; pip >= 10 +include_package_data = True +zip_safe = False +packages = find: +install_requires = h5py + madrigalWeb + numpy + pandas + pysat + xarray + +[coverage:report] +omit = */instruments/templates/ + +[flake8] +max-line-length = 80 +ignore = W503 + [tool:pytest] markers = all_inst: tests all instruments @@ -5,6 +62,4 @@ markers = no_download: tests for instruments without download support first: first tests to run second: second tests to run -flake8-ignore = - *.py W503 - docs/conf.py ALL + diff --git a/setup.py b/setup.py index 75a7f7f..490d94a 100644 --- a/setup.py +++ b/setup.py @@ -4,49 +4,9 @@ # Full license can be found in License.md and AUTHORS.md # ----------------------------------------------------------------------------- -import codecs -import os -from setuptools import setup, find_packages +from setuptools import setup -here = os.path.abspath(os.path.dirname(__file__)) -with codecs.open(os.path.join(here, 'description.txt'), encoding='utf-8') as f: - long_description = f.read() -version_filename = os.path.join('pysatMadrigal', 'version.txt') -with codecs.open(os.path.join(here, version_filename)) as version_file: - version = version_file.read().strip() - -# Define requirements -install_requires = ['pysat', 'pandas', 'xarray', 'numpy'] -# packages with Fortran code -fortran_install = ['madrigalWeb', 'h5py'] -# flag, True if on readthedocs -on_rtd = os.environ.get('READTHEDOCS') == 'True' -# include Fortran for normal install -# read the docs doesn't do Fortran -if not on_rtd: - # not on ReadTheDocs, add Fortran - install_requires.extend(fortran_install) - - -# Run setup -setup(name='pysatMadrigal', - version=version, - url='http://github.com/pysat/pysatMadrigal', - author='Angeline G. Burrell, Russell Stoneback, Jeff Klenzing', - author_email='angeline.burrell@nrl.navy.mil', - description='Madrigal instrument support for the pysat ecosystem', - long_description=long_description, - packages=find_packages(), - classifiers=["Development Status :: 4 - Beta", - "Topic :: Scientific/Engineering :: Physics", - "Intended Audience :: Science/Research", - 'License :: OSI Approved :: BSD License', - "Natural Language :: English", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Operating System :: MacOS :: MacOS X"], - include_package_data=True, - zip_safe=False, - install_requires=install_requires,) +# Run setup. Setuptools will look for parameters in [metadata] section of +# setup.cfg +setup() diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 0000000..03aca35 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,8 @@ +coveralls +flake8 +m2r2 +numpydoc +pytest-cov +pytest-ordering +sphinx +sphinx_rtd_theme