Merge pull request #33 from pysat/develop

Develop update

.travis.yml

@@ -1,10 +1,14 @@
 language: python
 dist: xenial
-matrix:
+jobs:
   include:
-    - python: 3.6
-    - python: 3.7
-    - python: 3.8
+    - python: '3.6'
+      script: pytest --cov=pysatMadrigal/
+    - name: '3.7 with flake8'
+      python: '3.7'
+      script: pytest --flake8 --cov=pysatMadrigal/
+    - python: '3.8'
+      script: pytest --cov=pysatMadrigal/
 
 services: xvfb
 cache: pip
@@ -28,17 +32,21 @@ install:
   # Useful for debugging any issues with conda
   - conda info -a
   # Create conda test environment
-  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION numpy scipy pandas xarray requests beautifulsoup4 lxml netCDF4 h5py pytest-cov pytest-ordering coveralls future
+  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION
   - conda activate test-environment
-  # Dependencies not available through conda, install through pip
-  - pip install pytest-flake8
+  # Once pysat 3.0 is released, remove the custom pysat install and change
+  # this to:
+  # - while read requirement; do conda install --yes $requirement; done < requirements.txt
+  - conda install --yes h5py numpy pandas xarray
   - pip install madrigalWeb
-  - pip install PyForecastTools
-  - pip install pysatCDF >/dev/null
+  # Get latest coveralls from pip, not conda
+  - pip install -r test_requirements.txt
   # Custom pysat install
   - cd ..
   - git clone --single-branch --branch develop-3 https://github.com/pysat/pysat.git
   - cd pysat
+  # Install the pysat requirements
+  - while read requirement; do conda install --yes $requirement; done < requirements.txt
   - python setup.py install
   - export PYTHONPATH=$PYTHONPATH:$(pwd)
   # set up data directory
@@ -47,9 +55,5 @@ install:
   # install pysatMadrigal
   - python setup.py install
 
-# command to run tests
-script:
- - pytest -vs --cov=pysatMadrigal/ --flake8
-
 after_success:
  - coveralls --rcfile=setup.cfg

pysatMadrigal/instruments/methods/madrigal.py

@@ -333,23 +333,11 @@ def download(date_array, inst_code=None, kindat=None, data_path=None,
         raise ValueError("Unknown file format {:}, accepts {:}".format(
             file_type, file_types))
 
-    if inst_code is None:
-        raise ValueError("Must supply Madrigal instrument code")
+    _check_madrigal_params(inst_code=inst_code, user=user, password=password)
 
     if kindat is None:
         raise ValueError("Must supply Madrigal experiment code")
 
-    # currently passes things along if no user and password supplied
-    # need to do this for testing
-    # TODO, implement user and password values in test code
-    # specific to each instrument
-    if user is None:
-        logger.info('No user information supplied for download.')
-        user = 'pysat_testing'
-    if password is None:
-        logger.info('Please provide email address in password field.')
-        password = 'pysat_testing@not_real_email.org'
-
     # Initialize the connection to Madrigal
     web_data = madrigalWeb.MadrigalData(url)
 
@@ -430,25 +418,13 @@ def get_remote_filenames(inst_code=None, kindat=None, user=None,
 
     """
 
-    if inst_code is None:
-        raise ValueError("Must supply Madrigal instrument code")
+    _check_madrigal_params(inst_code=inst_code, user=user, password=password)
 
     if kindat is None:
         kindat = []
     else:
         kindat = [int(kk) for kk in kindat.split(",")]
 
-    # currently passes things along if no user and password supplied
-    # need to do this for testing
-    # TODO, implement user and password values in test code
-    # specific to each instrument
-    if user is None:
-        logger.warning('No user information supplied for download.')
-        user = 'pysat_testing'
-    if password is None:
-        logger.warning('Please provide email address in password field.')
-        password = 'pysat_testing@not_real_email.org'
-
     # If date_array supplied, overwrite start and stop
     if date_array is not None:
         if len(date_array) == 0:
@@ -591,19 +567,8 @@ def list_remote_files(tag, inst_id, inst_code=None, kindat=None, user=None,
                                               inst_code=madrigal_inst_code)
 
     """
-    if inst_code is None:
-        raise ValueError("Must supply Madrigal instrument code")
 
-    # currently passes things along if no user and password supplied
-    # need to do this for testing
-    # TODO, implement user and password values in test code
-    # specific to each instrument
-    if user is None:
-        logger.info('No user information supplied for download.')
-        user = 'pysat_testing'
-    if password is None:
-        logger.info('Please provide email address in password field.')
-        password = 'pysat_testing@not_real_email.org'
+    _check_madrigal_params(inst_code=inst_code, user=user, password=password)
 
     # Test input
     try:
@@ -678,3 +643,29 @@ def filter_data_single_date(inst):
         inst.data = inst[idx]
 
     return
+
+
+def _check_madrigal_params(inst_code, user, password):
+    """Checks that parameters requried by Madrigal database are passed through.
+    Default values of None will raise an error.
+
+    inst_code : str
+        Madrigal instrument code(s), cast as a string.  If multiple are used,
+        separate them with commas.
+    user : str
+        The user's names should be provided in field user. Ruby Payne-Scott
+        should be entered as Ruby+Payne-Scott
+    password : str
+        The password field should be the user's email address. These parameters
+            are passed to Madrigal when downloading.
+    """
+
+    if inst_code is None:
+        raise ValueError("Must supply Madrigal instrument code")
+
+    if not (isinstance(user, str) and isinstance(password, str)):
+        raise ValueError(' '.join(("The madrigal database requries a username",
+                                   "and password.  Please input these as",
+                                   "user='firstname+lastname' and",
+                                   "password='myname@email.address' in this",
+                                   "function.")))

pysatMadrigal/tests/test_general_methods.py

@@ -0,0 +1,33 @@
+import pytest
+from pysatMadrigal.instruments.methods import madrigal as pm_meth
+
+
+class TestBasic():
+
+    def setup(self):
+        """Runs before every method to create a clean testing setup."""
+        self.kwargs = {'inst_code': 'inst_code',
+                       'user': 'username',
+                       'password': 'password'}
+
+    def cleanup(self):
+        """Runs after every method to clean up previous testing."""
+        del self.kwargs
+
+    @pytest.mark.parametrize("del_val",
+                             ['inst_code', 'user', 'password'])
+    def test_check_madrigal_params_no_input(self, del_val):
+        """Test that an error is thrown if None is passed through"""
+        self.kwargs[del_val] = None
+
+        with pytest.raises(ValueError):
+            pm_meth._check_madrigal_params(**self.kwargs)
+
+    @pytest.mark.parametrize("del_val",
+                             ['user', 'password'])
+    def test_check_madrigal_params_bad_input(self, del_val):
+        """Test that an error is thrown if non-string is passed through"""
+        self.kwargs[del_val] = 17
+
+        with pytest.raises(ValueError):
+            pm_meth._check_madrigal_params(**self.kwargs)

pysatMadrigal/tests/test_instruments.py

@@ -7,7 +7,23 @@
 
 import pysatMadrigal
 
-instruments = generate_instrument_list(inst_loc=pysatMadrigal.instruments)
+# Optional code to pass through user and password info to test instruments
+# dict, keyed by pysat instrument, with a list of usernames and passwords
+# user_info = {'platform_name': {'user': 'pysat_user',
+#                                'password': 'None'}}
+user_info = {module: {'user': 'pysat+CI_tests',
+                      'password': 'pysat.developers@gmail.com'}
+             for module in pysatMadrigal.instruments.__all__}
+
+# Developers for instrument libraries should update the following line to
+# point to their own subpackage location
+# e.g.,
+# instruments = generate_instrument_list(inst_loc=mypackage.inst)
+# If user and password info supplied, use the following instead
+# instruments = generate_instrument_list(inst_loc=mypackage.inst,
+#                                        user_info=user_info)
+instruments = generate_instrument_list(inst_loc=pysatMadrigal.instruments,
+                                       user_info=user_info)
 method_list = [func for func in dir(InstTestClass)
                if callable(getattr(InstTestClass, func))]
 

requirements.txt

@@ -0,0 +1,6 @@
+h5py
+madrigalWeb
+numpy
+pandas
+pysat>=3.0
+xarray

setup.cfg

@@ -1,3 +1,50 @@
+[metadata]
+name = pysatMadrigal
+version = file: pysatMadrigal/version.txt
+url = https://github.com/pysat/pysatMadrigal
+author = Angeline G. Burrell, et al.
+author_email = pysat.developers@gmail.com
+description = 'Madrigal instrument support for the pysat ecosystem'
+keywords =
+  pysat
+  ionosphere
+  Madrigal
+  CEDAR
+  thermosphere
+  GPS
+  GNSS
+  TEC
+  Jicamarca
+  DMSP
+  ISR
+  Incoherent scatter radar
+classifiers =
+  Development Status :: 4 - Beta
+  Topic :: Scientific/Engineering :: Physics
+  Topic :: Scientific/Engineering :: Atmospheric Science
+  Intended Audience :: Science/Research
+  License :: OSI Approved :: BSD License
+  Natural Language :: English
+  Programming Language :: Python :: 3 :: Only
+  Programming Language :: Python :: 3.6
+  Programming Language :: Python :: 3.7
+  Programming Language :: Python :: 3.8
+  Operating System :: MacOS :: MacOS X
+  Operating System :: POSIX :: Linux
+license_file = LICENSE
+long_description = file: README.md
+long_description_content_type = text/markdown
+
+  [options]
+  python_requires = >= 3.5
+  setup_requires =
+    setuptools >= 38.6
+    pip >= 10
+  include_package_data = True
+  zip_safe = False
+  packages = find:
+  install_requires = file: requirements.txt
+
 [coverage:report]
 omit =
   */instruments/templates/

setup.py

@@ -4,54 +4,9 @@
 # Full license can be found in License.md and AUTHORS.md
 # -----------------------------------------------------------------------------
 
-import codecs
-import os
-from setuptools import setup, find_packages
+from setuptools import setup
 
 
-here = os.path.abspath(os.path.dirname(__file__))
-with codecs.open(os.path.join(here, 'description.txt'), encoding='utf-8') as f:
-    long_description = f.read()
-version_filename = os.path.join('pysatMadrigal', 'version.txt')
-with codecs.open(os.path.join(here, version_filename)) as version_file:
-    version = version_file.read().strip()
-
-# Define requirements
-# netCDF support requires netCDF4-Python or scipy installation
-install_requires = ['pysat', 'pandas', 'xarray', 'numpy', 'netCDF4']
-
-# packages with Fortran code
-fortran_install = ['madrigalWeb', 'h5py']
-
-# flag, True if on readthedocs
-on_rtd = os.environ.get('READTHEDOCS') == 'True'
-
-# include Fortran for normal install
-# read the docs doesn't do Fortran
-if not on_rtd:
-    # not on ReadTheDocs, add Fortran
-    install_requires.extend(fortran_install)
-
-
-# Run setup
-setup(name='pysatMadrigal',
-      version=version,
-      url='http://github.com/pysat/pysatMadrigal',
-      author='Angeline G. Burrell, Russell Stoneback, Jeff Klenzing',
-      author_email='angeline.burrell@nrl.navy.mil',
-      description='Madrigal instrument support for the pysat ecosystem',
-      long_description=long_description,
-      packages=find_packages(),
-      classifiers=["Development Status :: 4 - Beta",
-                   "Topic :: Scientific/Engineering :: Physics",
-                   "Intended Audience :: Science/Research",
-                   'License :: OSI Approved :: BSD License',
-                   "Natural Language :: English",
-                   "Programming Language :: Python :: 3 :: Only",
-                   "Programming Language :: Python :: 3.6",
-                   "Programming Language :: Python :: 3.7",
-                   "Programming Language :: Python :: 3.8",
-                   "Operating System :: MacOS :: MacOS X"],
-      include_package_data=True,
-      zip_safe=False,
-      install_requires=install_requires,)
+# Run setup.  Setuptools will look for parameters in [metadata] section of
+# setup.cfg
+setup()

test_requirements.txt

@@ -0,0 +1,8 @@
+coveralls
+m2r2
+numpydoc
+pytest-cov
+pytest-ordering
+pytest-flake8
+sphinx
+sphinx_rtd_theme