Merge branch 'master' into sync

.coveragerc

@@ -0,0 +1,13 @@
+[run]
+branch = True
+
+[report]
+exclude_lines =
+    pragma: no cover
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if __name__ == .__main__.:

.gitignore

@@ -2,3 +2,6 @@
 *.csv
 __pycache__
 db.h5
+htmlcov
+build
+

.travis.yml

@@ -0,0 +1,21 @@
+language: python
+dist: xenial
+python: 3.7
+
+cache:
+  - pip
+  - directories:
+    - $HOME/miniconda/pkgs
+
+install:
+  - sudo apt-get update
+  - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+  - bash miniconda.sh -b -f -p $HOME/miniconda
+  - export PATH=":./bin:$HOME/miniconda/bin:$PATH"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  - conda env create -f environment.yml
+  - source activate mpr
+
+script: tests && coveralls

README.md

@@ -1,7 +1,11 @@
 mpr
 ===
 
-Repository for downloading, parsing, and analyzing [ Mandatory Price Reporting](https://mpr.datamart.ams.usda.gov/) from the USDA's [Agricultural Marketing Service](https://www.ams.usda.gov/). 
+Python package for downloading, parsing, and analyzing historical [ Mandatory Price Reporting](https://mpr.datamart.ams.usda.gov/) data from the USDA's [Agricultural Marketing Service](https://www.ams.usda.gov/).
+
+[![Build Status](https://travis-ci.com/gumballhead/mpr.svg?branch=master)](https://travis-ci.com/gumballhead/mpr)
+[![Coverage Status](https://coveralls.io/repos/github/gumballhead/mpr/badge.svg?branch=master)](https://coveralls.io/github/gumballhead/mpr?branch=master)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/gumballhead/mpr/master)
 
 ### Setup
 This project uses [conda](https://conda.io/en/latest/). Install python packages and activate environment:
@@ -15,12 +19,12 @@ It's a good idea to use something like [direnv](https://direnv.net/) to activate
 My `.envrc` file looks like this:
 ```bash
 export PATH=$PATH:./bin
-source ~/miniconda3/etc/profile.d/conda.sh
+source $HOME/miniconda3/etc/profile.d/conda.sh
 conda activate mpr
 ```
 
 ### Commands
-These all need to be run from the project root. If using direnv and an envrc like above, you can omit the path and run the script directly, eg: `tests`.
+These all need to be run from the project root. If using direnv and an envrc like above, you can omit the path and run the script directly (eg: `tests`).
 
 ```bash
 # View cash index report for last 20 days
@@ -29,9 +33,12 @@ bin/cash --days=20
 # View a USDA report by report slug
 bin/report lm_hg201
 
-# Start notebook server
-jupyter notebook
-
 # Run all tests
 bin/tests
 ```
+
+There are also some jupyter notebooks in this repository. Start the notebook server with:
+
+```bash
+jupyter notebook
+```

bin/tests

@@ -2,5 +2,7 @@
 
 export DB='test/db/db.h5'
 rm -f $DB
-python3 -m unittest "$@"
+coverage run --omit="test/*" -m unittest "$@"
 rm -f $DB
+coverage report
+

environment.yml

@@ -5,9 +5,11 @@ channels:
   - defaults
 dependencies:
   - aiohttp
+  - coveralls
   - jupyter
-  - matplotib
+  - matplotlib
   - numpy
   - pandas
   - pytables
   - python=3.7.*
+

mpr/cash_prices/report.py

@@ -56,8 +56,8 @@ def with_change(values: Series) -> Tuple[Series, Series]:
 def cash_prices_report(records: Iterator[Slaughter]) -> DataFrame:
     array = to_array(filter_types(records))
     columns = ['date', 'arrangement', 'head_count', 'carcass_weight', 'net_price']
-
     data = DataFrame(array, columns=columns).set_index(['date', 'arrangement'])
+
     head_count = data.head_count
     carcass_weight = data.carcass_weight
     net_price = data.net_price

mpr/data/db/__init__.py

@@ -1,10 +1,10 @@
 from os import environ
+from typing import Optional
 from importlib import import_module
 from pathlib import Path
-from typing import Optional
 
 import tables
-from tables import Node
+from .entity import Entity
 
 path = Path(environ.get('DB', 'mpr/data/db/db.h5'))
 
@@ -15,6 +15,6 @@
     connection.create_group('/', 'mpr', 'USDA Mandatory Price Reporting')
 
 
-def get(group: str, table: Optional[str] = None) -> Node:
+def get(group: str, table: Optional[str] = None):
     module = import_module(f".{group}", package='mpr.data.db')
-    return module.get(table)
+    return module if table is None else getattr(module, table)

mpr/data/db/entity/observation.py

@@ -2,8 +2,6 @@
 from typing import Iterator
 from datetime import date
 
-import numpy as np
-
 from . import Entity
 from . import Record
 
@@ -16,7 +14,7 @@ def get(cls) -> Iterator[Record]:
     @classmethod
     def get_date(cls, observation_date: date) -> Iterator[Record]:
         return super(Observation, cls).query("""date == observation_date""", {
-            'observation_date': observation_date
+            'observation_date': observation_date.toordinal()
         })
 
     @classmethod
@@ -26,17 +24,6 @@ def get_range(cls, start: date, end=date.today()) -> Iterator[Record]:
             'end': end.toordinal()
         })
 
-    @classmethod
-    def get_recent(cls, days: int) -> Iterator[Record]:
-        today = date.today()
-        start = np.busday_offset(today, -days).astype('O')
-
-        return cls.get_range(start, today)
-
-    @classmethod
-    def get_year(cls, year: int) -> Iterator[Record]:
-        return cls.get_range(date(year, 1, 1), date(year, 12, 31))
-
     @classmethod
     def dates(cls) -> Iterator[date]:
         return map(date.fromordinal, set(cls.table.cols.date[:]))

setup.py

@@ -4,4 +4,6 @@
   name="mpr",
   author="Andrew Kirkegaard",
   author_email="andrew.kirkegaard@gmail.com",
-  packages=["data"], requires=['tables', 'aiohttp', 'numpy', 'pandas'])
+  packages=["mpr"], 
+  requires=['tables', 'aiohttp', 'numpy', 'pandas']
+)

test/__init__.py

@@ -12,10 +12,6 @@ def parse_report(report: TextIO) -> Iterator[Attributes]:
     return parse_elements(elements)
 
 
-def load_report(report: str) -> Iterator[Attributes]:
-    return parse_report(StringIO(report))
-
-
 def load_resource(name: str) -> Iterator[Attributes]:
     with open(f"test/resources/{name}") as report:
         for element in parse_report(report):

test/api/test_slaughter.py

@@ -66,6 +66,15 @@ def test_loineye_area(self):
     def test_lean_percent(self):
         self.assertTrue(isclose(negotiated.lean_percent, 55.6))
 
+    def test_total_weight(self):
+        self.assertTrue(isclose(negotiated.total_weight, 2623291.16))
+
+    def test_total_value(self):
+        self.assertTrue(isclose(negotiated.total_value, 139716484.52))
+
+    def test_avg_price(self):
+        self.assertTrue(isclose(negotiated.avg_price, 53.26))
+
 
 class NegotiatedFormulaTest(TestCase):
     def test_date(self):
@@ -110,6 +119,15 @@ def test_loineye_area(self):
     def test_lean_percent(self):
         self.assertTrue(isnan(negotiated_formula.lean_percent))
 
+    def test_total_weight(self):
+        self.assertTrue(isnan(negotiated_formula.total_weight))
+
+    def test_total_value(self):
+        self.assertTrue(isnan(negotiated_formula.total_value))
+
+    def test_avg_price(self):
+        self.assertTrue(isnan(negotiated_formula.avg_price))
+
 
 class TestRecordArray(TestCase):
     def test_length(self):

test/db/test_hg200.py

@@ -13,8 +13,7 @@
 class TestHg200(TestCase):
     @classmethod
     def setUpClass(cls):
-        from mpr.data.db.lm_hg200 import barrows_gilts
-        cls.report = barrows_gilts
+        cls.report = db.get('lm_hg200', 'barrows_gilts')
 
     def test_create(self):
         self.assertTrue('/mpr/lm_hg200' in db.connection)
@@ -64,7 +63,7 @@ def test_query(self):
         self.assertTrue(isclose(negotiated.avg_price, 50.70))
 
     def test_array(self):
-        records = to_array(self.report.get())
+        records = to_array(self.report.get_date(date(2018, 1, 1)))
         self.assertEqual(len(records), 2)
         self.assertTrue(all(records.date == date(2018, 1, 1)))
         self.assertTrue(len(records.arrangement == Arrangement.NEGOTIATED), 1)

test/db/test_hg201.py

@@ -13,8 +13,7 @@
 class TestHg201(TestCase):
     @classmethod
     def setUpClass(cls):
-        from mpr.data.db.lm_hg201 import barrows_gilts
-        cls.report = barrows_gilts
+        cls.report = db.get('lm_hg201', 'barrows_gilts')
 
     def test_create(self):
         self.assertTrue('/mpr/lm_hg201' in db.connection)
@@ -76,7 +75,7 @@ def test_query(self):
         self.assertTrue(isclose(record.loineye_area, 7.83))
 
     def test_array(self):
-        records = to_array(self.report.get())
+        records = to_array(self.report.get_date(date(2019, 2, 1)))
         self.assertEqual(len(records), 2)
         self.assertTrue(all(records.date == date(2019, 2, 1)))
         self.assertTrue(len(records.arrangement == Arrangement.NEGOTIATED), 1)

test/db/test_hg202.py

@@ -13,8 +13,7 @@
 class TestHg202(TestCase):
     @classmethod
     def setUpClass(cls):
-        from mpr.data.db.lm_hg202 import barrows_gilts
-        cls.report = barrows_gilts
+        cls.report = db.get('lm_hg202', 'barrows_gilts')
 
     def test_create(self):
         self.assertTrue('/mpr/lm_hg202' in db.connection)
@@ -64,7 +63,7 @@ def test_query(self):
         self.assertTrue(isclose(negotiated.avg_price, 50.70))
 
     def test_array(self):
-        records = to_array(self.report.get())
+        records = to_array(self.report.get_date(date(2018, 1, 1)))
         self.assertEqual(len(records), 2)
         self.assertTrue(all(records.date == date(2018, 1, 1)))
         self.assertTrue(len(records.arrangement == Arrangement.NEGOTIATED), 1)

test/db/test_hg203.py

@@ -13,8 +13,7 @@
 class TestHg203(TestCase):
     @classmethod
     def setUpClass(cls):
-        from mpr.data.db.lm_hg203 import barrows_gilts
-        cls.report = barrows_gilts
+        cls.report = db.get('lm_hg203', 'barrows_gilts')
 
     def test_create(self):
         self.assertTrue('/mpr/lm_hg203' in db.connection)
@@ -64,7 +63,7 @@ def test_query(self):
         self.assertTrue(isclose(negotiated.avg_price, 50.70))
 
     def test_array(self):
-        records = to_array(self.report.get())
+        records = to_array(self.report.get_date(date(2018, 1, 1)))
         self.assertEqual(len(records), 2)
         self.assertTrue(all(records.date == date(2018, 1, 1)))
         self.assertTrue(len(records.arrangement == Arrangement.NEGOTIATED), 1)

test/db/test_pk602.py

@@ -10,8 +10,7 @@
 class TestPk602(TestCase):
     @classmethod
     def setUpClass(cls):
-        from mpr.data.db.lm_pk602 import cutout
-        cls.report = cutout
+        cls.report = db.get('lm_pk602').cutout
 
     def test_create(self):
         self.assertTrue('/mpr/lm_pk602' in db.connection)

test/db/test_pk603.py

@@ -10,8 +10,7 @@
 class TestPk603(TestCase):
     @classmethod
     def setUpClass(cls):
-        from mpr.data.db.lm_pk603 import cutout
-        cls.report = cutout
+        cls.report = db.get('lm_pk603').cutout
 
     def test_create(self):
         self.assertTrue('/mpr/lm_pk603' in db.connection)