Add report_date to slaughter and purchase models

mpr/data/api/purchase.py

@@ -12,6 +12,7 @@
 
 from . import Attributes
 from . import Report
+from . import get_optional
 from . import opt_int
 from . import opt_float
 from . import fetch
@@ -55,12 +56,19 @@ class Section(Enum):
 
 
 def parse_attributes(attr: Attributes) -> Purchase:
-    report_date = datetime.strptime(attr['reported_for_date'], date_format).date()
+    record_date_string = attr['reported_for_date']
+    report_date_string = get_optional(attr, 'report_date')
+
+    record_date = datetime.strptime(record_date_string, date_format).date()
+    report_date = record_date if report_date_string is None else \
+        datetime.strptime(report_date_string, date_format).date()
+
     purchase_type = attr['purchase_type']
     (seller, arrangement, basis) = purchase_types[purchase_type]
 
     return Purchase(
-        date=datetime64(report_date, 'D'),
+        date=datetime64(record_date, 'D'),
+        report_date=datetime64(report_date, 'D'),
         seller=seller.value,
         arrangement=arrangement.value,
         basis=basis.value,

mpr/data/api/slaughter.py

@@ -57,12 +57,15 @@ class Section(Enum):
 
 
 def parse_attributes(attr: Attributes) -> Slaughter:
-    report_date = datetime.strptime(attr['for_date_begin'], date_format).date()
+    record_date = datetime.strptime(attr['for_date_begin'], date_format).date()
+    report_date = datetime.strptime(attr['report_date'], date_format).date()
+
     purchase_type = attr['purchase_type']
     (seller, arrangement, basis) = purchase_types[purchase_type]
 
     return Slaughter(
-        date=datetime64(report_date, 'D'),
+        date=datetime64(record_date, 'D'),
+        report_date=datetime64(report_date, 'D'),
         seller=seller.value,
         arrangement=arrangement.value,
         basis=basis.value,

mpr/data/db/entity/purchase.py

@@ -16,6 +16,7 @@
 class PurchaseEntity(Observation[Purchase], ABC):
     schema = dtype([
         ('date', uint32),
+        ('report_date', uint32),
         ('seller', uint8),
         ('arrangement', uint8),
         ('basis', uint8),
@@ -27,8 +28,14 @@ class PurchaseEntity(Observation[Purchase], ABC):
 
     @staticmethod
     def from_row(row: Row) -> Purchase:
-        return Purchase(datetime64(date.fromordinal(row[0]), 'D'), *row[1:])
+        return Purchase(
+            datetime64(date.fromordinal(row[0]), 'D'),
+            datetime64(date.fromordinal(row[1]), 'D'),
+            *row[2:])
 
     @staticmethod
     def to_row(record: Purchase) -> Tuple:
-        return (record[0].astype(date).toordinal(), *record[1:])
+        return (
+            record[0].astype(date).toordinal(),
+            record[1].astype(date).toordinal(),
+            *record[2:])

mpr/data/db/entity/slaughter.py

@@ -16,6 +16,7 @@
 class SlaughterEntity(Observation[Slaughter], ABC):
     schema = dtype([
         ('date', uint32),
+        ('report_date', uint32),
         ('seller', uint8),
         ('arrangement', uint8),
         ('basis', uint8),
@@ -35,8 +36,14 @@ class SlaughterEntity(Observation[Slaughter], ABC):
 
     @staticmethod
     def from_row(row: Row) -> Slaughter:
-        return Slaughter(datetime64(date.fromordinal(row[0]), 'D'), *row[1:])
+        return Slaughter(
+            datetime64(date.fromordinal(row[0]), 'D'),
+            datetime64(date.fromordinal(row[1]), 'D'),
+            *row[2:])
 
     @staticmethod
     def to_row(record: Slaughter) -> Tuple:
-        return (record[0].astype(date).toordinal(), *record[1:])
+        return (
+            record[0].astype(date).toordinal(),
+            record[1].astype(date).toordinal(),
+            *record[2:])

mpr/data/model/purchase.py

@@ -14,6 +14,7 @@
 
 class Purchase(NamedTuple):
     date: Date
+    report_date: Date
     seller: uint8
     arrangement: uint8
     basis: uint8
@@ -26,12 +27,13 @@ def __hash__(self) -> int:
         return hash((self.date.astype(date).toordinal(), self.seller, self.arrangement, self.basis))
 
     def __eq__(self, other) -> bool:
-        return isinstance(other, Purchase) and hash(self) == hash(other) and np.allclose(self[4:], other[4:])
+        return isinstance(other, Purchase) and hash(self) == hash(other) and np.allclose(self[5:], other[5:])
 
 
 def to_array(records: Iterator[Purchase]) -> recarray:
     return np.rec.array(list(records), dtype=np.dtype([
         ('date', date_type),
+        ('report_date', date_type),
         ('seller', uint8),
         ('arrangement', uint8),
         ('basis', uint8),

mpr/data/model/slaughter.py

@@ -14,6 +14,7 @@
 
 class Slaughter(NamedTuple):
     date: Date
+    report_date: Date
     seller: uint8
     arrangement: uint8
     basis: uint8
@@ -34,7 +35,7 @@ def __hash__(self) -> int:
         return hash((self.date.astype(date).toordinal(), self.seller, self.arrangement, self.basis))
 
     def __eq__(self, other) -> bool:
-        return isinstance(other, Slaughter) and hash(self) == hash(other) and np.allclose(self[4:], other[4:])
+        return isinstance(other, Slaughter) and hash(self) == hash(other) and np.allclose(self[5:], other[5:])
 
     @property
     def total_weight(self) -> float:
@@ -52,6 +53,7 @@ def avg_price(self) -> float:
 def to_array(records: Iterator[Slaughter]) -> recarray:
     return np.rec.array(list(records), dtype=np.dtype([
         ('date', date_type),
+        ('report_date', date_type),
         ('seller', uint8),
         ('arrangement', uint8),
         ('basis', uint8),

test/api/test_purchase.py

@@ -18,9 +18,12 @@
 
 
 class NegotiatedPurchaseTest(TestCase):
-    def test_report_date(self):
+    def test_date(self):
         self.assertEqual(negotiated.date, date(2019, 1, 31))
 
+    def test_report_date(self):
+        self.assertEqual(negotiated.report_date, date(2019, 2, 1))
+
     def test_seller(self):
         self.assertEqual(negotiated.seller, Seller.ALL)
 

test/api/test_slaughter.py

@@ -17,10 +17,13 @@
 records = to_array([negotiated, negotiated_formula])
 
 
-class NegotiatedPurchaseTest(TestCase):
+class NegotiatedTest(TestCase):
     def test_date(self):
         self.assertEqual(negotiated.date, date(2019, 2, 1))
 
+    def test_report_date(self):
+        self.assertEqual(negotiated.report_date, date(2019, 2, 4))
+
     def test_seller(self):
         self.assertEqual(negotiated.seller, Seller.PRODUCER)
 

test/db/test_hg201.py

@@ -30,6 +30,7 @@ def tearDownClass(cls):
     def setUp(self):
         records = (parse_attributes({
             'for_date_begin': '02/01/2019',
+            'report_date': '02/04/2019',
             'purchase_type': 'Prod. Sold Negotiated',
             'head_count': '12,771',
             'base_price': '51.80',
@@ -45,6 +46,7 @@ def setUp(self):
             'loineye_area': '7.83'
         }), parse_attributes({
             'for_date_begin': '02/01/2019',
+            'report_date': '02/04/2019',
             'purchase_type': 'Prod. Sold Negotiated Formula',
             'head_count': '683'
         }))

test/db/test_hg202.py

@@ -30,6 +30,7 @@ def tearDownClass(cls):
     def setUp(self):
         purchases = (parse_attributes({
             'reported_for_date': '1/1/2018',
+            'reported_date': '1/2/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '11,234',
             'price_low': '48.00',
@@ -79,6 +80,7 @@ def test_array(self):
     def test_index(self):
         purchase = parse_attributes({
             'reported_for_date': '1/2/2018',
+            'reported_date': '1/3/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '14141',
             'price_low': '48.00',

test/db/test_hg203.py

@@ -30,6 +30,7 @@ def tearDownClass(cls):
     def setUp(self):
         purchases = (parse_attributes({
             'reported_for_date': '1/1/2018',
+            'report_date': '1/2/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '11,234',
             'price_low': '48.00',
@@ -79,6 +80,7 @@ def test_array(self):
     def test_index(self):
         purchase = parse_attributes({
             'reported_for_date': '1/2/2018',
+            'report_date': '1/3/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '14141',
             'price_low': '48.00',

test/model/test_purchase.py

@@ -6,6 +6,7 @@ class TestPurchase(TestCase):
     def test_objects_are_the_same(self):
         first = parse_attributes({
             'reported_for_date': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '14,141',
             'price_low': '48.00',
@@ -15,6 +16,7 @@ def test_objects_are_the_same(self):
 
         second = parse_attributes({
             'reported_for_date': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '0'
         })
@@ -24,12 +26,14 @@ def test_objects_are_the_same(self):
     def test_objects_are_not_the_same(self):
         first = parse_attributes({
             'reported_for_date': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Negotiated Formula (carcass basis)',
             'head_count': '0'
         })
 
         second = parse_attributes({
             'reported_for_date': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Negotiated Formula (live basis)',
             'head_count': '0'
         })
@@ -39,6 +43,7 @@ def test_objects_are_not_the_same(self):
     def test_contents_are_the_same(self):
         first = parse_attributes({
             'reported_for_date': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '14,141',
             'price_low': '48.00',
@@ -48,6 +53,7 @@ def test_contents_are_the_same(self):
 
         second = parse_attributes({
             'reported_for_date': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '14,141',
             'price_low': '48.00',
@@ -60,6 +66,7 @@ def test_contents_are_the_same(self):
     def test_contents_are_not_the_same(self):
         first = parse_attributes({
             'reported_for_date': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '14,141',
             'price_low': '48.00',
@@ -69,6 +76,7 @@ def test_contents_are_not_the_same(self):
 
         second = parse_attributes({
             'reported_for_date': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Negotiated (carcass basis)',
             'head_count': '14,141'
         })

test/model/test_slaughter.py

@@ -6,12 +6,14 @@ class TestSlaughter(TestCase):
     def test_objects_are_the_same(self):
         first = parse_attributes({
             'for_date_begin': '02/01/2019',
+            'report_date': '02/02/2018',
             'purchase_type': 'Prod. Sold Negotiated',
             'head_count': '12,771'
         })
 
         second = parse_attributes({
             'for_date_begin': '02/01/2019',
+            'report_date': '02/02/2018',
             'purchase_type': 'Prod. Sold Negotiated',
             'head_count': '0'
         })
@@ -21,12 +23,14 @@ def test_objects_are_the_same(self):
     def test_objects_are_not_the_same(self):
         first = parse_attributes({
             'for_date_begin': '01/03/2018',
+            'report_date': '01/04/2018',
             'purchase_type': 'Prod. Sold Negotiated Formula',
             'head_count': '0'
         })
 
         second = parse_attributes({
             'for_date_begin': '01/02/2018',
+            'report_date': '01/03/2018',
             'purchase_type': 'Prod. Sold Negotiated Formula',
             'head_count': '0'
         })
@@ -36,6 +40,7 @@ def test_objects_are_not_the_same(self):
     def test_contents_are_the_same(self):
         first = parse_attributes({
             'for_date_begin': '02/01/2019',
+            'report_date': '02/02/2018',
             'purchase_type': 'Prod. Sold Negotiated',
             'head_count': '12,771',
             'base_price': '51.80',
@@ -53,6 +58,7 @@ def test_contents_are_the_same(self):
 
         second = parse_attributes({
             'for_date_begin': '02/01/2019',
+            'report_date': '02/02/2018',
             'purchase_type': 'Prod. Sold Negotiated',
             'head_count': '12,771',
             'base_price': '51.80',
@@ -73,6 +79,7 @@ def test_contents_are_the_same(self):
     def test_contents_are_not_the_same(self):
         first = parse_attributes({
             'for_date_begin': '02/01/2019',
+            'report_date': '02/02/2018',
             'purchase_type': 'Prod. Sold Negotiated',
             'head_count': '12,771',
             'base_price': '51.80',
@@ -90,6 +97,7 @@ def test_contents_are_not_the_same(self):
 
         second = parse_attributes({
             'for_date_begin': '02/01/2018',
+            'report_date': '02/02/2018',
             'purchase_type': 'Prod. Sold Negotiated',
             'head_count': '0'
         })

test/reports/test_cash_index.py

@@ -2,13 +2,16 @@
 from datetime import date
 import numpy as np
 
+from mpr.data.api import filter_section
+from mpr.data.api.slaughter import Section
 from mpr.data.api.slaughter import parse_attributes
 from mpr.data.model.slaughter import to_array
 from mpr.data.model.purchase_type import Arrangement
 
 from . import load_resource
 
-records = to_array(map(parse_attributes, load_resource('cash_prices.xml')))
+report = filter_section(load_resource('cash_prices.xml'), Section.BARROWS_AND_GILTS.value)
+records = to_array(map(parse_attributes, report))
 
 negotiated = records.arrangement == Arrangement.NEGOTIATED
 market_formula = records.arrangement == Arrangement.MARKET_FORMULA

test/reports/test_cash_prices.py

@@ -1,12 +1,16 @@
 from unittest import TestCase
 from numpy import allclose
 
+from mpr.data.api import filter_section
+from mpr.data.api.slaughter import Section
 from mpr.data.api.slaughter import parse_attributes
 from mpr.cash_prices.report import cash_prices_report
+
 from . import load_resource
 
 # Given a Daily Slaughtered Swine Report from Feb 20, 2019
-records = map(parse_attributes, load_resource('cash_prices.xml'))
+report = filter_section(load_resource('cash_prices.xml'), Section.BARROWS_AND_GILTS.value)
+records = map(parse_attributes, report)
 
 # When I run a cash prices report for the last 10 days
 report = cash_prices_report(records).tail(10)