Merge pull request #94 from pitthsls/turnaways

more turnaways support

pycounter/constants.py

@@ -59,11 +59,11 @@
     u"DB1": u"Total Searches, Result Clicks and Record Views by Month and " u"Database",
     u"DB2": u"Access Denied by Month, Database and Category",
     u"JR1": u"Number of Successful Full-Text Article Requests by Month and " u"Journal",
-    u"JR1GOA": u"Number of Successful Gold Open Access Full-Text Article "
+    u"JR1 GOA": u"Number of Successful Gold Open Access Full-Text Article "
     u"Requests by Month and Journal",
     u"JR1a": u"Number of Successful Full-Text Article Requests from an "
     u"Archive by Month and Journal",
-    u"JR2": u"Access Denied to Full Text Articles by Month, Journal, and " u"Category",
+    u"JR2": u"Access Denied to Full Text Articles by Month, Journal, and Category",
     u"JR3": u"Number of Successful Item Requests and Turnaways by Month, "
     u"Journal, and Page-Type",
     u"JR3mobile": u"Number of Successful Item Requests by Month, Journal, "
@@ -106,7 +106,7 @@
         u"Reporting Period HTML",
         u"Reporting Period PDF",
     ),
-    "JR2": (
+    "JR1 GOA": (
         u"Journal",
         u"Publisher",
         u"Platform",
@@ -118,6 +118,17 @@
         u"Reporting Period HTML",
         u"Reporting Period PDF",
     ),
+    "JR2": (
+        u"Journal",
+        u"Publisher",
+        u"Platform",
+        u"Journal DOI",
+        u"Proprietary Identifier",
+        u"Print ISSN",
+        u"Online ISSN",
+        u"Access Denied Category",
+        u"Reporting Period Total",
+    ),
     "JR3": (
         u"Journal",
         u"Publisher",
@@ -158,6 +169,7 @@
         u"Proprietary Identifier",
         u"ISBN",
         u"ISSN",
+        u"Access Denied Category",
         u"Reporting Period Total",
     ),
     "DB1": (
@@ -192,7 +204,9 @@
 
 TOTAL_TEXT = {
     "JR1": u"Total for all journals",
+    "JR2": u"Total for all journals",
     "BR1": u"Total for all titles",
     "BR2": u"Total for all titles",
+    "BR3": u"Total for all titles",
     "DB2": u"Total for all databases",
 }

pycounter/report.py

@@ -170,7 +170,7 @@ def as_generic(self):
         output_lines.append([u"Date run:"])
         output_lines.append([self.date_run.strftime("%Y-%m-%d")])
         output_lines.append(self._table_header())
-        if self.report_type in ("JR1", "BR1", "BR2", "DB2"):
+        if self.report_type in ("JR1", "BR1", "BR2", "DB2", "JR2", "BR3"):
             output_lines.extend(self._totals_lines())
         elif self.report_type.startswith("DB"):
             self._ensure_required_metrics()
@@ -207,9 +207,9 @@ def _totals_line(self, metric):
             total_cells.append(platforms.pop())
         else:
             total_cells.append(u"")
-        if self.report_type in ("JR1", "BR1", "BR2"):
+        if self.report_type in ("JR1", "BR1", "BR2", "JR2", "BR3"):
             total_cells.extend([u""] * 4)
-        elif self.report_type == "DB2":
+        if self.report_type in ("DB2", "JR2", "BR3"):
             total_cells.append(metric)
         total_usage = 0
         pdf_usage = 0
@@ -426,9 +426,12 @@ def as_generic(self):
         for data in self:
             total_usage += data[2]
             month_data.append(six.text_type(data[2]))
+        if self.metric.startswith("Access"):
+            data_line.append(self.metric)
         data_line.append(six.text_type(total_usage))
-        data_line.append(six.text_type(self.html_total))
-        data_line.append(six.text_type(self.pdf_total))
+        if not self.metric.startswith("Access"):
+            data_line.append(six.text_type(self.html_total))
+            data_line.append(six.text_type(self.pdf_total))
         data_line.extend(month_data)
         return data_line
 
@@ -523,6 +526,8 @@ def as_generic(self):
         for data in self:
             total_usage += data[2]
             month_data.append(six.text_type(data[2]))
+        if self.metric and self.metric.startswith("Access"):
+            data_line.append(self.metric)
         data_line.append(six.text_type(total_usage))
         data_line.extend(month_data)
         return data_line
@@ -803,7 +808,11 @@ def _parse_line(line, report, last_col):
 
     metric = report.metric
     if report.report_version >= 4:
-        if report.report_type.startswith("JR1") or report.report_type == "TR_J1":
+        if (
+            report.report_type.startswith("JR1")
+            or report.report_type == "TR_J1"
+            or report.report_type == "TR_J2"
+        ):
             old_line = line
             line = line[0:3] + line[5:7] + line[10:last_col]
             doi = old_line[3]
@@ -812,6 +821,8 @@ def _parse_line(line, report, last_col):
             pdf_total = format_stat(old_line[9])
             issn = line[3].strip()
             eissn = line[4].strip()
+            if report.report_type == "TR_J2":
+                metric = old_line[9]
 
         elif report.report_type in ("BR1", "BR2"):
             line = line[0:3] + line[5:7] + line[8:last_col]
@@ -851,7 +862,11 @@ def _parse_line(line, report, last_col):
     for data in line[months_start_idx:]:
         month_data.append((curr_month, format_stat(data)))
         curr_month = next_month(curr_month)
-    if report.report_type.startswith("JR") or report.report_type == "TR_J1":
+    if (
+        report.report_type.startswith("JR")
+        or report.report_type == "TR_J1"
+        or report.report_type == "TR_J2"
+    ):
         return CounterJournal(
             metric=metric,
             month_data=month_data,

pycounter/sushi.py

@@ -275,7 +275,7 @@ def raw_to_full(raw_report):
         html_usage = 0
         pdf_usage = 0
 
-        metrics_for_db = collections.defaultdict(list)
+        metrics_for_db = collections.OrderedDict()
 
         for perform_item in item.ItemPerformance:
             item_date = convert_date_run(perform_item.Period.Begin.text)
@@ -289,18 +289,19 @@ def raw_to_full(raw_report):
                         pdf_usage += int(inst.Count)
                     elif inst.MetricType == "ft_html":
                         html_usage += int(inst.Count)
-                    elif (
-                        report.report_type.startswith("DB")
-                        or report.report_type == "PR1"
+                    elif report.report_type.startswith("DB") or report.report_type in (
+                        "PR1",
+                        "JR2",
+                        "BR3",
                     ):
-                        metrics_for_db[inst.MetricType].append(
+                        metrics_for_db.setdefault(inst.MetricType, []).append(
                             (item_date, int(inst.Count))
                         )
             if usage is not None:
                 month_data.append((item_date, int(usage)))
 
         if report.report_type:
-            if report.report_type.startswith("JR"):
+            if report.report_type == "JR1":
                 report.pubs.append(
                     pycounter.report.CounterJournal(
                         title=title,
@@ -317,7 +318,26 @@ def raw_to_full(raw_report):
                         pdf_total=pdf_usage,
                     )
                 )
+            elif report.report_type == "BR3":
+                for metric_code, month_data in six.iteritems(metrics_for_db):
+                    metric = pycounter.constants.DB_METRIC_MAP[metric_code]
+                    report.pubs.append(
+                        pycounter.report.CounterBook(
+                            title=title,
+                            platform=platform,
+                            publisher=publisher_name,
+                            period=report.period,
+                            metric=metric,
+                            issn=issn,
+                            print_isbn=print_isbn,
+                            online_isbn=online_isbn,
+                            doi=doi,
+                            proprietary_id=prop_id,
+                            month_data=month_data,
+                        )
+                    )
             elif report.report_type.startswith("BR"):
+                # BR1, BR2
                 report.pubs.append(
                     pycounter.report.CounterBook(
                         title=title,
@@ -358,4 +378,21 @@ def raw_to_full(raw_report):
                             month_data=month_data,
                         )
                     )
+            elif report.report_type == "JR2":
+                for metric_code, month_data in six.iteritems(metrics_for_db):
+                    metric = pycounter.constants.DB_METRIC_MAP[metric_code]
+                    report.pubs.append(
+                        pycounter.report.CounterJournal(
+                            title=title,
+                            platform=platform,
+                            publisher=publisher_name,
+                            period=report.period,
+                            metric=metric,
+                            issn=issn,
+                            eissn=eissn,
+                            doi=doi,
+                            proprietary_id=prop_id,
+                            month_data=month_data,
+                        )
+                    )
     return report

pycounter/test/conftest.py

@@ -65,6 +65,8 @@ def parse_sushi_file(filename):
         "sushi_simple_br1.xml",
         "sushi_simple_db1.xml",
         "sushi_db1_missing_record_view.xml",
+        "sushi_br3.xml",
+        "sushi_jr2.xml",
     ]
 )
 def sushi_report_all(request):
@@ -97,6 +99,18 @@ def sushi_report_jr1(request):
     return parse_sushi_file(request.param)
 
 
+@pytest.fixture
+def sushi_report_jr2():
+    """Journal turnaways."""
+    return parse_sushi_file("sushi_jr2.xml")
+
+
+@pytest.fixture
+def sushi_report_br3():
+    """Book turnaways."""
+    return parse_sushi_file("sushi_br3.xml")
+
+
 @pytest.fixture(
     params=[
         "C4BR1.tsv",
@@ -155,3 +169,28 @@ def br3_report():
 def jr2_report():
     """Journal report 2 (turnaways)."""
     return parsedata("C4JR2.csv")
+
+
+@pytest.fixture(
+    params="""C4BR1.tsv
+C4BR2.tsv
+C4BR3.csv
+C4DB1.tsv
+C4DB1_split_year.tsv
+C4DB2.tsv
+C4JR1.csv
+C4JR1_bad.csv
+C4JR1big.csv
+C4JR1GOA.csv
+C4JR1mul.csv
+C4JR1my.csv
+C4JR2.csv
+PR1.tsv
+simpleBR1.csv
+simpleJR1.csv
+simpleJR1.tsv
+""".split()
+)
+def all_reports(request):
+    """All COUNTER 4 reports."""
+    return parsedata(request.param)

pycounter/test/counter5/conftest.py

@@ -33,6 +33,14 @@ def trj1_report():
     )
 
 
+@pytest.fixture
+def trj2_report():
+    """Tab-separated title turnaways report."""
+    return pycounter.report.parse(
+        os.path.join(os.path.dirname(__file__), "data", "tr_j2.tsv")
+    )
+
+
 @pytest.fixture
 def sushi5_report_trb1():
     """JSON SUSHI report."""

pycounter/test/counter5/data/tr_j2.tsv

@@ -0,0 +1,18 @@
+Report_Name	"Journal Access Denied	"															
+Report_ID	TR_J2															
+Release	5															
+Institution_Name	Sample University															
+Institution_ID	isni=1234567890															
+Metric_Types	Limit_Exceeded; No_License															
+Report_Filters	Data_Type=Journal; Access_Method=Regular															
+Report_Attributes																
+Exceptions																
+Reporting_Period	2017-01-01 to 2017-06-30															
+Created	2017-05-25															
+Created_By	Platform X															
+
+Title	Publisher	Publisher_ID	Platform	DOI	Proprietary_ID	Print_ISSN	Online_ISSN	URI	Metric_Type	Reporting_ Period_Total	Jan-2017	Feb-2017	Mar-2017	Apr-2017	May-2017	Jun-2017
+Journal A	Publisher X	isni=1234123412341234	PlatformX	"/12.1.0.1/1111.2.222	"	pubx:jnlA	1111-22222	1111-1223		Limit_Exceeded	3	3					
+Journal A	Publisher X	isni=1234123412341234	PlatformX	/12.1.0.1/1111.2.222	pubx:jnlA	1111-22222	1111-1223		No_License							
+Journal B	Publisher X	isni=1234123412341234	PlatformX	/12.1.0.1/1111.2.211	pubx:jnlB	1111-22211	1111-1213		Limit_Exceeded							
+Journal B	Publisher X	isni=1234123412341234	PlatformX	/12.1.0.1/1111.2.211	pubx:jnlB	1111-22211	1111-1213		No_License							

pycounter/test/counter5/test_trj2.py

@@ -0,0 +1,18 @@
+"""Test parsing of COUNTER 5 TRJ2 report (turnaways)"""
+
+import datetime
+
+
+def test_metric(trj2_report):
+    assert trj2_report.metric is None  # Multiple metrics per report
+
+
+def test_type(trj2_report):
+    assert trj2_report.report_type == u"TR_J2"
+
+
+def test_data(trj2_report):
+    i = iter(trj2_report)
+    row = next(i)
+    item = next(iter(row))
+    assert item == (datetime.date(2017, 1, 1), u"Limit_Exceeded", 3)

pycounter/test/data/C4JR2.csv

@@ -5,7 +5,7 @@
 "2011-01-01 to 2011-12-31"
 "Date run:"
 2012-02-21
-Journal,Publisher,Platform,"Journal DOI","Proprietary Identifier","Print ISSN","Online ISSN","Access Denied Category","Reporting Period Total","Reporting Period HTML","Reporting Period PDF",Jan-2011,Feb-2011,Mar-2011,Apr-2011,May-2011,Jun-2011,Jul-2011,Aug-2011,Sep-2011,Oct-2011,Nov-2011,Dec-2011
+Journal,Publisher,Platform,"Journal DOI","Proprietary Identifier","Print ISSN","Online ISSN","Access Denied Category","Reporting Period Total",Jan-2011,Feb-2011,Mar-2011,Apr-2011,May-2011,Jun-2011,Jul-2011,Aug-2011,Sep-2011,Oct-2011,Nov-2011,Dec-2011
 "Total for all journals","Maximegalon University Press",MJO,,,,,16,0,16,2,1,0,0,0,5,1,1,0,5,1,0
 "Abstracts of Working Papers in Economics","Maximegalon University Press",MJO,,,0951-0079,0951-0079,Access denied: concurrent/simultaneous user license limit exceeded,3,3,0,0,0,0,0,0,0,0,0,0,0,0,0
 "Acta Numerica","Maximegalon University Press",MJO,,,0962-4929,1474-0508,Access denied: concurrent/simultaneous user license limit exceeded,16,0,16,2,1,0,0,0,5,1,1,0,5,1,0