Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merged scdl_fixes branch into master.

  • Loading branch information...
commit ed5fde56ed3173718a842e093b76f9f4deaa21cb 1 parent a1b4b67
Szymon Guz authored
View
76 lib/akamod/enrich-date.py
@@ -124,22 +124,18 @@ def test_parse_date_or_range():
assert res == DATE_TESTS[i], "For input '%s', expected '%s' but got '%s'"%(i,DATE_TESTS[i],res)
-@simple_service('POST', 'http://purl.org/la/dp/enrich-date', 'enrich-date', HTTP_TYPE_JSON)
-def enrichdate(body, ctype, action="enrich-format", prop="aggregatedCHO/date"):
- """
- Service that accepts a JSON document and extracts the "created date" of the item, using the
- following rules:
+def convert_dates(data, prop, earliest):
+ """Converts dates.
- a) Looks in the list of fields specified by the 'prop' parameter
- b) Extracts all dates, and sets the created date to the earliest date
- """
- try :
- data = json.loads(body)
- except:
- response.code = HTTP_INTERNAL_SERVER_ERROR
- response.add_header(HTTP_HEADER_TYPE, HTTP_TYPE_TEXT)
- return "Unable to parse body as JSON"
+ Arguments:
+ data Dict - Data for conversion.
+ prop Str - Properties dividided with comma.
+ earliest Bool - True - the function will set only the earliest date.
+ False - the function will set all dates.
+ Returns:
+ Nothing, the replacement is done in place.
+ """
dates = []
for p in prop.split(','):
if exists(data, p):
@@ -155,23 +151,47 @@ def enrichdate(body, ctype, action="enrich-format", prop="aggregatedCHO/date"):
"end": b,
"displayDate" : stripped
})
+
dates.sort(key=lambda d: d["begin"] if d["begin"] is not None else DEFAULT_DATETIME_STR)
- if len(dates) == 1:
- setprop(data, p, dates[0])
+ value_to_set = dates
+ if earliest:
+ value_to_set = dates[0]
+
+ if value_to_set:
+ setprop(data, p, value_to_set)
else:
- setprop(data, p, dates)
+ delprop(data, p)
+
+@simple_service('POST', 'http://purl.org/la/dp/enrich_earliest_date', 'enrich_earliest_date', HTTP_TYPE_JSON)
+def enrich_earliest_date(body, ctype, action="enrich_earliest_date", prop="aggregatedCHO/date"):
+ """
+ Service that accepts a JSON document and extracts the "created date" of the item, using the
+ following rules:
+
+ a) Looks in the list of fields specified by the 'prop' parameter
+ b) Extracts all dates, and sets the created date to the earliest date
+ """
+ try :
+ data = json.loads(body)
+ except:
+ response.code = HTTP_INTERNAL_SERVER_ERROR
+ response.add_header(HTTP_HEADER_TYPE, HTTP_TYPE_TEXT)
+ return "Unable to parse body as JSON"
+
+ convert_dates(data, prop, True)
return json.dumps(data)
-@simple_service('POST', 'http://purl.org/la/dp/enrich-temporal-date', 'enrich-temporal-date', HTTP_TYPE_JSON)
-def enrich_temporal_date(body, ctype, prop="aggregatedCHO/temporal"):
+
+@simple_service('POST', 'http://purl.org/la/dp/enrich_date', 'enrich_date', HTTP_TYPE_JSON)
+def enrich_date(body, ctype, action="enrich_date", prop="aggregatedCHO/temporal"):
"""
Service that accepts a JSON document and extracts the "created date" of the item, using the
following rules:
a) Looks in the list of fields specified by the 'prop' parameter
- b) Extracts all dates, and sets the created date to the earliest date
+ b) Extracts all dates
"""
try :
data = json.loads(body)
@@ -180,19 +200,5 @@ def enrich_temporal_date(body, ctype, prop="aggregatedCHO/temporal"):
response.add_header(HTTP_HEADER_TYPE, HTTP_TYPE_TEXT)
return "Unable to parse body as JSON"
- date_candidates = []
- for p in prop.split(','):
- if exists(data, p):
- v = getprop(data, p)
- for s in v:
- stripped = remove_brackets_and_strip(s)
- a, b = parse_date_or_range(stripped)
- date_candidates.append( {
- "begin": a,
- "end": b,
- "displayDate" : stripped
- })
- if date_candidates:
- setprop(data, p, date_candidates)
-
+ convert_dates(data, prop, False)
return json.dumps(data)
View
5 profiles/artstor.pjs
@@ -15,9 +15,10 @@
"http://localhost:8879/shred?prop=aggregatedCHO%2FphysicalMedium&delim=%3b",
"http://localhost:8879/move_date_values?prop=aggregatedCHO%2Fsubject",
"http://localhost:8879/move_date_values?prop=aggregatedCHO%2Fspatial",
- "http://localhost:8879/enrich-date",
+ "http://localhost:8878/shred?prop=aggregatedCHO%2Fspatial&delim=--",
+ "http://localhost:8879/enrich_earliest_date",
"http://localhost:8879/enrich-subject",
- "http://localhost:8879/enrich-temporal-date",
+ "http://localhost:8879/enrich_date",
"http://localhost:8879/cleanup_value",
"http://localhost:8879/enrich-type",
"http://localhost:8879/enrich-format",
View
5 profiles/bhl.pjs
@@ -13,9 +13,10 @@
"http://localhost:8878/shred?prop=isShownAt%2Fformat",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fsubject",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fspatial",
- "http://localhost:8878/enrich-date",
+ "http://localhost:8878/shred?prop=aggregatedCHO%2Fspatial&delim=--",
+ "http://localhost:8878/enrich_earliest_date",
"http://localhost:8878/enrich-subject",
- "http://localhost:8878/enrich-temporal-date",
+ "http://localhost:8878/enrich_date",
"http://localhost:8875/cleanup_value",
"http://localhost:8878/enrich-type",
"http://localhost:8878/enrich-format",
View
5 profiles/clemson.pjs
@@ -14,10 +14,11 @@
"http://localhost:8878/shred?prop=isShownAt%2Fformat",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fsubject",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fspatial",
- "http://localhost:8878/enrich-date",
+ "http://localhost:8878/shred?prop=aggregatedCHO%2Fspatial&delim=--",
+ "http://localhost:8878/enrich_earliest_date",
"http://localhost:8878/cleanup_value",
"http://localhost:8878/enrich-subject",
- "http://localhost:8878/enrich-temporal-date",
+ "http://localhost:8878/enrich_date",
"http://localhost:8878/enrich-subject?prop=aggregatedCHO%2Fcreator",
"http://localhost:8878/enrich-type",
"http://localhost:8878/enrich-format",
View
5 profiles/georgia.pjs
@@ -14,8 +14,9 @@
"http://localhost:8879/shred?prop=isShownAt%2Fformat",
"http://localhost:8879/move_date_values?prop=aggregatedCHO%2Fsubject",
"http://localhost:8879/move_date_values?prop=aggregatedCHO%2Fspatial",
- "http://localhost:8879/enrich-date",
- "http://localhost:8879/enrich-temporal-date",
+ "http://localhost:8878/shred?prop=aggregatedCHO%2Fspatial&delim=--",
+ "http://localhost:8879/enrich_earliest_date",
+ "http://localhost:8879/enrich_date",
"http://localhost:8879/enrich-subject",
"http://localhost:8875/cleanup_value",
"http://localhost:8879/enrich-type",
View
5 profiles/kentucky.pjs
@@ -11,8 +11,9 @@
"http://localhost:8878/shred?prop=isShownAt%2Fformat",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fsubject",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fspatial",
- "http://localhost:8878/enrich-date",
- "http://localhost:8878/enrich-temporal-date",
+ "http://localhost:8878/shred?prop=aggregatedCHO%2Fspatial&delim=--",
+ "http://localhost:8878/enrich_earliest_date",
+ "http://localhost:8878/enrich_date",
"http://localhost:8878/enrich-subject",
"http://localhost:8875/cleanup_value",
"http://localhost:8878/enrich-type",
View
5 profiles/minnesota.pjs
@@ -13,8 +13,9 @@
"http://localhost:8873/shred?prop=isShownAt%2Fformat",
"http://localhost:8873/move_date_values?prop=aggregatedCHO%2Fsubject",
"http://localhost:8873/move_date_values?prop=aggregatedCHO%2Fspatial",
- "http://localhost:8873/enrich-date",
- "http://localhost:8873/enrich-temporal-date",
+ "http://localhost:8878/shred?prop=aggregatedCHO%2Fspatial&delim=--",
+ "http://localhost:8873/enrich_earliest_date",
+ "http://localhost:8873/enrich_date",
"http://localhost:8873/enrich-subject",
"http://localhost:8875/cleanup_value",
"http://localhost:8873/enrich-type",
View
5 profiles/scdl-charleston.pjs
@@ -14,8 +14,9 @@
"http://localhost:8878/shred?prop=isShownAt%2Fformat",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fsubject",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fspatial",
- "http://localhost:8878/enrich-date",
- "http://localhost:8878/enrich-temporal-date",
+ "http://localhost:8878/shred?prop=aggregatedCHO%2Fspatial&delim=--",
+ "http://localhost:8878/enrich_earliest_date",
+ "http://localhost:8878/enrich_date",
"http://localhost:8878/enrich-subject",
"http://localhost:8878/cleanup_value",
"http://localhost:8878/enrich-type",
View
5 profiles/scdl-usc.pjs
@@ -14,8 +14,9 @@
"http://localhost:8878/shred?prop=isShownAt%2Fformat",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fsubject",
"http://localhost:8878/move_date_values?prop=aggregatedCHO%2Fspatial",
- "http://localhost:8878/enrich-date",
- "http://localhost:8878/enrich-temporal-date",
+ "http://localhost:8878/shred?prop=aggregatedCHO%2Fspatial&delim=--",
+ "http://localhost:8878/enrich_earliest_date",
+ "http://localhost:8878/enrich_date",
"http://localhost:8878/enrich-subject",
"http://localhost:8878/cleanup_value",
"http://localhost:8878/enrich-type",
View
69 test/test_enrich_date.py
@@ -18,7 +18,7 @@ def test_enrich_dates_bogus_date():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
print_error_log()
@@ -41,7 +41,7 @@ def test_enrich_date_single():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -56,26 +56,18 @@ def test_enrich_date_date_multiple():
"date" : ["1928", "1406"]
}
EXPECTED = {
- u'date' : [
- {
+ u'date' : {
u'begin': u'1406',
u'end': u'1406',
u'displayDate': u'1406'
- },
- {
- u'begin': u'1928',
- u'end': u'1928',
- u'displayDate': u'1928'
- }
- ]
+ }
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
- result = json.loads(content)
- assert result['date'] == EXPECTED[u'date']
+ assert_same_jsons(EXPECTED, content)
def test_enrich_date_date_parse_format_yyyy_mm_dd():
@@ -91,7 +83,7 @@ def test_enrich_date_date_parse_format_yyyy_mm_dd():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -114,7 +106,7 @@ def test_enrich_date_parse_format_date_with_slashes():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -136,7 +128,7 @@ def test_enrich_date_date_parse_format_natural_string():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -157,7 +149,7 @@ def test_enrich_date_date_parse_format_ca_string():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -179,7 +171,7 @@ def test_enrich_date_date_parse_format_c_string():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -201,7 +193,7 @@ def test_enrich_date_parse_format_date_range1():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -228,7 +220,7 @@ def test_date_with_brackets():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -255,7 +247,7 @@ def test_range_years_with_brackets():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -283,7 +275,7 @@ def test_range_with_brackets():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp, content = H.request(url, "POST", body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -305,7 +297,7 @@ def test_enrich_date_parse_format_date_range2():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -327,7 +319,7 @@ def test_enrich_date_parse_format_date_range3():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -349,7 +341,7 @@ def test_enrich_date_parse_format_date_range4():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -372,7 +364,7 @@ def test_enrich_date_parse_century_date():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -394,7 +386,7 @@ def test_enrich_date_parse_century_date_with_P():
}
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
@@ -406,7 +398,7 @@ def test_enrich_date_parse_century_date_with_P():
def test_enrich_temporal_date():
"""Correctly enrich temporal dates"""
- # TODO: disabled dates are not supported by enrich-date parsers
+ # TODO: disabled dates are not supported by enrich_earliest_date parsers
INPUT = {
"aggregatedCHO": {
@@ -425,10 +417,10 @@ def test_enrich_temporal_date():
"temporal": [
{"begin": "1901", "end": "1999", "displayDate": "1901-1999"},
{"begin": "1901", "end": "1999", "displayDate": "1901 - 1999"},
+ {"begin": "1901", "end": "1901", "displayDate": "1901"},
{"begin": "1901-01-01", "end": "1901-01-01", "displayDate": "1901 / 01 / 01"},
- {"begin": "1905-04-12", "end": "1905-04-12", "displayDate": "1905-04-12"},
{"begin": "1901-01-01", "end": "1901-01-01", "displayDate": "01/01/1901"},
- {"begin": "1901", "end": "1901", "displayDate": "1901"}
+ {"begin": "1905-04-12", "end": "1905-04-12", "displayDate": "1905-04-12"},
],
"spatial" : ["North Carolina"]}
}
@@ -438,7 +430,7 @@ def test_enrich_temporal_date():
print_error_log()
assert resp.status == 200
- url = server() + "enrich-temporal-date"
+ url = server() + "enrich_date"
resp, content = H.request(url, "POST", body=content)
print_error_log()
assert resp.status == 200
@@ -451,21 +443,14 @@ def test_enrich_date_date_parse_format_natural_string_for_multiple_dates():
"date" : "May 20, 1928; 2002-01-01"
}
EXPECTED = {
- 'date' : [
- {
+ 'date' : {
'begin': u'1928-05-20',
'end': u'1928-05-20',
'displayDate': u'May 20, 1928'
- },
- {
- 'begin': u'2002-01-01',
- 'end': u'2002-01-01',
- 'displayDate': u'2002-01-01'
- }
- ]
+ }
}
- url = server() + "enrich-date?prop=date"
+ url = server() + "enrich_earliest_date?prop=date"
resp,content = H.request(url,"POST",body=json.dumps(INPUT))
assert str(resp.status).startswith("2")
Please sign in to comment.
Something went wrong with that request. Please try again.