Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
...
  • 20 commits
  • 3 files changed
  • 0 commit comments
  • 2 contributors
Commits on Mar 13, 2013
@migbot migbot Fix for coverage string
- Certain modules may expect the spatial field to be a list, so
  added a spatial_transform to oai-to-dpla
68fa190
@migbot migbot Added contributor, language, and temporal to qa_reports
-Updated export_reports script to account for additionl fields.
8cd0aea
@migbot migbot Updated qa_reports to work with creator['name'] 55157e8
Commits on Mar 14, 2013
@migbot migbot Updated qa_reports.js
-The creator field is not a dictionary. Removed "name" key.
87db045
Szymon Guz Added initial smithsonian profile. 9edd0d8
Szymon Guz Changed smithsonian profile 595a95d
Szymon Guz Modified the smithsonian profile. a67d0b8
Szymon Guz First working version, can ingest smithsonian data.
However not all files, as it increases load to 17 and takes about 8GB of
memory.
5e178b2
Szymon Guz First working version of ingesting smithsonian. cbd549e
Szymon Guz Added comments, removed comments. 0c1b557
Szymon Guz Fixed a bug in poll_profiles 22ceec8
Szymon Guz Updated the smithsonian profile. 7ef0797
Szymon Guz Added first draft of edan-to-dpla module for smithsonian ingestion. 577029b
Szymon Guz First working draft of edan-to-dpla 987c34e
Szymon Guz Further changes in edan-to-dpla. 92c7db3
Szymon Guz Renamed edan-to-dpla module. 2f9b53f
Szymon Guz First working draft of edan ingestion. 6c0c243
Szymon Guz Cleared the edan-to-dpla a little bit. 503cd1c
Szymon Guz Added other modules to the smithsonian profile. 1aae6a3
Szymon Guz Merge branch 'smithsonian_ingestion' of github.com:dpla/ingestion int…
…o smithsonian_ingestion
2df55e1
View
21 couchdb_views/qa_reports.js
@@ -99,6 +99,27 @@
"collection_count": {
"map": "function(doc) {if (doc.ingestType == 'item') {cname = doc.collection.name;emit(cname,1);}}",
"reduce": "_count"
+ },
+ "contributor": {
+ "map": "function(doc) {if (doc.ingestType == 'item') {contributor = doc.aggregatedCHO.contributor;if (contributor.constructor.toString().indexOf('Array') == -1) { contributor = new Array(contributor); }for (i=0; i<contributor.length; i++) {emit(doc['id'], contributor[i]);}}}"
+ },
+ "contributor_count": {
+ "map": "function(doc) {if (doc.ingestType == 'item') {contributor = doc.aggregatedCHO.contributor;if (contributor.constructor.toString().indexOf('Array') == -1) { contributor = new Array(contributor); }for (i=0; i<contributor.length; i++) {emit(contributor[i],1);}}}",
+ "reduce": "_count"
+ },
+ "language": {
+ "map": "function(doc) {if (doc.ingestType == 'item') {language = doc.aggregatedCHO.language;if (language.constructor.toString().indexOf('Array') == -1) { language = new Array(language); }for (i=0; i<language.length; i++) {emit(doc['id'], language[i]['name']);}}}"
+ },
+ "language_count": {
+ "map": "function(doc) {if (doc.ingestType == 'item') {language = doc.aggregatedCHO.language;if (language.constructor.toString().indexOf('Array') == -1) { language = new Array(language); }for (i=0; i<language.length; i++) {emit(language[i]['name'],1);}}}",
+ "reduce": "_count"
+ },
+ "temporal": {
+ "map": "function(doc) {if (doc.ingestType == 'item') {temporal = doc.aggregatedCHO.temporal;if (temporal.constructor.toString().indexOf('Array') == -1) { temporal = new Array(temporal); }for (i=0; i<temporal.length; i++) {emit(doc['id'], temporal[i]['displayDate']+' ('+d[i]['begin']+' to '+d[i]['end']+')');}}}"
+ },
+ "temporal_count": {
+ "map": "function(doc) {if (doc.ingestType == 'item') {temporal = doc.aggregatedCHO.temporal;if (temporal.constructor.toString().indexOf('Array') == -1) { temporal = new Array(temporal); }for (i=0; i<temporal.length; i++) {emit(temporal[i]['displayDate']+' ('+d[i]['begin']+' to '+d[i]['end']+')',1);}}}",
+ "reduce": "_count"
}
},
"lists": {
View
9 lib/akamod/oai-to-dpla.py
@@ -57,11 +57,18 @@ def is_shown_at_transform(d):
}
}
+def spatial_transform(d):
+ spatial = d["coverage"]
+ if spatial and not isinstance(spatial, list):
+ spatial = [spatial]
+
+ return {"spatial": spatial} if spatial else {}
+
# Structure mapping the original property to a function returning a single
# item dict representing the new property and its value
CHO_TRANSFORMER = {
"contributor" : lambda d: {"contributor": d.get("contributor",None)},
- "coverage" : lambda d: {"spatial": d.get("coverage",None)},
+ "coverage" : spatial_transform,
"creator" : lambda d: {"creator": d.get("creator",None)},
"description" : lambda d: {"description": d.get("description",None)},
"date" : lambda d: {"date": d.get("date",None)},
View
15 scripts/export_reports
@@ -24,12 +24,15 @@ report[4]="rights"
report[5]="subject"
report[6]="creator"
report[7]="provider"
-report[8]="publisher"
-report[9]="collection"
-report[10]="description"
-report[11]="spatial_name"
-report[12]="spatial_state"
-report[13]="data_provider"
+report[8]="language"
+report[9]="temporal"
+report[10]="publisher"
+report[11]="collection"
+report[12]="contributor"
+report[13]="description"
+report[14]="spatial_name"
+report[15]="spatial_state"
+report[16]="data_provider"
dir="${provider}_exports"
rm -r $dir "${dir}.zip"

No commit comments for this range

Something went wrong with that request. Please try again.