Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make make_validation_schema simpler and useful for extensions #412

Merged
merged 5 commits into from
May 2, 2017
Merged

Make make_validation_schema simpler and useful for extensions #412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
607ef71
Make make_validation_schema simpler and useful for extensions
kindly Aug 11, 2016
461c858
rebuild validataion schema
kindly Jan 6, 2017
2f6e56f
[#303] Ensure versioned release schema is consitently sorted
Bjwebb Feb 8, 2017
343d274
Fix create version validation schema to use new merge stratagies
kindly May 2, 2017
d712e5f
Updates to versioning and merging documentation
timgdavies May 2, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions standard/docs/en/schema/merging.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,14 +124,16 @@ A reference implementation of the merge routine in python [is available on GitHu

There are some situations in which it is important to be able to see how data about a contracting process has changed over time. For example, to identify how contract values have altered, or milestones moved through stages of implementation.

The versioned release schema provides a model for representing this data.
The [versioned release validation schema](../../versioned-release-validation-schema.json) provides a model for representing this data.

In a versioned release, instead of over-writing past values when combining multiple releases, each field becomes an array of objects, indicating the:
In a versioned release, instead of over-writing past values when combining multiple releases, each field (with the exception of the ```id``` property of objects within an array) becomes an array of objects, indicating the:

* The date, id and tag of the releases where a field-value pair was first encountered;
* The value of the field-value pair at that point;

As a result, the history of any field can be easily read from the data structure.
As a result, the history of any field can be easily read from the data structure.

The property ```"versionId":true``` is used to explicitly declare the cases where an ```id``` field **should** be versioned (i.e. within an object that is not within an array).

### Example

Expand All @@ -140,8 +142,16 @@ As a result, the history of any field can be easily read from the data structure
.. jsoninclude:: docs/en/examples/merging/versioned.json
:jsonpointer: /records/0/versionedRelease/tender/value
:expand: value, amount
:title: versioned
:title: versioned_extract

```

```eval_rst

.. jsoninclude:: docs/en/examples/merging/versioned.json
:jsonpointer:
:expand:
:title: full_versioned_file

```

8 changes: 5 additions & 3 deletions standard/schema/release-schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,8 @@
"string",
"integer"
],
"minLength": 1
"minLength": 1,
"versionId": true
},
"title": {
"title": "Tender title",
Expand Down Expand Up @@ -1401,7 +1402,8 @@
"type": [
"string",
"null"
]
],
"versionId": true
},
"name": {
"title": "Name",
Expand Down Expand Up @@ -1878,4 +1880,4 @@
}
}
}
}
}
138 changes: 63 additions & 75 deletions standard/schema/utils/make_validation_schema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,43 +3,41 @@
from collections import OrderedDict
import copy

version_template = OrderedDict({
"type": "array",
"items": {
"properties": {
"releaseDate": {
"format": "date-time",
version_template = OrderedDict([
("type", "array"),
("items", OrderedDict([
("properties", OrderedDict([
("releaseDate", OrderedDict([
("format", "date-time"),
("type", "string")
])),
("releaseID", {
"type": "string"
},
"releaseID": {
"type": "string"
},
"value": {
},
"releaseTag": {
"type": "array",
"items": {"type": "string"}
}
}
}
})
}),
("value", {
}),
("releaseTag", OrderedDict([
("type", "array"),
("items", {"type": "string"})
]))
]))
]))
])

def add_versions(schema, location=''):
for key, value in list(schema['properties'].items()):
prop_type = value.get('type')
value.pop("title", None)
value.pop("description", None)
value.pop("mergeStrategy", None)
value.pop("mergeOptions", None)
value.pop("omitWhenMerged", None)
value.pop("wholeListMerge", None)
omitWhenMerged = value.pop("omitWhenMerged", None)
wholeListMerge = value.pop("wholeListMerge", None)
versionId = value.pop("versionId", None)
if not prop_type:
continue
if key == 'id':
if location not in ("Budget", "Tender", "Classification", "Identifier"):
continue
if key == 'id' and not versionId:
continue
if prop_type == ["string", "null"] and "enum" not in value:
new_value = {}
new_value = OrderedDict()
format = value.get('format')
if format == 'uri':
new_value["$ref"] = "#/definitions/StringNullUriVersioned"
Expand All @@ -51,25 +49,14 @@ def add_versions(schema, location=''):
elif prop_type == "array":
version = copy.deepcopy(version_template)
version_properties = version["items"]["properties"]
if key in ('tenderers', 'suppliers'):
version_properties["value"] = {"type": "array",
"items": {"$ref": "#/definitions/OrganizationUnversioned"},
"uniqueItems": True}
schema['properties'][key] = version
if key == 'additionalIdentifiers':
version_properties["value"] = {"type": "array",
"items": {"$ref": "#/definitions/IdentifierUnversioned"},
"uniqueItems": True}
schema['properties'][key] = version
if key == 'additionalClassifications':
version_properties["value"] = {"type": "array",
"items": {"$ref": "#/definitions/ClassificationUnversioned"},
"uniqueItems": True}
schema['properties'][key] = version
if key == 'changes':
version_properties["value"] = {"type": "array",
"items": value["items"]}
if wholeListMerge:
new_value = copy.deepcopy(value)

if '$ref' in new_value['items']:
new_value['items']["$ref"] = value['items']['$ref'] + "Unversioned"
version_properties["value"] = new_value
schema['properties'][key] = version


elif prop_type == "object":
add_versions(value, key)
Expand All @@ -85,39 +72,37 @@ def add_versions(schema, location=''):


def add_string_definitions(schema):
for item, format in {"StringNullUriVersioned": "uri",
"StringNullDateTimeVersioned": "date-time",
"StringNullVersioned": None}.items():
for item, format in [("StringNullUriVersioned", "uri"),
("StringNullDateTimeVersioned", "date-time"),
("StringNullVersioned", None)]:
version = copy.deepcopy(version_template)
version_properties = version["items"]["properties"]
version_properties["value"] = {"type": ["string", "null"]}
version_properties["value"] = OrderedDict([("type", ["string", "null"])])
if format:
version_properties["value"]["format"] = format
schema['definitions'][item] = version

def unversion_refs(schema):
for key, value in schema.items():
if key == '$ref':
schema[key] = value + 'Unversioned'
if isinstance(value, dict):
unversion_refs(value)


def get_versioned_validation_schema(versioned_release):
versioned_release["id"] = "http://standard.open-contracting.org/schema/1__0__2/versioned-release-validation-schema.json" # nopep8
versioned_release["$schema"] = "http://json-schema.org/draft-04/schema#" # nopep8
versioned_release["title"] = "Schema for a compiled, versioned Open Contracting Release." # nopep8

definitions = versioned_release['definitions']
for key, value in definitions.items():
value.pop("title", None)
value.pop("description", None)
for prop_value in value['properties'].values():
prop_value.pop("mergeStrategy", None)
prop_value.pop("mergeOptions", None)
prop_value.pop("title", None)
prop_value.pop("description", None)
prop_value.pop("omitWhenMerged", None)
prop_value.pop("wholeListMerge", None)

OrganizationUnversioned = copy.deepcopy(definitions['Organization'])
IdentifierUnversioned = copy.deepcopy(definitions['Identifier'])
ClassificationUnversioned = copy.deepcopy(definitions['Classification'])
AddressUnversioned = copy.deepcopy(definitions['Address'])
ContactPointUnversioned = copy.deepcopy(definitions['ContactPoint'])
new_definitions = OrderedDict()
for key, value in copy.deepcopy(versioned_release['definitions']).items():
new_definitions[key + 'Unversioned'] = value

unversion_refs(new_definitions)


ocid = versioned_release['properties'].pop("ocid")
versioned_release['properties'].pop("date")
Expand All @@ -129,22 +114,25 @@ def get_versioned_validation_schema(versioned_release):
"initiationType"
]

#types_count = Counter()
#get_types(versioned_release, types_count)
add_versions(versioned_release)

versioned_release['properties']["ocid"] = ocid
definitions['IdentifierUnversioned'] = IdentifierUnversioned
definitions['ClassificationUnversioned'] = ClassificationUnversioned
definitions['AddressUnversioned'] = AddressUnversioned
definitions['ContactPointUnversioned'] = ContactPointUnversioned
OrganizationUnversioned["properties"]["identifier"]["$ref"] = "#/definitions/IdentifierUnversioned"
OrganizationUnversioned["properties"]["additionalIdentifiers"]["items"]["$ref"] = "#/definitions/IdentifierUnversioned"
OrganizationUnversioned["properties"]["address"]["$ref"] = "#/definitions/AddressUnversioned"
OrganizationUnversioned["properties"]["contactPoint"]["$ref"] = "#/definitions/ContactPointUnversioned"
definitions['OrganizationUnversioned'] = OrganizationUnversioned

definitions.update(new_definitions)
add_string_definitions(versioned_release)

for key, value in definitions.items():
value.pop("title", None)
value.pop("description", None)
if not 'properties' in value:
continue
for prop_value in value['properties'].values():
prop_value.pop("title", None)
prop_value.pop("description", None)
prop_value.pop("omitWhenMerged", None)
prop_value.pop("wholeListMerge", None)
prop_value.pop("versionId", None)

return versioned_release


Expand Down
Loading