Skip to content

Commit

Permalink
Export using sorted tuples instead of dict
Browse files Browse the repository at this point in the history 
To preserve order of exported XML, for stable diffs.

This changes the export format, so exports need to be re-done.
Here's how I did it:

{{{py

import glob
dirname = 'src/Products.CMFPlomino/Products/CMFPlomino/tests/samples/'
databases = glob.glob(os.path.join(dirname, '*.xml'))

def import_databases():
    for fn in databases:
        print 'importing', fn
        f = open(fn)
        i = os.path.splitext(os.path.basename(fn))[0]
        portal.invokeFactory('PlominoDatabase', id=i)
        db = getattr(portal, i)
        db.at_post_create_script()
        wf_tool = portal.portal_workflow
        wf_tool.doActionFor(db, 'publish')
        xmlstring=f.read()
        db.importDesignFromXML(xmlstring)
        db.refreshDB()

import_databases()

databases = [
    "src/Products.CMFPlomino/Products/CMFPlomino/tests/filestoimport/devplomino.xml",
    "src/Products.CMFPlomino/Products/CMFPlomino/tests/filestoimport/hideWhenTest.xml",
    "src/Products.CMFPlomino/Products/CMFPlomino/tests/filestoimport/multilineDatagridTest.xml",
    "src/Products.CMFPlomino/Products/CMFPlomino/tests/filestoimport/replicadb.xml",
    "src/Products.CMFPlomino/Products/CMFPlomino/tests/filestoimport/with_datagrid_and_overlay.xml",
    ]

import_databases()

xmlstring = open("src/Products.CMFPlomino/Products/CMFPlomino/tests/filestoimport/replicadb-all-documents.xml").read()
print 'importing', fn
portal.replicadb.importFromXML(xmlstring=xmlstring)

import glob
dirname = 'src/Products.CMFPlomino/Products/CMFPlomino/tests/filestoimport/'
databases = glob.glob(os.path.join(dirname, '*.xml'))

def export_databases():
    for fn in databases:
        print 'exporting', fn
        i = os.path.splitext(os.path.basename(fn))[0]
        db = getattr(portal, i, None)
        if not db:
            continue
        xmlstring = db.exportDesignAsXML(dbsettings=False)
        f = open(os.path.basename(fn), 'w')
        f.write(xmlstring)
        f.close()

export_databases()

dirname = 'src/Products.CMFPlomino/Products/CMFPlomino/tests/samples/'
databases = glob.glob(os.path.join(dirname, '*.xml'))

export_databases()

ss = portal.replicadb.exportAsXML()
f = open(os.path.join(dirname, 'replicadb-all-documents.xml'), 'w')
f.write(ss)
f.close()
  • Loading branch information
@jean
jean committed Oct 27, 2013
1 parent 9d1c1d1 commit 6adf113
Show file tree
Hide file tree
Showing 22 changed files with 2,870 additions and 3,028 deletions.
54 changes: 24 additions & 30 deletions Products/CMFPlomino/PlominoDesignManager.py
View file
Expand Up @@ -79,9 +79,6 @@
from Products.CMFPlomino.PlominoView import schema as view_schema
from Products.CMFPlomino import get_resource_directory

# 3rd-party
from jsonutil import jsonutil as json

plomino_schemas = {
'PlominoAction': action_schema,
'PlominoAgent': agent_schema,
Expand Down Expand Up @@ -1040,19 +1037,13 @@ def exportElementAsXML(self, xmldoc, obj, isDatabase=False):
if field_parameters:
# Preserve order in exports for stable diffs
field_parameters = tuple(sorted(field_parameters.items()))
parameters_json = json.dumps(field_parameters)
parameters = xmldoc.createElement('parameters')
str_items = xmlrpclib.dumps(field_parameters, allow_none=1)
try:
parameters.appendChild(xmldoc.createCDATASection(parameters_json))
dom_items = parseString(str_items)
except ExpatError:
parameters.appendChild(xmldoc.createCDATASection(escape_xml_illegal_chars(parameters_json)))
node.appendChild(parameters)
# try:
# dom_items = parseString(str_items)
# except ExpatError:
# dom_items = parseString(
# escape_xml_illegal_chars(str_items))
# node.appendChild(dom_items.documentElement)
dom_items = parseString(
escape_xml_illegal_chars(str_items))
node.appendChild(dom_items.documentElement)
if not isDatabase:
elementslist = obj.objectIds()
if elementslist:
Expand Down Expand Up @@ -1301,24 +1292,28 @@ def importElementFromXML(self, container, node):
if subchild.nodeType == subchild.ELEMENT_NODE:
self.importElementFromXML(obj, subchild)
subchild = subchild.nextSibling
elif name == 'parameters':
elif name == 'params':
# current object is a field, the params tag contains the
# specific settings
# parameters = json.loads(node.toxml().encode('utf-8'))
parameters = json.loads(child.firstChild.data.encode('utf-8'))
for key, v in parameters:
result, method = xmlrpclib.loads(node.toxml().encode('utf-8'))
try:
parameters = dict(result)
except:
import pdb; pdb.set_trace()
for key in parameters.keys():
v = parameters[key]
if v is not None:
# if hasattr(v, 'encode'):
# v = unicode(v)
# else:
# if hasattr(v, 'append'):
# uv = []
# for e in v:
# if hasattr(e, 'encode'):
# uv.append(unicode(e))
# else:
# uv.append(e)
# v = uv
if hasattr(v, 'encode'):
v = unicode(v)
else:
if hasattr(v, 'append'):
uv = []
for e in v:
if hasattr(e, 'encode'):
uv.append(unicode(e))
else:
uv.append(e)
v = uv
settings_values[key] = v
elif name == "CustomData":
# Only one non.text child is expected
Expand Down Expand Up @@ -1498,4 +1493,3 @@ def importTemplateElement(self, source):
xml = source.readFile(name)
self.importDesignFromXML(xmlstring=xml)


36 changes: 14 additions & 22 deletions Products/CMFPlomino/PlominoReplicationManager.py
View file
Expand Up @@ -45,9 +45,6 @@
from Products.CMFPlomino.PlominoUtils import escape_xml_illegal_chars
from Products.CMFPlomino.PlominoUtils import plomino_decimal

# 3rd-party
from jsonutil import jsonutil as json

REMOTE_DOC_ID_SEPARATOR = '#'
REMOTE_DOC_DATE_SEPARATOR = '@'
REMOTE_DOC_IDS_HEADER = 'REMOTE_DOC_IDS'
Expand Down Expand Up @@ -1351,18 +1348,12 @@ def exportDocumentAsXML(self, xmldoc, doc):
# export items
# Preserve order in exports for stable diffs
items = tuple(sorted(doc.items.items()))
str_items = json.dumps(items)
items = xmldoc.createElement('items')
str_items = xmlrpclib.dumps(items, allow_none=True)
try:
items.appendChild(xmldoc.createCDATASection(str_items))
dom_items = parseString(str_items)
except ExpatError:
items.appendChild(xmldoc.createCDATASection(escape_xml_illegal_chars(str_items)))
node.appendChild(items)
# try:
# dom_items = parseString(str_items)
# except ExpatError:
# dom_items = parseString(escape_xml_illegal_chars(str_items))
# node.appendChild(dom_items.documentElement)
dom_items = parseString(escape_xml_illegal_chars(str_items))
node.appendChild(dom_items.documentElement)

# export attached files
for f in doc.getFilenames():
Expand Down Expand Up @@ -1512,16 +1503,17 @@ def importDocumentFromXML(self, node):
doc = self.createDocument(docid)

# restore items
itemnode = node.getElementsByTagName("items")[0]
itemnode = node.getElementsByTagName("params")[0]
#result, method = xmlrpclib.loads(node.firstChild.toxml())
items = json.loads(itemnode.firstChild.data.encode('utf-8'))
# for k in items.keys():
# # convert xmlrpclib.DateTime into DateTime
# if items[k].__class__.__name__ == 'DateTime':
# # XXX: import loses timezone data
# items[k] = StringToDate(
# items[k].value[:19],
# format="%Y-%m-%dT%H:%M:%S")
result, method = xmlrpclib.loads(itemnode.toxml().encode('utf-8'))
items = dict(result)
for k in items.keys():
# convert xmlrpclib.DateTime into DateTime
if items[k].__class__.__name__ == 'DateTime':
# XXX: import loses timezone data
items[k] = StringToDate(
items[k].value[:19],
format="%Y-%m-%dT%H:%M:%S")
doc.items = PersistentDict(items)

# restore files
Expand Down

4 comments on commit 6adf113

@jean
Copy link
Member Author

@jean jean commented on 6adf113 Oct 27, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ebrehault this is a noisy commit, and migration of old databases is laborious.
But being confronted with unstable diffs on every commit, due to dictionary key ordering, is also a huge pain, clutters commit logs, and makes it easy to miss things.
Maybe we should keep both code paths around during 1.18.*, and choose based on database version in the XML.

@fulv
Copy link

@fulv fulv commented on 6adf113 Oct 27, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1

@jean
Copy link
Member Author

@jean jean commented on 6adf113 Oct 28, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm I see comments were stripped from the migration code sample I gave above. The import part should be done on Plomino <= 1.18.1, and the export after upgrade to > 1.18.1.

I'd like to change the commit message as described here:
http://darrinholst.com/post/359817782.html

Is this OK to do on an unmerged branch?

@jean
Copy link
Member Author

@jean jean commented on 6adf113 Oct 28, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rebased, see cbc0c25 for new version ..

Please sign in to comment.