From 766d555db16514f0fa01b79885d3e9e67e67428f Mon Sep 17 00:00:00 2001 From: Jonathan Schultz Date: Thu, 1 Jun 2017 14:25:05 +1000 Subject: [PATCH] Print diagnostic messages to stderr; removed unused imports --- AdjustDate.py | 5 ++-- DenormaliseNVP.py | 14 +++++----- DenormaliseNVPX.py | 17 ++++++------ NVP2RQDA.py | 23 ++++++++--------- NVPX2RQDA.py | 19 +++++++------- NVivo.py | 63 +++++++++++++++++++++++---------------------- NVivo2RQDADB.py | 14 +++++----- Norm2RQDA.py | 12 ++++----- NormaliseNVP.py | 12 +++++---- NormaliseNVPX.py | 17 ++++++------ NormaliseOpenQDA.py | 15 ++++++----- RQDA.py | 42 ++++++++++++++++-------------- RQDA2NVP.py | 23 ++++++++--------- RQDA2NVPX.py | 19 +++++++------- RQDA2NVivoDB.py | 14 +++++----- RQDA2Norm.py | 12 ++++----- Subtract.py | 15 ++++++----- extractTagging.py | 10 +++---- tests/testnodes.csv | 12 +++++++++ 19 files changed, 183 insertions(+), 175 deletions(-) create mode 100644 tests/testnodes.csv diff --git a/AdjustDate.py b/AdjustDate.py index c59877b..a4223c6 100755 --- a/AdjustDate.py +++ b/AdjustDate.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function from sqlalchemy import * from sqlalchemy import exc import warnings @@ -68,10 +69,10 @@ if modifiedDate <= before: modifiedDate += adjust if createdDate > datetimeNow or modifiedDate > datetimeNow: - print("WARNING: future date") + print("WARNING: future date", file=sys.stderr) if createdDate > modifiedDate: - print("WARNING: created date later than modified date") + print("WARNING: created date later than modified date", file=sys.stderr) #row['_ModifiedDate'] = row['_CreatedDate'] row['_CreatedDate'] = createdDate diff --git a/DenormaliseNVP.py b/DenormaliseNVP.py index 6febee4..be57cb5 100755 --- a/DenormaliseNVP.py +++ b/DenormaliseNVP.py @@ -16,9 +16,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse import NVivo import os +import sys import shutil import subprocess import tempfile @@ -115,7 +117,7 @@ def executescript(script, arglist=None): instancename = regquerydata[0] instanceversion = regquerydata[2].split('.')[0] if args.verbosity >= 2: - print("Found SQL server instance " + instancename + " version " + instanceversion) + print("Found SQL server instance " + instancename + " version " + instanceversion, file=sys.stderr) if (args.nvivoversion == '10' and instanceversion == 'MSSQL10_50') or (args.nvivoversion == '11' and instanceversion == 'MSSQL12'): args.instance = instancename break @@ -123,21 +125,21 @@ def executescript(script, arglist=None): raise RuntimeError('No suitable SQL server instance found') if args.verbosity > 0: - print("Using MSSQL instance: " + args.instance) + print("Using MSSQL instance: " + args.instance, file=sys.stderr) if args.port is None: regquery = executecommand(['reg', 'query', 'HKLM\\SOFTWARE\\Microsoft\\Microsoft SQL Server\\' + args.instance + '\\MSSQLServer\\SuperSocketNetLib\\Tcp']).splitlines() args.port = int(regquery[1].split()[2]) if args.verbosity > 0: - print("Using port: " + str(args.port)) + print("Using port: " + str(args.port), file=sys.stderr) # Get reasonably distinct yet recognisable DB name dbname = 'nvivo' + str(os.getpid()) executescript('mssqlAttach.bat', [tmpoutfilename, dbname, args.instance]) if args.verbosity > 0: - print("Attached database " + dbname) + print("Attached database " + dbname, file=sys.stderr) try: args.indb = 'sqlite:///' + args.infile @@ -147,7 +149,7 @@ def executescript(script, arglist=None): executescript('mssqlSave.bat', [tmpoutfilename, dbname, args.instance]) if args.verbosity > 0: - print("Saved database " + dbname) + print("Saved database " + dbname, file=sys.stderr) if args.server is None: shutil.move(tmpoutfilename, args.outfilename) @@ -160,7 +162,7 @@ def executescript(script, arglist=None): finally: executescript('mssqlDrop.bat', [dbname, args.instance]) if args.verbosity > 0: - print("Dropped database " + dbname) + print("Dropped database " + dbname, file=sys.stderr) if __name__ == '__main__': DenormaliseNVP(None) diff --git a/DenormaliseNVPX.py b/DenormaliseNVPX.py index 1410cde..c69f132 100644 --- a/DenormaliseNVPX.py +++ b/DenormaliseNVPX.py @@ -16,7 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import NVivo +import os +import sys +import shutil +from subprocess import Popen, PIPE +import tempfile parser = argparse.ArgumentParser(description='Create an NVivo for Mac file from a normalised SQLite file.') @@ -66,14 +73,6 @@ args.mac = True args.windows = False -import NVivo -import os -import shutil -import signal -from subprocess import Popen, PIPE -import tempfile -import time - tmpinfilename = tempfile.mktemp() tmpinfileptr = file(tmpinfilename, 'wb') tmpinfileptr.write(args.infile.read()) @@ -111,7 +110,7 @@ break if args.verbosity > 0: - print("Started database server on port " + freeport) + print("Started database server on port " + freeport, file=sys.stderr) args.indb = 'sqlite:///' + tmpinfilename args.outdb = 'sqlalchemy_sqlany://wiwalisataob2aaf:iatvmoammgiivaam@localhost:' + freeport + '/NVivo' + freeport diff --git a/NVP2RQDA.py b/NVP2RQDA.py index 7237d75..78684dd 100644 --- a/NVP2RQDA.py +++ b/NVP2RQDA.py @@ -16,7 +16,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import NVivo +import RQDA +import os +import sys +import shutil +import tempfile +from subprocess import Popen, PIPE parser = argparse.ArgumentParser(description='Convert an NVivo for Windows (.nvp) file into an RQDA project.') @@ -62,15 +70,6 @@ args.mac = False args.windows = True -import NVivo -import RQDA -import os -import shutil -import signal -import tempfile -import time -from subprocess import Popen, PIPE - tmpinfilename = tempfile.mktemp() tmpinfileptr = file(tmpinfilename, 'wb') tmpinfileptr.write(args.infile.read()) @@ -88,7 +87,7 @@ proc = Popen([helperpath + 'mssqlInstance.bat'], stdout=PIPE) args.instance = proc.stdout.readline()[0:-len(os.linesep)] if args.verbosity > 0: - print("Using MSSQL instance: " + args.instance) + print("Using MSSQL instance: " + args.instance, file=sys.stderr) # Get reasonably distinct yet recognisable DB name dbname = 'nt' + str(os.getpid()) @@ -96,7 +95,7 @@ proc = Popen([helperpath + 'mssqlAttach.bat', tmpinfilename, dbname, args.instance]) proc.wait() if args.verbosity > 0: - print("Attached database " + dbname) + print("Attached database " + dbname, file=sys.stderr) args.indb = 'mssql+pymssql://nvivotools:nvivotools@localhost/' + dbname args.outdb = 'sqlite:///' + tmpnormfilename @@ -106,7 +105,7 @@ proc = Popen([helperpath + 'mssqlDrop.bat', dbname, args.instance]) proc.wait() if args.verbosity > 0: - print("Dropped database " + dbname) + print("Dropped database " + dbname, file=sys.stderr) os.remove(tmpinfilename) tmpoutfilename = tempfile.mktemp() diff --git a/NVPX2RQDA.py b/NVPX2RQDA.py index f8580ef..fd0ef76 100644 --- a/NVPX2RQDA.py +++ b/NVPX2RQDA.py @@ -16,7 +16,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import NVivo +import RQDA +import os +import sys +import shutil +import tempfile +from subprocess import Popen, PIPE parser = argparse.ArgumentParser(description='Convert an NVivo for Mac (.nvpx) file into an RQDA project.') @@ -64,15 +72,6 @@ args.mac = True args.windows = False -import NVivo -import RQDA -import os -import shutil -import signal -import tempfile -import time -from subprocess import Popen, PIPE - tmpinfilename = tempfile.mktemp() tmpinfileptr = file(tmpinfilename, 'wb') tmpinfileptr.write(args.infile.read()) @@ -101,7 +100,7 @@ break if args.verbosity > 0: - print("Started database server on port " + freeport) + print("Started database server on port " + freeport, file=sys.stderr) args.indb = 'sqlalchemy_sqlany://wiwalisataob2aaf:iatvmoammgiivaam@localhost:' + freeport + '/NVivo' + freeport args.outdb = 'sqlite:///' + tmpnormfilename diff --git a/NVivo.py b/NVivo.py index 04bf2d6..c32d685 100644 --- a/NVivo.py +++ b/NVivo.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function from builtins import chr from sqlalchemy import * from sqlalchemy import exc @@ -308,7 +309,7 @@ def Normalise(args): # Users if args.users != 'skip': if args.verbosity > 0: - print("Normalising users") + print("Normalising users", file=sys.stderr) users = [dict(row) for row in nvivodb.execute(select([ nvivoUserProfile.c.Id, @@ -320,7 +321,7 @@ def Normalise(args): # Project if args.project != 'skip': if args.verbosity > 0: - print("Normalising project") + print("Normalising project", file=sys.stderr) project = dict(nvivodb.execute(select([ nvivoProject.c.Title, @@ -355,7 +356,7 @@ def Normalise(args): # Node Categories if args.node_categories != 'skip': if args.verbosity > 0: - print("Normalising node categories") + print("Normalising node categories", file=sys.stderr) nodecategories = [dict(row) for row in nvivodb.execute(select([ nvivoItem.c.Id, @@ -384,7 +385,7 @@ def Normalise(args): # Nodes if args.nodes != 'skip': if args.verbosity > 0: - print("Normalising nodes") + print("Normalising nodes", file=sys.stderr) nvivoCategoryRole = nvivoRole.alias(name='CategoryRole') nvivoParentRole = nvivoRole.alias(name='ParentRole') @@ -429,7 +430,7 @@ def Normalise(args): # Node attributes if args.node_attributes != 'skip': if args.verbosity > 0: - print("Normalising node attributes") + print("Normalising node attributes", file=sys.stderr) nvivoNodeItem = nvivoItem.alias(name='NodeItem') nvivoNameItem = nvivoItem.alias(name='NameItem') @@ -521,7 +522,7 @@ def Normalise(args): # Source categories if args.source_categories != 'skip': if args.verbosity > 0: - print("Normalising source categories") + print("Normalising source categories", file=sys.stderr) sourcecats = [dict(row) for row in nvivodb.execute(select([ nvivoItem.c.Id, @@ -549,7 +550,7 @@ def Normalise(args): # Sources if args.sources != 'skip': if args.verbosity > 0: - print("Normalising sources") + print("Normalising sources", file=sys.stderr) nvivoCategoryRole = nvivoRole.alias(name='CategoryRole') nvivoParentRole = nvivoRole.alias(name='ParentRole') @@ -612,7 +613,7 @@ def Normalise(args): # Source attributes if args.source_attributes != 'skip': if args.verbosity > 0: - print("Normalising source attributes") + print("Normalising source attributes", file=sys.stderr) nvivoNameItem = nvivoItem.alias(name='NameItem') nvivoNameRole = nvivoRole.alias(name='NameRole') @@ -742,7 +743,7 @@ def build_tagging_or_annotation(item): if args.taggings != 'skip': if args.verbosity > 0: - print("Normalising taggings") + print("Normalising taggings", file=sys.stderr) taggings = [dict(row) for row in nvivodb.execute(select([ nvivoNodeReference.c.Id, @@ -771,7 +772,7 @@ def build_tagging_or_annotation(item): # Annotations if args.annotations != 'skip': if args.verbosity > 0: - print("Normalising annotations") + print("Normalising annotations", file=sys.stderr) annotations = [dict(row) for row in nvivodb.execute(select([ nvivoAnnotation.c.Id, @@ -868,7 +869,7 @@ def Denormalise(args): # Users if args.users != 'skip': if args.verbosity > 0: - print("Denormalising users") + print("Denormalising users", file=sys.stderr) users = [dict(row) for row in normdb.execute(select([ normUser.c.Id, @@ -935,7 +936,7 @@ def Denormalise(args): notapplicablelabel = u''.join(map(lambda ch: chr(ord(ch) + 0x377), notapplicablelabel)) if args.project != 'skip': - print("Denormalising project") + print("Denormalising project", file=sys.stderr) project['Description'] = project['Description'] or u'' if args.windows: @@ -975,7 +976,7 @@ def itemname(id): def skip_merge_or_overwrite_categories(normtable, itemtype, name, operation): if operation != 'skip': if args.verbosity > 0: - print('Denormalising ' + name.lower() + ' categories') + print('Denormalising ' + name.lower() + ' categories', file=sys.stderr) # Look up head category headcategoryname = name.title() + u' Classifications' if args.windows: @@ -994,7 +995,7 @@ def skip_merge_or_overwrite_categories(normtable, itemtype, name, operation): raise RuntimeError("NVivo file contains no head " + name + " category.") else: if args.verbosity > 1: - print("Found head " + name + " category Id: " + str(headcategory['Id'])) + print("Found head " + name + " category Id: " + str(headcategory['Id']), file=sys.stderr) categories = [dict(row) for row in normdb.execute(select([ normtable.c.Id, normtable.c.Name, @@ -1063,7 +1064,7 @@ def skip_merge_or_overwrite_categories(normtable, itemtype, name, operation): # Nodes if args.nodes != 'skip': if args.verbosity > 0: - print("Denormalising nodes") + print("Denormalising nodes", file=sys.stderr) # Look up head node headnodename = u'Nodes' @@ -1083,7 +1084,7 @@ def skip_merge_or_overwrite_categories(normtable, itemtype, name, operation): raise RuntimeError("NVivo file contains no head node.") else: if args.verbosity > 1: - print("Found head node Id: " + str(headnode['Id'])) + print("Found head node Id: " + str(headnode['Id']), file=sys.stderr) nodes = [dict(row) for row in normdb.execute(select([ normNode.c.Id, @@ -1157,7 +1158,7 @@ def tagchildnodes(TopParent, Parent, AggregateList, depth): nodestoinsert = [node for node in nodes if not {'_Id':node['Id']} in curids] if args.verbosity > 1: for node in nodestoinsert: - print "Inserting node: " + node['PlainTextName'] + print("Inserting node: " + node['PlainTextName'], file=sys.stderr) tagchildnodes(None, None, [], 0) aggregatepairs = [] @@ -1384,7 +1385,7 @@ def skip_merge_or_overwrite_attributes(attributes, values, name, operation): if curitem is not None: # Need to adjust every instance of this attribute/category combination if args.verbosity > 1: - print("Duplicating " + name + " attribute '" + attribute['PlainTextName'] + "' for category '" + itemname(value['Category']) + "' with tag: " + str(maxattributetags[value['Category']])) + print("Duplicating " + name + " attribute '" + attribute['PlainTextName'] + "' for category '" + itemname(value['Category']) + "' with tag: " + str(maxattributetags[value['Category']]), file=sys.stderr) attribute = attribute.copy() attributes += [attribute] attribute['Id'] = uuid.uuid4() @@ -1394,7 +1395,7 @@ def skip_merge_or_overwrite_attributes(attributes, values, name, operation): valueiter['Attribute'] = attribute['Id'] else: if args.verbosity > 1: - print("Creating " + name + " attribute '" + attribute['PlainTextName'] + "' for category '" + itemname(value['Category']) + "' with tag: " + str(maxattributetags[value['Category']])) + print("Creating " + name + " attribute '" + attribute['PlainTextName'] + "' for category '" + itemname(value['Category']) + "' with tag: " + str(maxattributetags[value['Category']]), file=sys.stderr) attribute['Tag'] = maxattributetags[value['Category']] attribute['Category'] = value['Category'] @@ -1562,7 +1563,7 @@ def skip_merge_or_overwrite_attributes(attributes, values, name, operation): maxvaluetags[categoryattribute] = (maxvaluetags[categoryattribute] or -1) + 1 value['Tag'] = maxvaluetags[categoryattribute] if args.verbosity > 1: - print("Creating value '" + value['PlainTextValue'] + "' for " + name + " attribute '" + attribute['PlainTextName'] + "' with tag: "+ str(value['Tag'])) + print("Creating value '" + value['PlainTextValue'] + "' for " + name + " attribute '" + attribute['PlainTextName'] + "' with tag: "+ str(value['Tag']), file=sys.stderr) value['Id'] = uuid.uuid4() nvivocon.execute(nvivoItem.insert().values({ @@ -1593,7 +1594,7 @@ def skip_merge_or_overwrite_attributes(attributes, values, name, operation): value.update(valuestatus) if valuestatus['ExistingValueId'] is not None: if args.verbosity > 1: - print("Deassigning existing value '" + itemname(value['ExistingValueId']) + "' from " + name + " attribute '" + attribute['PlainTextName'] + "' of " + name + " '" + itemname(value['Item']) + "'") + print("Deassigning existing value '" + itemname(value['ExistingValueId']) + "' from " + name + " attribute '" + attribute['PlainTextName'] + "' of " + name + " '" + itemname(value['Item']) + "'", file=sys.stderr) nvivocon.execute(nvivoRole.delete(and_( nvivoRole.c.Item1_Id == bindparam('Item'), nvivoRole.c.Item2_Id == bindparam('ExistingValueId'), @@ -1601,7 +1602,7 @@ def skip_merge_or_overwrite_attributes(attributes, values, name, operation): )), value ) if args.verbosity > 1: - print("Assigning value '" + value['PlainTextValue'] + "' to " + name + " attribute '" + attribute['PlainTextName'] + "' of " + name + " '" + itemname(value['Item']) + "'") + print("Assigning value '" + value['PlainTextValue'] + "' to " + name + " attribute '" + attribute['PlainTextName'] + "' of " + name + " '" + itemname(value['Item']) + "'", file=sys.stderr) nvivocon.execute(nvivoRole.insert().values({ 'Item1_Id': bindparam('Item'), 'Item2_Id': bindparam('NewValueId'), @@ -1613,7 +1614,7 @@ def skip_merge_or_overwrite_attributes(attributes, values, name, operation): # Set value of undefined attribute to 'Unassigned' attributes = [dict(row) for row in nvivocon.execute(missingvaluesel, addedattribute)] if len(attributes) > 0 and args.verbosity > 1: - print("Assigning default value '" + itemname(addedattribute['DefaultValueId']) + "' to attribute '" + itemname(addedattribute['Attribute']) + "' of " + str(len(attributes)) + " " + name + "(s).") + print("Assigning default value '" + itemname(addedattribute['DefaultValueId']) + "' to attribute '" + itemname(addedattribute['Attribute']) + "' of " + str(len(attributes)) + " " + name + "(s).", file=sys.stderr) for attribute in attributes: attribute.update(addedattribute) @@ -1628,7 +1629,7 @@ def skip_merge_or_overwrite_attributes(attributes, values, name, operation): # Node attributes if args.node_attributes != 'skip': if args.verbosity > 0: - print("Denormalising node attributes") + print("Denormalising node attributes", file=sys.stderr) attributes = [dict(row) for row in normdb.execute(select([ normNodeAttribute.c.Id, @@ -1725,7 +1726,7 @@ def rebuild_category_records(itemtype): # Function to massage source data def massagesource(source): if args.verbosity > 1: - print("Source: " + source['Name']) + print("Source: " + source['Name'], file=sys.stderr) source['Item_Id'] = source['Item_Id'] or uuid.uuid4() source['Description'] = source['Description'] or u'' source['PlainTextName'] = source['Name'] @@ -1836,7 +1837,7 @@ def massagesource(source): # Look for unoconvcmd just once if massagesource.unoconvcmd is None: if args.verbosity > 1: - print("Searching for unoconv executable.") + print("Searching for unoconv executable.", file=sys.stderr) # Look first on path for OS installed version, otherwise use our copy for path in os.environ["PATH"].split(os.pathsep): unoconvpath = os.path.join(path, 'unoconv') @@ -1966,7 +1967,7 @@ def massagesource(source): # Sources if args.sources != 'skip': if args.verbosity > 0: - print("Denormalising sources") + print("Denormalising sources", file=sys.stderr) # Look up head source headsourcename = u'Internals' @@ -1985,7 +1986,7 @@ def massagesource(source): raise RuntimeError("NVivo file contains no head Internal source.") else: if args.verbosity > 1: - print("Found head source Id: " + str(headsource['Id'])) + print("Found head source Id: " + str(headsource['Id']), file=sys.stderr) sources = [dict(row) for row in normdb.execute(select([ normSource.c.Id.label('Item_Id'), @@ -2100,7 +2101,7 @@ def massagesource(source): # Source attributes if args.source_attributes != 'skip': if args.verbosity > 0: - print("Denormalising source attributes") + print("Denormalising source attributes", file=sys.stderr) attributes = [dict(row) for row in normdb.execute(select([ normSourceAttribute.c.Id, @@ -2131,7 +2132,7 @@ def massagesource(source): # Taggings and annotations if args.taggings != 'skip' or args.annotations != 'skip': if args.verbosity > 0: - print("Denormalising taggings and/or annotations") + print("Denormalising taggings and/or annotations", file=sys.stderr) sources = [dict(row) for row in nvivocon.execute(select([ nvivoSource.c.Item_Id, @@ -2161,7 +2162,7 @@ def massagesource(source): tagging['ClusterId'] = None matchfragment = re.match("([0-9]+):([0-9]+)(?:,([0-9]+)(?::([0-9]+))?)?", tagging['Fragment']) if matchfragment is None: - print("WARNING: Unrecognised tagging fragment: " + tagging['Fragment'] + " for Source: " + itemname(tagging['Source']) ) + print("WARNING: Unrecognised tagging fragment: " + tagging['Fragment'] + " for Source: " + itemname(tagging['Source']) , file=sys.stderr) taggings.remove(tagging) continue diff --git a/NVivo2RQDADB.py b/NVivo2RQDADB.py index 0769f1e..291b218 100755 --- a/NVivo2RQDADB.py +++ b/NVivo2RQDADB.py @@ -16,7 +16,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import NVivo +import RQDA +import os +import shutil +import tempfile parser = argparse.ArgumentParser(description='Convert an NVivo project to an RQDA project.') @@ -59,14 +65,6 @@ args = parser.parse_args() -import NVivo -import RQDA -import os -import shutil -import signal -import tempfile -import time - if args.outrqdadb is None: args.outrqdadb = args.innvivodb.rsplit('.',1)[0] + '.rqda' diff --git a/Norm2RQDA.py b/Norm2RQDA.py index e49b2f1..4993dfb 100755 --- a/Norm2RQDA.py +++ b/Norm2RQDA.py @@ -16,7 +16,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import RQDA +import os +import shutil +import tempfile parser = argparse.ArgumentParser(description='Convert a normalised NVivo project to RQDA.') @@ -50,13 +55,6 @@ args = parser.parse_args() -import RQDA -import os -import shutil -import signal -import tempfile -import time - tmpinfilename = tempfile.mktemp() tmpinfileptr = file(tmpinfilename, 'wb') tmpinfileptr.write(args.infile.read()) diff --git a/NormaliseNVP.py b/NormaliseNVP.py index 026dcba..a55da54 100755 --- a/NormaliseNVP.py +++ b/NormaliseNVP.py @@ -16,9 +16,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse import NVivo import os +import sys import shutil import subprocess import tempfile @@ -111,7 +113,7 @@ def executescript(script, arglist=None): instancename = regquerydata[0] instanceversion = regquerydata[2].split('.')[0] if args.verbosity >= 2: - print("Found SQL server instance " + instancename + " version " + instanceversion) + print("Found SQL server instance " + instancename + " version " + instanceversion, file=sys.stderr) if (args.nvivoversion == '10' and instanceversion == 'MSSQL10_50') or (args.nvivoversion == '11' and instanceversion == 'MSSQL12'): args.instance = instancename break @@ -119,21 +121,21 @@ def executescript(script, arglist=None): raise RuntimeError('No suitable SQL server instance found') if args.verbosity > 0: - print("Using MSSQL instance: " + args.instance) + print("Using MSSQL instance: " + args.instance, file=sys.stderr) if args.port is None: regquery = executecommand(['reg', 'query', 'HKLM\\SOFTWARE\\Microsoft\\Microsoft SQL Server\\' + args.instance + '\\MSSQLServer\\SuperSocketNetLib\\Tcp']).splitlines() args.port = int(regquery[1].split()[2]) if args.verbosity > 0: - print("Using port: " + str(args.port)) + print("Using port: " + str(args.port), file=sys.stderr) # Get reasonably distinct yet recognisable DB name dbname = 'nvivo' + str(os.getpid()) executescript('mssqlAttach.bat', [infilename, dbname, args.instance]) if args.verbosity > 0: - print("Attached database " + dbname) + print("Attached database " + dbname, file=sys.stderr) try: args.indb = 'mssql+pymssql://nvivotools:nvivotools@' + (args.server or 'localhost') + ((':' + str(args.port)) if args.port else '') + '/' + dbname @@ -149,7 +151,7 @@ def executescript(script, arglist=None): finally: executescript('mssqlDrop.bat', [dbname, args.instance]) if args.verbosity > 0: - print("Dropped database " + dbname) + print("Dropped database " + dbname, file=sys.stderr) if __name__ == '__main__': NormaliseNVP(None) diff --git a/NormaliseNVPX.py b/NormaliseNVPX.py index 9fc41f2..424e843 100644 --- a/NormaliseNVPX.py +++ b/NormaliseNVPX.py @@ -16,7 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import NVivo +import os +import sys +import shutil +from subprocess import Popen, PIPE +import tempfile parser = argparse.ArgumentParser(description='Normalise an NVivo for Mac file.') @@ -62,14 +69,6 @@ args.mac = True args.windows = False -import NVivo -import os -import shutil -import signal -from subprocess import Popen, PIPE -import tempfile -import time - tmpinfilename = tempfile.mktemp() tmpinfileptr = file(tmpinfilename, 'wb') tmpinfileptr.write(args.infile.read()) @@ -98,7 +97,7 @@ break if args.verbosity > 0: - print("Started database server on port " + freeport) + print("Started database server on port " + freeport, file=sys.stderr) args.indb = 'sqlalchemy_sqlany://wiwalisataob2aaf:iatvmoammgiivaam@localhost:' + freeport + '/NVivo' + freeport args.outdb = 'sqlite:///' + tmpoutfilename diff --git a/NormaliseOpenQDA.py b/NormaliseOpenQDA.py index 91933a7..c6b56b0 100755 --- a/NormaliseOpenQDA.py +++ b/NormaliseOpenQDA.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function from builtins import chr from sqlalchemy import * from sqlalchemy import exc @@ -199,7 +200,7 @@ # Users if args.users != 'skip': - print("Normalising users") + print("Normalising users", file=sys.stderr) # Collect unique users from images and imageCoding tables userlist = [row.owner for row in oqdadb.execute(select([oqdaimageCoding.c.owner]))] + \ @@ -217,7 +218,7 @@ # Project #if args.project != 'skip': if True: - print("Normalising project") + print("Normalising project", file=sys.stderr) # Get earliest and latest timestamp dateset = set([row.date for row in oqdadb.execute(select([oqdaimageCoding.c.date]))] + @@ -233,7 +234,7 @@ # Nodes if args.nodes != 'skip': - print("Normalising nodes") + print("Normalising nodes", file=sys.stderr) sel = select([oqdacodes.c.id, oqdacodes.c.name, @@ -260,7 +261,7 @@ # Sources if args.sources != 'skip': - print("Normalising sources") + print("Normalising sources", file=sys.stderr) # Create dummy source category sourcecatuuid = uuid.uuid4() @@ -294,7 +295,7 @@ source['uuid'] = uuid.uuid4() source['Category'] = sourcecatuuid source['ObjectType'] = 'JPEG' - print("Downloading " + source['name']) + print("Downloading " + source['name'], file=sys.stderr) if args.user != None: opener.open(args.url + source['name']) urllib2.install_opener(opener) @@ -316,7 +317,7 @@ # Source attributes if args.source_attributes != 'skip': - print("Normalising source attributes") + print("Normalising source attributes", file=sys.stderr) sel = select([oqdaimageAttributes.c.images_id, oqdaimageAttributes.c.value, @@ -342,7 +343,7 @@ # Tagging if args.taggings != 'skip': - print("Normalising taggings") + print("Normalising taggings", file=sys.stderr) sel = select([oqdaimageCoding.c.images_id, oqdaimageCoding.c.codes_id, diff --git a/RQDA.py b/RQDA.py index 37d80a1..01b259f 100644 --- a/RQDA.py +++ b/RQDA.py @@ -16,9 +16,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function from sqlalchemy import * from sqlalchemy import exc import os +import sys import re from datetime import date, time, datetime from dateutil import parser as dateparser @@ -277,7 +279,7 @@ def Norm2RQDA(args): # Project if args.project != 'skip': if args.verbosity > 0: - print("Converting project") + print("Converting project", file=sys.stderr) curproject = rqdacon.execute(select([rqdaproject.c.memo])).first() if curproject is not None and args.project == 'overwrite': @@ -306,7 +308,7 @@ def Norm2RQDA(args): # Source categories if args.source_categories != 'skip': if args.verbosity > 0: - print("Converting source categories") + print("Converting source categories", file=sys.stderr) sourcecats = [dict(row) for row in normdb.execute(select([ normSourceCategory.c.Id.label('Uuid'), @@ -335,7 +337,7 @@ def Norm2RQDA(args): # Sources if args.sources != 'skip': if args.verbosity > 0: - print("Converting sources") + print("Converting sources", file=sys.stderr) sources = [dict(row) for row in normdb.execute(select([ normSource.c.Id.label('Uuid'), @@ -424,12 +426,12 @@ def create_or_test_attribute(value): }) else: if attribute['class'] != attrclass: - print("WARNING: Inconsistent type for attribute: " + value['variable']) + print("WARNING: Inconsistent type for attribute: " + value['variable'], file=sys.stderr) # Source attributes if args.source_attributes != 'skip': if args.verbosity > 0: - print("Converting source attributes") + print("Converting source attributes", file=sys.stderr) normAttributeUser = normUser.alias(name='AttributeUser') sourcevalues = [dict(row) for row in normdb.execute(select([ @@ -465,7 +467,7 @@ def create_or_test_attribute(value): # Node categories if args.node_categories != 'skip': if args.verbosity > 0: - print("Converting node categories") + print("Converting node categories", file=sys.stderr) codecats = [dict(row) for row in normdb.execute(select([ normNodeCategory.c.Id.label('Uuid'), @@ -496,7 +498,7 @@ def create_or_test_attribute(value): # Nodes if args.nodes != 'skip': if args.verbosity > 0: - print("Converting nodes") + print("Converting nodes", file=sys.stderr) # Nodes without any attributes are mapped to RQDA codes, those with attributes # are mapped to RQDA cases. @@ -626,7 +628,7 @@ def create_or_test_attribute(value): # Tagging if args.taggings != 'skip': if args.verbosity > 0: - print("Converting taggings") + print("Converting taggings", file=sys.stderr) taggings = [dict(row) for row in normdb.execute(select([ normTagging.c.Id, @@ -652,7 +654,7 @@ def create_or_test_attribute(value): tagging['status'] = 1 matchfragment = re.match("([0-9]+):([0-9]+)(?:,([0-9]+)(?::([0-9]+))?)?", tagging['Fragment']) if matchfragment is None: - print("WARNING: Unrecognised tagging fragment: " + tagging['Fragment'] + " for Source: " + sourcename[tagging['SourceUuid']]) + print("WARNING: Unrecognised tagging fragment: " + tagging['Fragment'] + " for Source: " + sourcename[tagging['SourceUuid']], file=sys.stderr) continue if tagging['Node'] is None: @@ -922,7 +924,7 @@ def find_or_create_user(name): # Project if args.project != 'skip': if args.verbosity > 0: - print("Converting project") + print("Converting project", file=sys.stderr) project = dict(rqdadb.execute(select([ rqdaproject.c.databaseversion, @@ -949,7 +951,7 @@ def find_or_create_user(name): # Source categories if args.source_categories != 'skip': if args.verbosity > 0: - print("Converting source categories") + print("Converting source categories", file=sys.stderr) sourcecats = [dict(row) for row in rqdadb.execute(select([ rqdafilecat.c.catid, @@ -977,7 +979,7 @@ def find_or_create_user(name): # Sources if args.sources != 'skip': if args.verbosity > 0: - print("Converting sources") + print("Converting sources", file=sys.stderr) sources = [dict(row) for row in rqdadb.execute(select([ rqdasource.c.id.label('fid'), @@ -1010,7 +1012,7 @@ def find_or_create_user(name): 'fid': source['fid'] })] if len(sourcecats) > 1: - print("WARNING: Source: " + source['Name'] + " belongs to more than one category. Only first category will be retained") + print("WARNING: Source: " + source['Name'] + " belongs to more than one category. Only first category will be retained", file=sys.stderr) if len(sourcecats) > 0: source['Category'] = sourcecatuuid[sourcecats[0]['catid']] @@ -1020,7 +1022,7 @@ def find_or_create_user(name): # Source attributes if args.source_attributes != 'skip': if args.verbosity > 0: - print("Converting source attributes") + print("Converting source attributes", file=sys.stderr) sourcevalues = [dict(row) for row in rqdadb.execute(select([ rqdafileAttr.c.variable.label('Name'), @@ -1078,7 +1080,7 @@ def find_or_create_user(name): # Node categories if args.node_categories != 'skip': if args.verbosity > 0: - print("Converting node categories") + print("Converting node categories", file=sys.stderr) nodecats = [dict(row) for row in rqdadb.execute(select([ rqdacodecat.c.catid, @@ -1106,7 +1108,7 @@ def find_or_create_user(name): # Nodes if args.nodes != 'skip': if args.verbosity > 0: - print("Converting nodes") + print("Converting nodes", file=sys.stderr) nodes = [dict(row) for row in rqdadb.execute(select([ rqdafreecode.c.id.label('cid'), @@ -1136,7 +1138,7 @@ def find_or_create_user(name): 'cid': node['cid'] })] if len(nodecats) > 1: - print("WARNING: Node: " + node['Name'] + " belongs to more than one category. Only first category will be retained") + print("WARNING: Node: " + node['Name'] + " belongs to more than one category. Only first category will be retained", file=sys.stderr) if len(nodecats) > 0: node['Category'] = nodecatuuid[nodecats[0]['catid']] @@ -1146,7 +1148,7 @@ def find_or_create_user(name): # Cases if args.cases != 'skip': if args.verbosity > 0: - print("Converting cases") + print("Converting cases", file=sys.stderr) cases = [dict(row) for row in rqdadb.execute(select([ rqdacases.c.id.label('caseid'), @@ -1174,7 +1176,7 @@ def find_or_create_user(name): # Case attributes if args.case_attributes != 'skip': if args.verbosity > 0: - print("Converting case attributes") + print("Converting case attributes", file=sys.stderr) casevalues = [dict(row) for row in rqdadb.execute(select([ rqdacaseAttr.c.variable.label('Name'), @@ -1265,7 +1267,7 @@ def find_or_create_user(name): # Annotations, codings and case linkages if args.taggings != 'skip': if args.verbosity > 0: - print("Converting annotations, codings and case linkages") + print("Converting annotations, codings and case linkages", file=sys.stderr) taggings = [] diff --git a/RQDA2NVP.py b/RQDA2NVP.py index e9aa522..6afdb70 100644 --- a/RQDA2NVP.py +++ b/RQDA2NVP.py @@ -16,7 +16,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import NVivo +import RQDA +import os +import sys +import shutil +import tempfile +from subprocess import Popen, PIPE parser = argparse.ArgumentParser(description='Convert an RQDA project to NVivo for Windows (.nvp) format.') @@ -65,15 +73,6 @@ args.mac = False args.windows = True -import NVivo -import RQDA -import os -import shutil -import signal -import tempfile -import time -from subprocess import Popen, PIPE - tmpinfilename = tempfile.mktemp() tmpinfileptr = file(tmpinfilename, 'wb') tmpinfileptr.write(args.infile.read()) @@ -106,7 +105,7 @@ proc = Popen([helperpath + 'mssqlInstance.bat'], stdout=PIPE) args.instance = proc.stdout.readline()[0:-len(os.linesep)] if args.verbosity > 0: - print("Using MSSQL instance: " + args.instance) + print("Using MSSQL instance: " + args.instance, file=sys.stderr) # Get reasonably distinct yet recognisable DB name dbname = 'nt' + str(os.getpid()) @@ -114,7 +113,7 @@ proc = Popen([helperpath + 'mssqlAttach.bat', tmpoutfilename, dbname, args.instance]) proc.wait() if args.verbosity > 0: - print("Attached database " + dbname) + print("Attached database " + dbname, file=sys.stderr) args.indb = 'sqlite:///' + tmpnormfilename args.outdb = 'mssql+pymssql://nvivotools:nvivotools@localhost/' + dbname @@ -127,7 +126,7 @@ proc = Popen([helperpath + 'mssqlSave.bat', tmpoutfilename, dbname, args.instance]) proc.wait() if args.verbosity > 0: - print("Saved database " + dbname) + print("Saved database " + dbname, file=sys.stderr) shutil.move(tmpoutfilename, args.outfilename) os.remove(tmpnormfilename) diff --git a/RQDA2NVPX.py b/RQDA2NVPX.py index 327bae0..9a77d8b 100644 --- a/RQDA2NVPX.py +++ b/RQDA2NVPX.py @@ -16,7 +16,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import NVivo +import RQDA +import os +import sys +import shutil +import tempfile +from subprocess import Popen, PIPE parser = argparse.ArgumentParser(description='Convert an RQDA project to NVivo for Mac (.nvpx) format.') @@ -67,15 +75,6 @@ args.mac = True args.windows = False -import NVivo -import RQDA -import os -import shutil -import signal -import tempfile -import time -from subprocess import Popen, PIPE - tmpinfilename = tempfile.mktemp() tmpinfileptr = file(tmpinfilename, 'wb') tmpinfileptr.write(args.infile.read()) @@ -119,7 +118,7 @@ break if args.verbosity > 0: - print("Started database server on port " + freeport) + print("Started database server on port " + freeport, file=sys.stderr) args.indb = 'sqlite:///' + tmpnormfilename args.outdb = 'sqlalchemy_sqlany://wiwalisataob2aaf:iatvmoammgiivaam@localhost:' + freeport + '/NVivo' + freeport diff --git a/RQDA2NVivoDB.py b/RQDA2NVivoDB.py index 17f3087..9aa0dce 100755 --- a/RQDA2NVivoDB.py +++ b/RQDA2NVivoDB.py @@ -16,7 +16,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import NVivo +import RQDA +import os +import shutil +import tempfile parser = argparse.ArgumentParser(description='Convert an RQDA project to NVivo format.') @@ -59,14 +65,6 @@ args = parser.parse_args() -import NVivo -import RQDA -import os -import shutil -import signal -import tempfile -import time - if args.outnvivodb is None: args.outnvivodb = args.inrqdadb.rsplit('.',1)[0] + '.nvivo' diff --git a/RQDA2Norm.py b/RQDA2Norm.py index 8edc3b2..cd6087a 100755 --- a/RQDA2Norm.py +++ b/RQDA2Norm.py @@ -16,7 +16,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse +import RQDA +import os +import shutil +import tempfile parser = argparse.ArgumentParser(description='Convert an RQDA project to normalised NVivo format.') @@ -52,13 +57,6 @@ args = parser.parse_args() -import RQDA -import os -import shutil -import signal -import tempfile -import time - tmpinfilename = tempfile.mktemp() tmpinfileptr = file(tmpinfilename, 'wb') tmpinfileptr.write(args.infile.read()) diff --git a/Subtract.py b/Subtract.py index c419c21..5a5a5f4 100755 --- a/Subtract.py +++ b/Subtract.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function from sqlalchemy import exc, TypeDecorator, CHAR, String, create_engine, MetaData, bindparam from sqlalchemy.engine import reflection import warnings @@ -56,7 +57,7 @@ for minuendtable in minuendmd.sorted_tables: if args.difference != None: if minuendtable.name not in differencemd.tables.keys(): - print("Creating table: " + minuendtable.name) + print("Creating table: " + minuendtable.name, file=sys.stderr) minuendtable.create(differenceconn) for minuendtable in minuendmd.sorted_tables: @@ -78,18 +79,18 @@ if len(differencerows) > 0: if args.difference != None: - print("Finding foreign key references for table " + minuendtable.name) + print("Finding foreign key references for table " + minuendtable.name, file=sys.stderr) for fk in inspector.get_foreign_keys(minuendtable.name): if not fk['name']: continue - #print(" " + fk['name']) + #print(" " + fk['name'], file=sys.stderr) fkreferredtable = minuendmd.tables[fk['referred_table']] fkselect = fkreferredtable.select() for referred_column, constrained_column in zip(fk['referred_columns'], fk['constrained_columns']): fkselect = fkselect.where(fkreferredtable.c[referred_column] == bindparam(constrained_column)) - #print(fkselect) + #print(fkselect, file=sys.stderr) fkrows = [] fkexists = [] @@ -103,13 +104,13 @@ fkinsert = [ x for x in fkrows if not x in fkexists ] if len(fkinsert) > 0: differencereferredtable = differencemd.tables[fk['referred_table']] - #print( "fkinsert: " + str(fkinsert)) differenceconn.execute(differencereferredtable.insert(), fkinsert) + #print( "fkinsert: " + str(fkinsert)) differenceconn.execute(differencereferredtable.insert(), fkinsert, file=sys.stderr) differenceconn.execute(minuendtable.insert(), differencerows) else: - print("-------------- " + minuendtable.name + " --------------") + print("-------------- " + minuendtable.name + " --------------", file=sys.stderr) for row in differencerows: - print(row) + print(row, file=sys.stderr) # All done. diff --git a/extractTagging.py b/extractTagging.py index be9d19b..3535167 100755 --- a/extractTagging.py +++ b/extractTagging.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import os import sys import argparse @@ -23,7 +24,6 @@ from sqlalchemy import exc import re - exec(open(os.path.dirname(os.path.realpath(__file__)) + os.path.sep + 'DataTypes.py').read()) @@ -76,15 +76,15 @@ }) for tagging in taggings: - print("Node: " + tagging['NodeName'] + " Source: " + tagging['SourceName'] + "[" + tagging['Fragment'] + "]") + print("Node: " + tagging['NodeName'] + " Source: " + tagging['SourceName'] + "[" + tagging['Fragment'] + "]", file=sys.stderr) matchfragment = re.match("([0-9]+):([0-9]+)(?:,([0-9]+)(?::([0-9]+))?)?", tagging['Fragment']) if matchfragment is None: - print("WARNING: Unrecognised tagging fragment") + print("WARNING: Unrecognised tagging fragment", file=sys.stderr) else: - print(tagging['Content'][int(matchfragment.group(1)):int(matchfragment.group(2))+1]) + print(tagging['Content'][int(matchfragment.group(1)):int(matchfragment.group(2))+1], file=sys.stderr) - print("") + print("", file=sys.stderr) normdb.dispose() diff --git a/tests/testnodes.csv b/tests/testnodes.csv new file mode 100644 index 0000000..aeaf4ab --- /dev/null +++ b/tests/testnodes.csv @@ -0,0 +1,12 @@ +Name,Parent,Description,Category,Aggregate,Color,integer attribute,Decimal attribute,Boolean attribute,Date attribute,Time attribute,Datetime attribute,Text attribute +With attributes,,Node with all attribute types,All attribute types,,,123,123.456,TRUE,1967-12-10,16:20:00,01/03/16 09:15,Now is the time… +Top level no category,,,,,,,,,,,, +Second level not aggregating no category,Top level no category,,,,,,,,,,, +Third level not aggregating no category,Second level not aggregating no category,,,,,,,,,,, +Second level aggregating no category,Top level no category,,,TRUE,,,,,,,, +Third level aggregating no category,Second level aggregating no category,,,TRUE,,,,,,,, +Top level with category,,,Random category,,,,,,,,, +Second level not aggregating with category,Top level with category,,Random category,,,,,,,,, +Third level not aggregating with category,Second level not aggregating with category,,Random category,,,,,,,,, +Second level aggregating with category,Top level with category,,Random category,TRUE,,,,,,,, +Third level aggregating with category,Second level aggregating with category,,Random category,TRUE,,,,,,,,