From ace7b13deb2b801b8397120b4a3f6d83a8f5fdf9 Mon Sep 17 00:00:00 2001 From: Alex Kalderimis Date: Wed, 28 Nov 2012 12:34:48 +0000 Subject: [PATCH] Added files --- COPYING.txt | 57 + MANIFEST | 63 + MANIFEST.in | 6 + README.md | 111 + create_docs.sh | 1 + intermine/__init__.py | 0 intermine/constraints.py | 1131 ++++++++++ intermine/errors.py | 12 + intermine/lists/__init__.py | 1 + intermine/lists/list.py | 399 ++++ intermine/lists/listmanager.py | 342 +++ intermine/model.py | 958 +++++++++ intermine/pathfeatures.py | 113 + intermine/query.py | 1871 +++++++++++++++++ intermine/results.py | 683 ++++++ intermine/util.py | 26 + intermine/webservice.py | 488 +++++ samples/alleles.py | 54 + setup.py | 164 ++ tests/__init__.py | 0 tests/acceptance.py | 17 + tests/data/test-identifiers.list | 6 + tests/live_lists.py | 373 ++++ tests/live_registry.py | 20 + tests/live_results.py | 101 + tests/live_summary_test.py | 46 + tests/test.py | 1263 +++++++++++ tests/test_lists.py | 53 + tests/test_templates.py | 155 ++ tests/testserver.py | 67 + tests/testservice/countservice/service/model | 94 + .../countservice/service/query/results | 1 + .../countservice/service/template/results | 1 + .../countservice/service/templates/xml | 128 ++ .../countservice/service/version/release | 1 + .../countservice/service/version/ws | 1 + tests/testservice/csvservice/service/model | 94 + .../csvservice/service/query/results | 2 + .../csvservice/service/template/results | 2 + .../csvservice/service/templates/xml | 128 ++ .../csvservice/service/version/release | 1 + .../testservice/csvservice/service/version/ws | 1 + tests/testservice/legacyjsonrows/lists/json | 29 + tests/testservice/legacyjsonrows/model | 94 + .../testservice/legacyjsonrows/query/results | 5 + .../legacyjsonrows/template/results | 5 + .../testservice/legacyjsonrows/templates/xml | 128 ++ .../legacyjsonrows/version/release | 1 + tests/testservice/legacyjsonrows/version/ws | 1 + tests/testservice/service/lists/json | 29 + tests/testservice/service/model | 94 + tests/testservice/service/query/results | 5 + tests/testservice/service/template/results | 5 + tests/testservice/service/templates/xml | 128 ++ tests/testservice/service/version/release | 1 + tests/testservice/service/version/ws | 1 + .../testservice/testresultobjs/service/model | 94 + .../testresultobjs/service/query/results | 10 + .../testresultobjs/service/template/results | 10 + .../testresultobjs/service/version/release | 1 + .../testresultobjs/service/version/ws | 1 + tests/testservice/tsvservice/service/model | 94 + .../tsvservice/service/query/results | 2 + .../tsvservice/service/template/results | 2 + .../tsvservice/service/templates/xml | 128 ++ .../tsvservice/service/version/release | 1 + .../testservice/tsvservice/service/version/ws | 1 + tox.ini | 24 + 68 files changed, 9929 insertions(+) create mode 100644 COPYING.txt create mode 100644 MANIFEST create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 create_docs.sh create mode 100644 intermine/__init__.py create mode 100644 intermine/constraints.py create mode 100644 intermine/errors.py create mode 100644 intermine/lists/__init__.py create mode 100644 intermine/lists/list.py create mode 100644 intermine/lists/listmanager.py create mode 100644 intermine/model.py create mode 100644 intermine/pathfeatures.py create mode 100644 intermine/query.py create mode 100644 intermine/results.py create mode 100644 intermine/util.py create mode 100644 intermine/webservice.py create mode 100644 samples/alleles.py create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/acceptance.py create mode 100644 tests/data/test-identifiers.list create mode 100644 tests/live_lists.py create mode 100644 tests/live_registry.py create mode 100644 tests/live_results.py create mode 100644 tests/live_summary_test.py create mode 100644 tests/test.py create mode 100644 tests/test_lists.py create mode 100644 tests/test_templates.py create mode 100644 tests/testserver.py create mode 100644 tests/testservice/countservice/service/model create mode 100644 tests/testservice/countservice/service/query/results create mode 100644 tests/testservice/countservice/service/template/results create mode 100644 tests/testservice/countservice/service/templates/xml create mode 100644 tests/testservice/countservice/service/version/release create mode 100644 tests/testservice/countservice/service/version/ws create mode 100644 tests/testservice/csvservice/service/model create mode 100644 tests/testservice/csvservice/service/query/results create mode 100644 tests/testservice/csvservice/service/template/results create mode 100644 tests/testservice/csvservice/service/templates/xml create mode 100644 tests/testservice/csvservice/service/version/release create mode 100644 tests/testservice/csvservice/service/version/ws create mode 100644 tests/testservice/legacyjsonrows/lists/json create mode 100644 tests/testservice/legacyjsonrows/model create mode 100644 tests/testservice/legacyjsonrows/query/results create mode 100644 tests/testservice/legacyjsonrows/template/results create mode 100644 tests/testservice/legacyjsonrows/templates/xml create mode 100644 tests/testservice/legacyjsonrows/version/release create mode 100644 tests/testservice/legacyjsonrows/version/ws create mode 100644 tests/testservice/service/lists/json create mode 100644 tests/testservice/service/model create mode 100644 tests/testservice/service/query/results create mode 100644 tests/testservice/service/template/results create mode 100644 tests/testservice/service/templates/xml create mode 100644 tests/testservice/service/version/release create mode 100644 tests/testservice/service/version/ws create mode 100644 tests/testservice/testresultobjs/service/model create mode 100644 tests/testservice/testresultobjs/service/query/results create mode 100644 tests/testservice/testresultobjs/service/template/results create mode 100644 tests/testservice/testresultobjs/service/version/release create mode 100644 tests/testservice/testresultobjs/service/version/ws create mode 100644 tests/testservice/tsvservice/service/model create mode 100644 tests/testservice/tsvservice/service/query/results create mode 100644 tests/testservice/tsvservice/service/template/results create mode 100644 tests/testservice/tsvservice/service/templates/xml create mode 100644 tests/testservice/tsvservice/service/version/release create mode 100644 tests/testservice/tsvservice/service/version/ws create mode 100644 tox.ini diff --git a/COPYING.txt b/COPYING.txt new file mode 100644 index 00000000..fb839977 --- /dev/null +++ b/COPYING.txt @@ -0,0 +1,57 @@ +GNU LESSER GENERAL PUBLIC LICENSE + +Version 3, 29 June 2007 + +Copyright © 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + +This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. + +0. Additional Definitions. +As used herein, “this License” refers to version 3 of the GNU Lesser General Public License, and the “GNU GPL” refers to version 3 of the GNU General Public License. + +“The Library” refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. + +An “Application” is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. + +A “Combined Work” is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the “Linked Version”. + +The “Minimal Corresponding Source” for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. + +The “Corresponding Application Code” for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. + +1. Exception to Section 3 of the GNU GPL. +You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. + +2. Conveying Modified Versions. +If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: + +a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or +b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. +3. Object Code Incorporating Material from Library Header Files. +The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: + +a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. +b) Accompany the object code with a copy of the GNU GPL and this license document. +4. Combined Works. +You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: + +a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. +b) Accompany the Combined Work with a copy of the GNU GPL and this license document. +c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. +d) Do one of the following: +0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. +1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. +e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) +5. Combined Libraries. +You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: + +a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. +b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. +6. Revised Versions of the GNU Lesser General Public License. +The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. + +If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. diff --git a/MANIFEST b/MANIFEST new file mode 100644 index 00000000..fb0ce1a6 --- /dev/null +++ b/MANIFEST @@ -0,0 +1,63 @@ +# file GENERATED by distutils, do NOT edit +COPYING.txt +setup.py +intermine/__init__.py +intermine/constraints.py +intermine/errors.py +intermine/model.py +intermine/pathfeatures.py +intermine/query.py +intermine/results.py +intermine/util.py +intermine/webservice.py +intermine/lists/__init__.py +intermine/lists/list.py +intermine/lists/listmanager.py +tests/__init__.py +tests/acceptance.py +tests/live_lists.py +tests/live_registry.py +tests/live_results.py +tests/live_summary_test.py +tests/test.py +tests/test_lists.py +tests/test_templates.py +tests/testserver.py +tests/data/test-identifiers.list +tests/testservice/countservice/service/model +tests/testservice/countservice/service/query/results +tests/testservice/countservice/service/template/results +tests/testservice/countservice/service/templates/xml +tests/testservice/countservice/service/version/release +tests/testservice/countservice/service/version/ws +tests/testservice/csvservice/service/model +tests/testservice/csvservice/service/query/results +tests/testservice/csvservice/service/template/results +tests/testservice/csvservice/service/templates/xml +tests/testservice/csvservice/service/version/release +tests/testservice/csvservice/service/version/ws +tests/testservice/legacyjsonrows/model +tests/testservice/legacyjsonrows/lists/json +tests/testservice/legacyjsonrows/query/results +tests/testservice/legacyjsonrows/template/results +tests/testservice/legacyjsonrows/templates/xml +tests/testservice/legacyjsonrows/version/release +tests/testservice/legacyjsonrows/version/ws +tests/testservice/service/model +tests/testservice/service/lists/json +tests/testservice/service/query/results +tests/testservice/service/template/results +tests/testservice/service/templates/xml +tests/testservice/service/version/release +tests/testservice/service/version/ws +tests/testservice/testresultobjs/service/model +tests/testservice/testresultobjs/service/query/results +tests/testservice/testresultobjs/service/template/results +tests/testservice/testresultobjs/service/version/release +tests/testservice/testresultobjs/service/version/ws +tests/testservice/tsvservice/service/model +tests/testservice/tsvservice/service/query/results +tests/testservice/tsvservice/service/template/results +tests/testservice/tsvservice/service/templates/xml +tests/testservice/tsvservice/service/version/release +tests/testservice/tsvservice/service/version/ws diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..7704e19f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include COPYING.txt +recursive-include docs *.html *.css *.png +recursive-include tests *.py +recursive-include tests/data * +recursive-include tests/testservice * +recursive-include intermine *.py diff --git a/README.md b/README.md new file mode 100644 index 00000000..02e50220 --- /dev/null +++ b/README.md @@ -0,0 +1,111 @@ +The InterMine Python Webservice Client +===================================== + +> An implementation of a webservice client +> for InterMine webservices, written in Python + +Who should use this software? +----------------------------- + +This software is intended for people who make +use of InterMine datawarehouses (ie. Biologists) +and who want a quicker, more automated way +to perform queries. Some examples of sites that +are powered by InterMine software, and thus offer +a compatible webservice API are: + +* FlyMine +* MetabolicMine +* modMine +* RatMine +* YeastMine + +Queries here refer to database queries over the +integrated datawarehouse. Instead of using +SQL, InterMine services use a flexible and +powerful sub-set of database query language +to enable wide-ranging and arbitrary queries. + +Downloading: +------------ + +The easiest way to install is to use easy_install: + + sudo easy_install intermine + +The source code can be downloaded from a variety of places: + +* From InterMine + + wget http://www.intermine.org/lib/python-webservice-client-0.96.00.tar.gz + +* From PyPi + + wget http://pypi.python.org/packages/source/i/intermine/intermine-0.96.00.tar.gz + +* From Github + + git clone git://github.com/alexkalderimis/intermine-ws-python.git + + +Running the Tests: +------------------ + +If you would like to run the test suite, you can do so by executing +the following command: (from the source directory) + + python setup.py test + +Installation: +------------- + +Once downloaded, you can install the module with the command (from the source directory): + + python setup.py install + +Further documentation: +---------------------- + +Extensive documentation is available by using the "pydoc" command, eg: + + pydoc intermine.query.Query + +Also see: + +* Documentation on PyPi: http://packages.python.org/intermine/ +* Documentation at InterMine: http://www.flymine.org/download/python-docs http://www.intermine.org/wiki/PythonClient + +License: +-------- + +All InterMine code is freely available under the LGPL license: http://www.gnu.org/licenses/lgpl.html + +Changes: +-------- + +0.98.13: Added query column summary support +0.98.14: Added status property to list objects +0.98.15: Added lazy-reference fetching for result objects, and list-tagging support +0.98.16: Fixed bug with XML parsing and subclasses where the subclass is mentioned in the first view. + better result format documentation and tests + added len() to results iterators + added ability to parse xml from the service object (see new_query()) + improved service.select() - now accepts plain class names which work equally well for results and lists + Allowed lists to be generated from queries with unambiguous selected classes. + Fixed questionable constraint parsing bug which lead to failed template parsing +0.99.00 Fixed bug with subclasses not being included in clones + Added support for new json format for ws versions >= 8. +0.99.01 Better representation of multiple sort-orders. +0.99.02 Allow sort-orders which are not in the view but are on selected classes +0.99.03 Allow query construction from Columns with "where" and "filter" + Allow list and query objects as the value in an add_constraint call with "IN" and "NOT IN" operators. + Ensure lists and queries share the same overloading +0.99.04 Merged 'list.to_query and 'list.to_attribute_query' in response to the changes in list upload behaviour. +0.99.05 Allow template parameters of the form 'A = "zen"', where only the value is being replaced. +0.99.06 Fixed bug whereby constraint codes in xml were being ignored when queries were deserialised. +0.99.07 Wed Jan 18 14:42:41 GMT 2012 + Fixed bugs with lazy reference fetching handling empty collections and null references. +0.99.08 Added simpler constraint definition with kwargs. + + + diff --git a/create_docs.sh b/create_docs.sh new file mode 100644 index 00000000..3c00240b --- /dev/null +++ b/create_docs.sh @@ -0,0 +1 @@ +epydoc --html -n "InterMine Python Webservice Client" -u "http://www.intermine.org" -v --exclude="intermine.intermine" --output docs intermine diff --git a/intermine/__init__.py b/intermine/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/intermine/constraints.py b/intermine/constraints.py new file mode 100644 index 00000000..4dd1429d --- /dev/null +++ b/intermine/constraints.py @@ -0,0 +1,1131 @@ +import re +import string +from intermine.pathfeatures import PathFeature, PATH_PATTERN +from intermine.util import ReadableException + +class Constraint(PathFeature): + """ + A class representing constraints on a query + =========================================== + + All constraints inherit from this class, which + simply defines the type of element for the + purposes of serialisation. + """ + child_type = "constraint" + +class LogicNode(object): + """ + A class representing nodes in a logic graph + =========================================== + + Objects which can be represented as nodes + in the AST of a constraint logic graph should + inherit from this class, which defines + methods for overloading built-in operations. + """ + + def __add__(self, other): + """ + Overloads + + =========== + + Logic may be defined by using addition to sum + logic nodes:: + + > query.set_logic(con_a + con_b + con_c) + > str(query.logic) + ... A and B and C + + """ + if not isinstance(other, LogicNode): + return NotImplemented + else: + return LogicGroup(self, 'AND', other) + + def __and__(self, other): + """ + Overloads & + =========== + + Logic may be defined by using the & operator:: + + > query.set_logic(con_a & con_b) + > sr(query.logic) + ... A and B + + """ + if not isinstance(other, LogicNode): + return NotImplemented + else: + return LogicGroup(self, 'AND', other) + + def __or__(self, other): + """ + Overloads | + =========== + + Logic may be defined by using the | operator:: + + > query.set_logic(con_a | con_b) + > str(query.logic) + ... A or B + + """ + if not isinstance(other, LogicNode): + return NotImplemented + else: + return LogicGroup(self, 'OR', other) + +class LogicGroup(LogicNode): + """ + A logic node that represents two sub-nodes joined in some way + ============================================================= + + A logic group is a logic node with two child nodes, which are + either connected by AND or by OR logic. + """ + + LEGAL_OPS = frozenset(['AND', 'OR']) + + def __init__(self, left, op, right, parent=None): + """ + Constructor + =========== + + Makes a new node composes of two nodes (left and right), + and some operator. + + Groups may have a reference to their parent. + """ + if not op in self.LEGAL_OPS: + raise TypeError(op + " is not a legal logical operation") + self.parent = parent + self.left = left + self.right = right + self.op = op + for node in [self.left, self.right]: + if isinstance(node, LogicGroup): + node.parent = self + + def __repr__(self): + """ + Provide a sensible representation of a node + """ + return '<' + self.__class__.__name__ + ': ' + str(self) + '>' + + def __str__(self): + """ + Provide a human readable version of the group. The + string version should be able to be parsed back into the + original logic group. + """ + core = ' '.join(map(str, [self.left, self.op.lower(), self.right])) + if self.parent and self.op != self.parent.op: + return '(' + core + ')' + else: + return core + + def get_codes(self): + """ + Get a list of all constraint codes used in this group. + """ + codes = [] + for node in [self.left, self.right]: + if isinstance(node, LogicGroup): + codes.extend(node.get_codes()) + else: + codes.append(node.code) + return codes + +class LogicParseError(ReadableException): + """ + An error representing problems in parsing constraint logic. + """ + pass + +class EmptyLogicError(ValueError): + """ + An error representing the fact that an the logic string to be parsed was empty + """ + pass + +class LogicParser(object): + """ + Parses logic strings into logic groups + ====================================== + + Instances of this class are used to parse logic strings into + abstract syntax trees, and then logic groups. This aims to provide + robust parsing of logic strings, with the ability to identify syntax + errors in such strings. + """ + + def __init__(self, query): + """ + Constructor + =========== + + Parsers need access to the query they are parsing for, in + order to reference the constraints on the query. + + @param query: The parent query object + @type query: intermine.query.Query + """ + self._query = query + + def get_constraint(self, code): + """ + Get the constraint with the given code + ====================================== + + This method fetches the constraint from the + parent query with the matching code. + + @see: intermine.query.Query.get_constraint + @rtype: intermine.constraints.CodedConstraint + """ + return self._query.get_constraint(code) + + def get_priority(self, op): + """ + Get the priority for a given operator + ===================================== + + Operators have a specific precedence, from highest + to lowest: + - () + - AND + - OR + + This method returns an integer which can be + used to compare operator priorities. + + @rtype: int + """ + return { + "AND": 2, + "OR" : 1, + "(" : 3, + ")" : 3 + }.get(op) + + ops = { + "AND" : "AND", + "&" : "AND", + "&&" : "AND", + "OR" : "OR", + "|" : "OR", + "||" : "OR", + "(" : "(", + ")" : ")" + } + + def parse(self, logic_str): + """ + Parse a logic string into an abstract syntax tree + ================================================= + + Takes a string such as "A and B or C and D", and parses it + into a structure which represents this logic as a binary + abstract syntax tree. The above string would parse to + "(A and B) or (C and D)", as AND binds more tightly than OR. + + Note that only singly rooted trees are parsed. + + @param logic_str: The logic defininition as a string + @type logic_str: string + + @rtype: LogicGroup + + @raise LogicParseError: if there is a syntax error in the logic + """ + def flatten(l): + """Flatten out a list which contains both values and sublists""" + ret = [] + for item in l: + if isinstance(item, list): + ret.extend(item) + else: + ret.append(item) + return ret + def canonical(x, d): + if x in d: + return d[x] + else: + return re.split("\b", x) + def dedouble(x): + if re.search("[()]", x): + return list(x) + else: + return x + + logic_str = logic_str.upper() + tokens = [t for t in re.split("\s+", logic_str) if t] + if not tokens: + raise EmptyLogicError() + tokens = flatten([canonical(x, self.ops) for x in tokens]) + tokens = flatten([dedouble(x) for x in tokens]) + self.check_syntax(tokens) + postfix_tokens = self.infix_to_postfix(tokens) + abstract_syntax_tree = self.postfix_to_tree(postfix_tokens) + return abstract_syntax_tree + + def check_syntax(self, infix_tokens): + """ + Check the syntax for errors before parsing + ========================================== + + Syntax is checked before parsing to provide better errors, + which should hopefully lead to more informative error messages. + + This checks for: + - correct operator positions (cannot put two codes next to each other without intervening operators) + - correct grouping (all brackets are matched, and contain valid expressions) + + @param infix_tokens: The input parsed into a list of tokens. + @type infix_tokens: iterable + + @raise LogicParseError: if there is a problem. + """ + need_an_op = False + need_binary_op_or_closing_bracket = False + processed = [] + open_brackets = 0 + for token in infix_tokens: + if token not in self.ops: + if need_an_op: + raise LogicParseError("Expected an operator after: '" + ' '.join(processed) + "'" + + " - but got: '" + token + "'") + if need_binary_op_or_closing_bracket: + raise LogicParseError("Logic grouping error after: '" + ' '.join(processed) + "'" + + " - expected an operator or a closing bracket") + + need_an_op = True + else: + need_an_op = False + if token == "(": + if processed and processed[-1] not in self.ops: + raise LogicParseError("Logic grouping error after: '" + ' '.join(processed) + "'" + + " - got an unexpeced opening bracket") + if need_binary_op_or_closing_bracket: + raise LogicParseError("Logic grouping error after: '" + ' '.join(processed) + "'" + + " - expected an operator or a closing bracket") + + open_brackets += 1 + elif token == ")": + need_binary_op_or_closing_bracket = True + open_brackets -= 1 + else: + need_binary_op_or_closing_bracket = False + processed.append(token) + if open_brackets != 0: + if open_brackets < 0: + message = "Unmatched closing bracket in: " + else: + message = "Unmatched opening bracket in: " + raise LogicParseError(message + '"' + ' '.join(infix_tokens) + '"') + + def infix_to_postfix(self, infix_tokens): + """ + Convert a list of infix tokens to postfix notation + ================================================== + + Take in a set of infix tokens and return the set parsed + to a postfix sequence. + + @param infix_tokens: The list of tokens + @type infix_tokens: iterable + + @rtype: list + """ + stack = [] + postfix_tokens = [] + for token in infix_tokens: + if token not in self.ops: + postfix_tokens.append(token) + else: + op = token + if op == "(": + stack.append(token) + elif op == ")": + while stack: + last_op = stack.pop() + if last_op == "(": + if stack: + previous_op = stack.pop() + if previous_op != "(": postfix_tokens.append(previous_op) + break + else: + postfix_tokens.append(last_op) + else: + while stack and self.get_priority(stack[-1]) <= self.get_priority(op): + prev_op = stack.pop() + if prev_op != "(": postfix_tokens.append(prev_op) + stack.append(op) + while stack: postfix_tokens.append(stack.pop()) + return postfix_tokens + + def postfix_to_tree(self, postfix_tokens): + """ + Convert a set of structured tokens to a single LogicGroup + ========================================================= + + Convert a set of tokens in postfix notation to a single + LogicGroup object. + + @param postfix_tokens: A list of tokens in postfix notation. + @type postfix_tokens: list + + @rtype: LogicGroup + + @raise AssertionError: is the tree doesn't have a unique root. + """ + stack = [] + try: + for token in postfix_tokens: + if token not in self.ops: + stack.append(self.get_constraint(token)) + else: + op = token + right = stack.pop() + left = stack.pop() + stack.append(LogicGroup(left, op, right)) + assert len(stack) == 1, "Tree doesn't have a unique root" + return stack.pop() + except IndexError: + raise EmptyLogicError() + +class EmptyLogicError (IndexError): + pass + +class CodedConstraint(Constraint, LogicNode): + """ + A parent class for all constraints that have codes + ================================================== + + Constraints that have codes are the principal logical + filters on queries, and need to be refered to individually + (hence the codes). They will all have a logical operation they + embody, and so have a reference to an operator. + + This class is not meant to be instantiated directly, but instead + inherited from to supply default behaviour. + """ + + OPS = set([]) + + def __init__(self, path, op, code="A"): + """ + Constructor + =========== + + @param path: The path to constrain + @type path: string + + @param op: The operation to apply - must be in the OPS set + @type op: string + """ + if op not in self.OPS: + raise TypeError(op + " not in " + str(self.OPS)) + self.op = op + self.code = code + super(CodedConstraint, self).__init__(path) + + def get_codes(self): + return [self.code] + + def __str__(self): + """ + Stringify to the code they are refered to by. + """ + return self.code + def to_string(self): + """ + Provide a human readable representation of the logic. + This method is called by repr. + """ + s = super(CodedConstraint, self).to_string() + return " ".join([s, self.op]) + + def to_dict(self): + """ + Return a dict object which can be used to construct a + DOM element with the appropriate attributes. + """ + d = super(CodedConstraint, self).to_dict() + d.update(op=self.op, code=self.code) + return d + +class UnaryConstraint(CodedConstraint): + """ + Constraints which have just a path and an operator + ================================================== + + These constraints are simple assertions about the + object/value refered to by the path. The set of valid + operators is: + - IS NULL + - IS NOT NULL + + """ + OPS = set(['IS NULL', 'IS NOT NULL']) + +class BinaryConstraint(CodedConstraint): + """ + Constraints which have an operator and a value + ============================================== + + These constraints assert a relationship between the + value represented by the path (it must be a representation + of a value, ie an Attribute) and another value - ie. the + operator takes two parameters. + + In all case the 'left' side of the relationship is the path, + and the 'right' side is the supplied value. + + Valid operators are: + - = (equal to) + - != (not equal to) + - < (less than) + - > (greater than) + - <= (less than or equal to) + - >= (greater than or equal to) + - LIKE (same as equal to, but with implied wildcards) + - CONTAINS (same as equal to, but with implied wildcards) + - NOT LIKE (same as not equal to, but with implied wildcards) + + """ + OPS = set(['=', '!=', '<', '>', '<=', '>=', 'LIKE', 'NOT LIKE', 'CONTAINS']) + def __init__(self, path, op, value, code="A"): + """ + Constructor + =========== + + @param path: The path to constrain + @type path: string + + @param op: The relationship between the value represented by the path and the value provided (must be a valid operator) + @type op: string + + @param value: The value to compare the stored value to + @type value: string or number + + @param code: The code for this constraint (default = "A") + @type code: string + """ + self.value = value + super(BinaryConstraint, self).__init__(path, op, code) + + def to_string(self): + """ + Provide a human readable representation of the logic. + This method is called by repr. + """ + s = super(BinaryConstraint, self).to_string() + return " ".join([s, str(self.value)]) + def to_dict(self): + """ + Return a dict object which can be used to construct a + DOM element with the appropriate attributes. + """ + d = super(BinaryConstraint, self).to_dict() + d.update(value=str(self.value)) + return d + +class ListConstraint(CodedConstraint): + """ + Constraints which refer to an objects membership of lists + ========================================================= + + These constraints assert a membership relationship between the + object represented by the path (it must always be an object, ie. + a Reference or a Class) and a List. Lists are collections of + objects in the database which are stored in InterMine + datawarehouses. These lists must be set up before the query is run, either + manually in the webapp or by using the webservice API list + upload feature. + + Valid operators are: + - IN + - NOT IN + + """ + OPS = set(['IN', 'NOT IN']) + def __init__(self, path, op, list_name, code="A"): + if hasattr(list_name, 'to_query'): + q = list_name.to_query() + l = q.service.create_list(q) + self.list_name = l.name + elif hasattr(list_name, "name"): + self.list_name = list_name.name + else: + self.list_name = list_name + super(ListConstraint, self).__init__(path, op, code) + + def to_string(self): + """ + Provide a human readable representation of the logic. + This method is called by repr. + """ + s = super(ListConstraint, self).to_string() + return " ".join([s, str(self.list_name)]) + def to_dict(self): + """ + Return a dict object which can be used to construct a + DOM element with the appropriate attributes. + """ + d = super(ListConstraint, self).to_dict() + d.update(value=str(self.list_name)) + return d + +class LoopConstraint(CodedConstraint): + """ + Constraints with refer to object identity + ========================================= + + These constraints assert that two paths refer to the same + object. + + Valid operators: + - IS + - IS NOT + + The operators IS and IS NOT map to the ops "=" and "!=" when they + are used in XML serialisation. + + """ + OPS = set(['IS', 'IS NOT']) + SERIALISED_OPS = {'IS':'=', 'IS NOT':'!='} + def __init__(self, path, op, loopPath, code="A"): + """ + Constructor + =========== + + @param path: The path to constrain + @type path: string + + @param op: The relationship between the path and the path provided (must be a valid operator) + @type op: string + + @param loopPath: The path to check for identity against + @type loopPath: string + + @param code: The code for this constraint (default = "A") + @type code: string + """ + self.loopPath = loopPath + super(LoopConstraint, self).__init__(path, op, code) + + def to_string(self): + """ + Provide a human readable representation of the logic. + This method is called by repr. + """ + s = super(LoopConstraint, self).to_string() + return " ".join([s, self.loopPath]) + def to_dict(self): + """ + Return a dict object which can be used to construct a + DOM element with the appropriate attributes. + """ + d = super(LoopConstraint, self).to_dict() + d.update(loopPath=self.loopPath, op=self.SERIALISED_OPS[self.op]) + return d + +class TernaryConstraint(BinaryConstraint): + """ + Constraints for broad, general searching over all fields + ======================================================== + + These constraints request a wide-ranging search for matching + fields over all aspects of an object, including up to coercion + from related classes. + + Valid operators: + - LOOKUP + + To aid disambiguation, Ternary constaints accept an extra_value as + well as the main value. + """ + OPS = set(['LOOKUP']) + def __init__(self, path, op, value, extra_value=None, code="A"): + """ + Constructor + =========== + + @param path: The path to constrain. Here is must be a class, or a reference to a class. + @type path: string + + @param op: The relationship between the path and the path provided (must be a valid operator) + @type op: string + + @param value: The value to check other fields against. + @type value: string + + @param extra_value: A further value for disambiguation. The meaning of this value varies by class + and configuration. For example, if the class of the object is Gene, then + extra_value will refer to the Organism. + @type extra_value: string + + @param code: The code for this constraint (default = "A") + @type code: string + """ + self.extra_value = extra_value + super(TernaryConstraint, self).__init__(path, op, value, code) + + def to_string(self): + """ + Provide a human readable representation of the logic. + This method is called by repr. + """ + s = super(TernaryConstraint, self).to_string() + if self.extra_value is None: + return s + else: + return " ".join([s, 'IN', self.extra_value]) + def to_dict(self): + """ + Return a dict object which can be used to construct a + DOM element with the appropriate attributes. + """ + d = super(TernaryConstraint, self).to_dict() + if self.extra_value is not None: + d.update(extraValue=self.extra_value) + return d + +class MultiConstraint(CodedConstraint): + """ + Constraints for checking membership of a set of values + ====================================================== + + These constraints require the value they constrain to be + either a member of a set of values, or not a member. + + Valid operators: + - ONE OF + - NONE OF + + These constraints are similar in use to List constraints, with + the following differences: + - The list in this case is a defined set of values that is passed + along with the query itself, rather than anything stored + independently on a server. + - The object of the constaint is the value of an attribute, rather + than an object's identity. + """ + OPS = set(['ONE OF', 'NONE OF']) + def __init__(self, path, op, values, code="A"): + """ + Constructor + =========== + + @param path: The path to constrain. Here it must be an attribute of some object. + @type path: string + + @param op: The relationship between the path and the path provided (must be a valid operator) + @type op: string + + @param values: The set of values which the object of the constraint either must or must not belong to. + @type values: set or list + + @param code: The code for this constraint (default = "A") + @type code: string + """ + if not isinstance(values, (set, list)): + raise TypeError("values must be a set or a list, not " + str(type(values))) + self.values = values + super(MultiConstraint, self).__init__(path, op, code) + + def to_string(self): + """ + Provide a human readable representation of the logic. + This method is called by repr. + """ + s = super(MultiConstraint, self).to_string() + return ' '.join([s, str(self.values)]) + def to_dict(self): + """ + Return a dict object which can be used to construct a + DOM element with the appropriate attributes. + """ + d = super(MultiConstraint, self).to_dict() + d.update(value=self.values) + return d + +class RangeConstraint(MultiConstraint): + """ + Constraints for testing where a value lies relative to a set of ranges + ====================================================================== + + These constraints require that the value of the path they constrain + should lie in relationship to the set of values passed according to + the specific operator. + + Valid operators: + - OVERLAPS : The value overlaps at least one of the given ranges + - WITHIN : The value is wholly outside the given set of ranges + - CONTAINS : The value contains all the given ranges + - DOES NOT CONTAIN : The value does not contain all the given ranges + - OUTSIDE : Some part is outside the given set of ranges + - DOES NOT OVERLAP : The value does not overlap with any of the ranges + + For example: + + 4 WITHIN [1..5, 20..25] => True + + The format of the ranges depends on the value being constrained and what range + parsers have been configured on the target server. A common range parser for + biological mines is the one for Locations: + + Gene.chromosomeLocation OVERLAPS [2X:54321..67890, 3R:12345..456789] + + """ + OPS = set(['OVERLAPS', 'DOES NOT OVERLAP', 'WITHIN', 'OUTSIDE', 'CONTAINS', 'DOES NOT CONTAIN']) + +class SubClassConstraint(Constraint): + """ + Constraints on the class of a reference + ======================================= + + If an object has a reference X to another object of type A, + and type B extends type A, then any object of type B may be + the value of the reference X. If you only want to see X's + which are B's, this may be achieved with subclass constraints, + which allow the type of an object to be limited to one of the + subclasses (at any depth) of the class type required + by the attribute. + + These constraints do not use operators. Since they cannot be + conditional (eg. "A is a B or A is a C" would not be possible + in an InterMine query), they do not have codes + and cannot be referenced in logic expressions. + """ + def __init__(self, path, subclass): + """ + Constructor + =========== + + @param path: The path to constrain. This must refer to a class or a reference to a class. + @type path: str + + @param subclass: The class to subclass the path to. This must be a simple class name (not a dotted name) + @type subclass: str + """ + if not PATH_PATTERN.match(subclass): + raise TypeError + self.subclass = subclass + super(SubClassConstraint, self).__init__(path) + def to_string(self): + """ + Provide a human readable representation of the logic. + This method is called by repr. + """ + s = super(SubClassConstraint, self).to_string() + return s + ' ISA ' + self.subclass + def to_dict(self): + """ + Return a dict object which can be used to construct a + DOM element with the appropriate attributes. + """ + d = super(SubClassConstraint, self).to_dict() + d.update(type=self.subclass) + return d + + +class TemplateConstraint(object): + """ + A mixin to supply the behaviour and state of constraints on templates + ===================================================================== + + Constraints on templates can also be designated as "on", "off" or "locked", which refers + to whether they are active or not. Inactive constraints are still configured, but behave + as if absent for the purpose of results. In addition, template constraints can be + editable or not. Only values for editable constraints can be provided when requesting results, + and only constraints that can participate in logic expressions can be editable. + """ + REQUIRED = "locked" + OPTIONAL_ON = "on" + OPTIONAL_OFF = "off" + def __init__(self, editable=True, optional="locked"): + """ + Constructor + =========== + + @param editable: Whether or not this constraint should accept new values. + @type editable: bool + + @param optional: Whether a value for this constraint must be provided when running. + @type optional: "locked", "on" or "off" + """ + self.editable = editable + if optional == TemplateConstraint.REQUIRED: + self.optional = False + self.switched_on = True + else: + self.optional = True + if optional == TemplateConstraint.OPTIONAL_ON: + self.switched_on = True + elif optional == TemplateConstraint.OPTIONAL_OFF: + self.switched_on = False + else: + raise TypeError("Bad value for optional") + + @property + def required(self): + """ + True if a value must be provided for this constraint. + + @rtype: bool + """ + return not self.optional + + @property + def switched_off(self): + """ + True if this constraint is currently inactive. + + @rtype: bool + """ + return not self.switched_on + + def get_switchable_status(self): + """ + Returns either "locked", "on" or "off". + """ + if not self.optional: + return "locked" + else: + if self.switched_on: + return "on" + else: + return "off" + + def switch_on(self): + """ + Make sure this constraint is active + =================================== + + @raise ValueError: if the constraint is not editable and optional + """ + if self.editable and self.optional: + self.switched_on = True + else: + raise ValueError, "This constraint is not switchable" + + def switch_off(self): + """ + Make sure this constraint is inactive + ===================================== + + @raise ValueError: if the constraint is not editable and optional + """ + if self.editable and self.optional: + self.switched_on = False + else: + raise ValueError, "This constraint is not switchable" + + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + if self.editable: + editable = "editable" + else: + editable = "non-editable" + return '(' + editable + ", " + self.get_switchable_status() + ')' + def separate_arg_sets(self, args): + """ + A static function to use when building template constraints. + ============================================================ + + dict -> (dict, dict) + + Splits a dictionary of arguments into two separate dictionaries, one with + arguments for the main constraint, and one with arguments for the template + portion of the behaviour + """ + c_args = {} + t_args = {} + for k, v in args.items(): + if k == "editable": + t_args[k] = v == "true" + elif k == "optional": + t_args[k] = v + else: + c_args[k] = v + return (c_args, t_args) + +class TemplateUnaryConstraint(UnaryConstraint, TemplateConstraint): + def __init__(self, *a, **d): + (c_args, t_args) = self.separate_arg_sets(d) + UnaryConstraint.__init__(self, *a, **c_args) + TemplateConstraint.__init__(self, **t_args) + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + return(UnaryConstraint.to_string(self) + + " " + TemplateConstraint.to_string(self)) + +class TemplateBinaryConstraint(BinaryConstraint, TemplateConstraint): + def __init__(self, *a, **d): + (c_args, t_args) = self.separate_arg_sets(d) + BinaryConstraint.__init__(self, *a, **c_args) + TemplateConstraint.__init__(self, **t_args) + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + return(BinaryConstraint.to_string(self) + + " " + TemplateConstraint.to_string(self)) + +class TemplateListConstraint(ListConstraint, TemplateConstraint): + def __init__(self, *a, **d): + (c_args, t_args) = self.separate_arg_sets(d) + ListConstraint.__init__(self, *a, **c_args) + TemplateConstraint.__init__(self, **t_args) + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + return(ListConstraint.to_string(self) + + " " + TemplateConstraint.to_string(self)) + +class TemplateLoopConstraint(LoopConstraint, TemplateConstraint): + def __init__(self, *a, **d): + (c_args, t_args) = self.separate_arg_sets(d) + LoopConstraint.__init__(self, *a, **c_args) + TemplateConstraint.__init__(self, **t_args) + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + return(LoopConstraint.to_string(self) + + " " + TemplateConstraint.to_string(self)) + +class TemplateTernaryConstraint(TernaryConstraint, TemplateConstraint): + def __init__(self, *a, **d): + (c_args, t_args) = self.separate_arg_sets(d) + TernaryConstraint.__init__(self, *a, **c_args) + TemplateConstraint.__init__(self, **t_args) + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + return(TernaryConstraint.to_string(self) + + " " + TemplateConstraint.to_string(self)) + +class TemplateMultiConstraint(MultiConstraint, TemplateConstraint): + def __init__(self, *a, **d): + (c_args, t_args) = self.separate_arg_sets(d) + MultiConstraint.__init__(self, *a, **c_args) + TemplateConstraint.__init__(self, **t_args) + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + return(MultiConstraint.to_string(self) + + " " + TemplateConstraint.to_string(self)) + +class TemplateRangeConstraint(RangeConstraint, TemplateConstraint): + def __init__(self, *a, **d): + (c_args, t_args) = self.separate_arg_sets(d) + RangeConstraint.__init__(self, *a, **c_args) + TemplateConstraint.__init__(self, **t_args) + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + return(RangeConstraint.to_string(self) + + " " + TemplateConstraint.to_string(self)) + +class TemplateSubClassConstraint(SubClassConstraint, TemplateConstraint): + def __init__(self, *a, **d): + (c_args, t_args) = self.separate_arg_sets(d) + SubClassConstraint.__init__(self, *a, **c_args) + TemplateConstraint.__init__(self, **t_args) + def to_string(self): + """ + Provide a template specific human readable representation of the + constraint. This method is called by repr. + """ + return(SubClassConstraint.to_string(self) + + " " + TemplateConstraint.to_string(self)) + +class ConstraintFactory(object): + """ + A factory for creating constraints from a set of arguments. + =========================================================== + + A constraint factory is responsible for finding an appropriate + constraint class for the given arguments and instantiating the + constraint. + """ + CONSTRAINT_CLASSES = set([ + UnaryConstraint, BinaryConstraint, TernaryConstraint, + MultiConstraint, SubClassConstraint, LoopConstraint, + ListConstraint, RangeConstraint]) + + def __init__(self): + """ + Constructor + =========== + + Creates a new ConstraintFactory + """ + self._codes = iter(string.ascii_uppercase) + + def get_next_code(self): + """ + Return the available constraint code. + + @return: A single uppercase character + @rtype: str + """ + return self._codes.next() + + def make_constraint(self, *args, **kwargs): + """ + Create a constraint from a set of arguments. + ============================================ + + Finds a suitable constraint class, and instantiates it. + + @rtype: Constraint + """ + for CC in self.CONSTRAINT_CLASSES: + try: + c = CC(*args, **kwargs) + if hasattr(c, "code") and c.code == "A": + c.code = self.get_next_code() + return c + except TypeError, e: + pass + raise TypeError("No matching constraint class found for " + + str(args) + ", " + str(kwargs)) + +class TemplateConstraintFactory(ConstraintFactory): + """ + A factory for creating constraints with template specific characteristics. + ========================================================================== + + A constraint factory is responsible for finding an appropriate + constraint class for the given arguments and instantiating the + constraint. TemplateConstraintFactories make constraints with the + extra set of TemplateConstraint qualities. + """ + CONSTRAINT_CLASSES = set([ + TemplateUnaryConstraint, TemplateBinaryConstraint, + TemplateTernaryConstraint, TemplateMultiConstraint, + TemplateSubClassConstraint, TemplateLoopConstraint, + TemplateListConstraint, TemplateRangeConstraint + ]) diff --git a/intermine/errors.py b/intermine/errors.py new file mode 100644 index 00000000..26af09a9 --- /dev/null +++ b/intermine/errors.py @@ -0,0 +1,12 @@ +from intermine.util import ReadableException + +class UnimplementedError(Exception): + pass + +class ServiceError(ReadableException): + """Errors in the creation and use of the Service object""" + pass + +class WebserviceError(IOError): + """Errors from interaction with the webservice""" + pass diff --git a/intermine/lists/__init__.py b/intermine/lists/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/intermine/lists/__init__.py @@ -0,0 +1 @@ + diff --git a/intermine/lists/list.py b/intermine/lists/list.py new file mode 100644 index 00000000..2812a3e1 --- /dev/null +++ b/intermine/lists/list.py @@ -0,0 +1,399 @@ +import weakref +import urllib + +from intermine.results import JSONIterator, EnrichmentLine +from intermine.model import ConstraintNode +from intermine.errors import ServiceError + +class List(object): + """ + Class for representing a List on an InterMine Webservice + ======================================================== + + Lists represent stored collections of data and saved result + sets in an InterMine data warehouse. This class is an abstraction + of this information, and provides mechanisms for managing the + data. + + SYNOPSIS + -------- + + example:: + + >>> from intermine.webservice import Service + >>> + >>> flymine = Service("www.flymine.org/query", "SOMETOKEN") + >>> new_list = flymine.create_list(["h", "zen", "eve", "bib"], "Gene", name="My New List") + >>> + >>> another_list = flymine.get_list("Some other list") + >>> combined_list = new_list | another_list # Same syntax as for sets + >>> combined_list.name = "Union of the other lists" + >>> + >>> print "The combination of the two lists has %d elements" % combined_list.size + >>> print "The combination of the two lists has %d elements" % len(combined_list) + >>> + >>> for row in combined_list: + ... print row + + OVERVIEW + -------- + + Lists are created from a webservice, and can be manipulated in various ways. + The operations are:: + * Union: this | that + * Intersection: this & that + * Symmetric Difference: this ^ that + * Asymmetric Difference (subtraction): this - that + * Appending: this += that + + Lists can be created from a list of identifiers that could be:: + * stored in a file + * held in a list or set + * contained in a string + In all these cases the syntax is the same: + + >>> new_list = service.create_list(content, type, name="Some name", description="Some description", tags=["some", "tags"]) + + Lists can also be created from a query's result with the exact + same syntax. In the case of queries, the type is not required, + but the query should have just one view, and it should be an id. + + >>> query = service.new_query() + >>> query.add_view("Gene.id") + >>> query.add_constraint("Gene.length", "<", 100) + >>> new_list = service.create_list(query, name="Short Genes") + + """ + + def __init__(self, **args): + """ + Constructor + =========== + + Do not construct these objects yourself. They should be + fetched from a service or constructed using the "create_list" + method. + """ + try: + self._service = args["service"] + self._manager = weakref.proxy(args["manager"]) + self._name = args["name"] + self._title = args["title"] + self._description = args.get("description") + self._list_type = args["type"] + self._size = int(args["size"]) + self._date_created = args.get("dateCreated") + self._is_authorized = args.get("authorized") + self._status = args.get("status") + + if self._is_authorized is None: self._is_authorized = True + + if "tags" in args: + tags = args["tags"] + else: + tags = [] + + self._tags = frozenset(tags) + except KeyError: + raise ValueError("Missing argument") + self.unmatched_identifiers = set([]) + + @property + def date_created(self): + """When this list was originally created""" + return self._date_created + + @property + def tags(self): + """The tags associated with this list""" + return self._tags + + @property + def description(self): + """The human readable description of this list""" + return self._description + + @property + def title(self): + """The fixed title of this list""" + return self._title + + @property + def status(self): + """The upgrade status of this list""" + return self._status + + @property + def is_authorized(self): + """Whether or not the current user is authorised to make changes to this list""" + return self._is_authorized + + @property + def list_type(self): + """The type of the InterMine objects this list can contain""" + return self._list_type + + def get_name(self): + """The name of the list used to access it programmatically""" + return self._name + + def set_name(self, new_name): + """ + Set the name of the list + ======================== + + Setting the list's name causes the list's name to be updated on the server. + """ + if self._name == new_name: + return + uri = self._service.root + self._service.LIST_RENAME_PATH + params = { + "oldname": self._name, + "newname": new_name + } + uri += "?" + urllib.urlencode(params) + resp = self._service.opener.open(uri) + data = resp.read() + resp.close() + new_list = self._manager.parse_list_upload_response(data) + self._name = new_name + + def del_name(self): + """Raises an error - lists must always have a name""" + raise AttributeError("List names cannot be deleted, only changed") + + @property + def size(self): + """Return the number of elements in the list. Also available as len(obj)""" + return self._size + + @property + def count(self): + """Alias for obj.size. Also available as len(obj)""" + return self.size + + def __len__(self): + """Returns the number of elements in the object""" + return self.size + + name = property(get_name, set_name, del_name, "The name of this list") + + def _add_failed_matches(self, ids): + if ids is not None: + self.unmatched_identifiers.update(ids) + + def __str__(self): + string = self.name + " (" + str(self.size) + " " + self.list_type + ")" + if self.date_created: + string += " " + self.date_created + if self.description: + string += " " + self.description + return string + + def delete(self): + """ + Delete this list from the webservice + ==================================== + + Calls the webservice to delete this list immediately. This + object should not be used after this method is called - attempts + to do so will raise errors. + """ + self._manager.delete_lists([self]) + + def to_query(self): + """ + Construct a query to fetch the items in this list + ================================================= + + Return a new query constrained to the objects in this list, + and with a single view column of the objects ids. + + @rtype: intermine.query.Query + """ + q = self._service.new_query(self.list_type) + q.add_constraint(self.list_type, "IN", self.name) + return q + + def make_list_constraint(self, path, op): + """ + Implementation of trait that allows use of these objects in list constraints + """ + return ConstraintNode(path, op, self.name) + + def __iter__(self): + """Return an iterator over the objects in this list, with all attributes selected for output""" + return iter(self.to_query()) + + def __getitem__(self, index): + """Get a member of this list by index""" + if not isinstance(index, int): + raise IndexError("Expected an integer key - got %s" % (index)) + if index < 0: # handle negative indices. + i = self.size + index + else: + i = index + + if i not in range(self.size): + raise IndexError("%d is not a valid index for a list of size %d" % (index, self.size)) + + return self.to_query().first(start=i, row="jsonobjects") + + def __and__(self, other): + """ + Intersect this list and another + """ + return self._manager.intersect([self, other]) + + def __iand__(self, other): + """ + Intersect this list and another, and replace this list with the result of the + intersection + """ + intersection = self._manager.intersect([self, other], description=self.description, tags=self.tags) + self.delete() + intersection.name = self.name + return intersection + + def __or__(self, other): + """ + Return the union of this list and another + """ + return self._manager.union([self, other]) + + def __add__(self, other): + """ + Return the union of this list and another + """ + return self._manager.union([self, other]) + + def __iadd__(self, other): + """ + Append other to this list. + """ + return self.append(other) + + def _do_append(self, content): + name = self.name + data = None + + try: + ids = open(content).read() + except (TypeError, IOError): + if isinstance(content, basestring): + ids = content + else: + try: + ids = "\n".join(map(lambda x: '"' + x + '"', iter(content))) + except TypeError: + content = self._manager._get_listable_query(content) + uri = content.get_list_append_uri() + params = content.to_query_params() + params["listName"] = name + params["path"] = None + form = urllib.urlencode(params) + resp = self._service.opener.open(uri, form) + data = resp.read() + + if data is None: + uri = self._service.root + self._service.LIST_APPENDING_PATH + query_form = {'name': name} + uri += "?" + urllib.urlencode(query_form) + data = self._service.opener.post_plain_text(uri, ids) + + new_list = self._manager.parse_list_upload_response(data) + self.unmatched_identifiers.update(new_list.unmatched_identifiers) + self._size = new_list.size + return self + + def append(self, appendix): + """Append the arguments to this list""" + try: + return self._do_append(self._manager.union(appendix)) + except: + return self._do_append(appendix) + + def calculate_enrichment(self, widget, background = None, correction = "Holm-Bonferroni", maxp = 0.05, filter = ''): + """ + Perform an enrichment calculation on this list + ============================================== + + example:: + + >>> for item in service.get_list("some list").calculate_enrichment("thingy_enrichment"): + ... print item.identifier, item.p_value + + Gets an iterator over the rows for an enrichment calculation. Each row represents + a record with the following properties: + * identifier {str} + * p-value {float} + * matches {int} + * description {str} + + The enrichment row object may be treated as an object with property access, or as + a dictionary, supporting key lookup with the [] operator: + + >>> p_value = row['p-value'] + + """ + if self._service.version < 8: + raise ServiceError("This service does not support enrichment requests") + params = dict(list = self.name, widget = widget, correction = correction, maxp = maxp, filter = filter) + if background is not None: + if self._service.version < 11: + raise ServiceError("This service does not support custom background populations") + params["population"] = background + form = urllib.urlencode(params) + uri = self._service.root + self._service.LIST_ENRICHMENT_PATH + resp = self._service.opener.open(uri, form) + return JSONIterator(resp, EnrichmentLine) + + def __xor__(self, other): + """Calculate the symmetric difference of this list and another""" + return self._manager.xor([self, other]) + + def __ixor__(self, other): + """Calculate the symmetric difference of this list and another and replace this list with the result""" + diff = self._manager.xor([self, other], description=self.description, tags=self.tags) + self.delete() + diff.name = self.name + return diff + + def __sub__(self, other): + """Subtract the other from this list""" + return self._manager.subtract([self], [other]) + + def __isub__(self, other): + """Replace this list with the subtraction of the other from this list""" + subtr = self._manager.subtract([self], [other], description=self.description, tags=self.tags) + self.delete() + subtr.name = self.name + return subtr + + def add_tags(self, *tags): + """ + Tag this list with one or more categories + ========================================= + + Calls the server to add these tags, and updates this lists tags. + """ + self._tags = frozenset(self._manager.add_tags(self, tags)) + + def remove_tags(self, *tags): + """ + Remove tags associated with this list. + ====================================== + + Calls the server to remove these tags, and updates this lists tags. + """ + self._tags = frozenset(self._manager.remove_tags(self, tags)) + + def update_tags(self, *tags): + """ + Remove tags associated with this list. + ====================================== + + Calls the server to remove these tags, and updates this lists tags. + """ + self._tags = frozenset(self._manager.get_tags(self)) + diff --git a/intermine/lists/listmanager.py b/intermine/lists/listmanager.py new file mode 100644 index 00000000..0c076a90 --- /dev/null +++ b/intermine/lists/listmanager.py @@ -0,0 +1,342 @@ +import weakref + +# Use core json for 2.6+, simplejson for <=2.5 +try: + import json +except ImportError: + import simplejson as json + +import urllib +import codecs + +from intermine.lists.list import List + +class ListManager(object): + """ + A Class for Managing List Content and Operations + ================================================ + + This class serves as a delegate for the intermine.webservice.Service class, + managing list content and operations. + + This class is not meant to be called itself, but rather for its + methods to be called by the service object. + + Note that the methods for creating lists can conflict in threaded applications, if + two threads are each allocated the same unused list name. You are + strongly advised to use locks to synchronise any list creation requests (create_list, + or intersect, union, subtract, diff) unless you are choosing your own names each time. + """ + + DEFAULT_LIST_NAME = "my_list_" + DEFAULT_DESCRIPTION = "List created with Python client library" + + INTERSECTION_PATH = '/lists/intersect/json' + UNION_PATH = '/lists/union/json' + DIFFERENCE_PATH = '/lists/diff/json' + SUBTRACTION_PATH = '/lists/subtract/json' + + def __init__(self, service): + self.service = weakref.proxy(service) + self.lists = None + self._temp_lists = set() + + def refresh_lists(self): + """Update the list information with the latest details from the server""" + self.lists = {} + url = self.service.root + self.service.LIST_PATH + sock = self.service.opener.open(url) + data = sock.read() + sock.close() + list_info = json.loads(data) + if not list_info.get("wasSuccessful"): + raise ListServiceError(list_info.get("error")) + for l in list_info["lists"]: + l = ListManager.safe_dict(l) # Workaround for python 2.6 unicode key issues + self.lists[l["name"]] = List(service=self.service, manager=self, **l) + + @staticmethod + def safe_dict(d): + """Recursively clone json structure with UTF-8 dictionary keys""" + if isinstance(d, dict): + return dict([(k.encode('utf-8'), v) for k,v in d.iteritems()]) + else: + return d + + def get_list(self, name): + """Return a list from the service by name, if it exists""" + if self.lists is None: + self.refresh_lists() + return self.lists.get(name) + + def l(self, name): + """Alias for get_list""" + return self.get_list(name) + + def get_all_lists(self): + """Get all the lists on a webservice""" + if self.lists is None: + self.refresh_lists() + return self.lists.values() + + def get_all_list_names(self): + """Get all the names of the lists in a particular webservice""" + if self.lists is None: + self.refresh_lists() + return self.lists.keys() + + def get_list_count(self): + """ + Return the number of lists accessible at the given webservice. + This number will vary depending on who you are authenticated as. + """ + return len(self.get_all_list_names()) + + def get_unused_list_name(self): + """ + Get an unused list name + ======================= + + This method returns a new name that does not conflict + with any currently existing list name. + + The list name is only guaranteed to be unused at the time + of allocation. + """ + list_names = self.get_all_list_names() + counter = 1 + name = self.DEFAULT_LIST_NAME + str(counter) + while name in list_names: + counter += 1 + name = self.DEFAULT_LIST_NAME + str(counter) + self._temp_lists.add(name) + return name + + def _get_listable_query(self, queryable): + q = queryable.to_query() + if not q.views: + q.add_view(q.root.name + ".id") + else: + # Check to see if the class of the selected items is unambiguous + up_to_attrs = set((v[0:v.rindex(".")] for v in q.views)) + if len(up_to_attrs) == 1: + q.select(up_to_attrs.pop() + ".id") + return q + + def _create_list_from_queryable(self, queryable, name, description, tags): + q = self._get_listable_query(queryable) + uri = q.get_list_upload_uri() + params = q.to_query_params() + params["listName"] = name + params["description"] = description + params["tags"] = ";".join(tags) + form = urllib.urlencode(params) + resp = self.service.opener.open(uri, form) + data = resp.read() + resp.close() + return self.parse_list_upload_response(data) + + def create_list(self, content, list_type="", name=None, description=None, tags=[]): + """ + Create a new list in the webservice + =================================== + + If no name is given, the list will be considered to be a temporary + list, and will be automatically deleted when the program ends. To prevent + this happening, give the list a name, either on creation, or by renaming it. + + This method is not thread safe for anonymous lists - it will need synchronisation + with locks if you intend to create lists with multiple threads in parallel. + + @rtype: intermine.lists.List + """ + if description is None: + description = self.DEFAULT_DESCRIPTION + + if name is None: + name = self.get_unused_list_name() + + try: + ids = open(content).read() + except (TypeError, IOError): + if isinstance(content, basestring): + ids = content + else: + try: + return self._create_list_from_queryable(content, name, description, tags) + except AttributeError: + ids = "\n".join(map(lambda x: '"' + x + '"', iter(content))) + + uri = self.service.root + self.service.LIST_CREATION_PATH + query_form = { + 'name': name, + 'type': list_type, + 'description': description, + 'tags': ";".join(tags) + } + uri += "?" + urllib.urlencode(query_form) + data = self.service.opener.post_plain_text(uri, ids) + return self.parse_list_upload_response(data) + + def parse_list_upload_response(self, response): + """ + Intepret the response from the webserver to a list request, and return the List it describes + """ + try: + response_data = json.loads(response) + except ValueError: + raise ListServiceError("Error parsing response: " + response) + if not response_data.get("wasSuccessful"): + raise ListServiceError(response_data.get("error")) + self.refresh_lists() + new_list = self.get_list(response_data["listName"]) + failed_matches = response_data.get("unmatchedIdentifiers") + new_list._add_failed_matches(failed_matches) + return new_list + + def delete_lists(self, lists): + """Delete the given lists from the webserver""" + all_names = self.get_all_list_names() + for l in lists: + if isinstance(l, List): + name = l.name + else: + name = str(l) + if name not in all_names: + continue + uri = self.service.root + self.service.LIST_PATH + query_form = {'name': name} + uri += "?" + urllib.urlencode(query_form) + response = self.service.opener.delete(uri) + response_data = json.loads(response) + if not response_data.get("wasSuccessful"): + raise ListServiceError(response_data.get("error")) + self.refresh_lists() + + def remove_tags(self, to_remove_from, tags): + """ + Add the tags to the given list + ============================== + + Returns the current tags of this list. + """ + uri = self.service.root + self.service.LIST_TAG_PATH + form = {"name": to_remove_from.name, "tags": ";".join(tags)} + uri += "?" + urllib.urlencode(form) + body = self.service.opener.delete(uri) + return self._body_to_json(body)["tags"] + + def add_tags(self, to_tag, tags): + """ + Add the tags to the given list + ============================== + + Returns the current tags of this list. + """ + uri = self.service.root + self.service.LIST_TAG_PATH + form = {"name": to_tag.name, "tags": ";".join(tags)} + resp = self.service.opener.open(uri, urllib.urlencode(form)) + body = resp.read() + resp.close() + return self._body_to_json(body)["tags"] + + def get_tags(self, im_list): + """ + Get the up-to-date set of tags for a given list + =============================================== + + Returns the current tags of this list. + """ + uri = self.service.root + self.service.LIST_TAG_PATH + form = {"name": im_list.name} + uri += "?" + urllib.urlencode(form) + resp = self.service.opener.open(uri) + body = resp.read() + resp.close() + return self._body_to_json(body)["tags"] + + def _body_to_json(self, body): + try: + data = json.loads(body) + except ValueError: + raise ListServiceError("Error parsing response: " + body) + if not data.get("wasSuccessful"): + raise ListServiceError(data.get("error")) + return data + + def delete_temporary_lists(self): + """Delete all the lists considered temporary (those created without names)""" + self.delete_lists(self._temp_lists) + self._temp_lists = set() + + def intersect(self, lists, name=None, description=None, tags=[]): + """Calculate the intersection of a given set of lists, and return the list representing the result""" + return self._do_operation(self.INTERSECTION_PATH, "Intersection", lists, name, description, tags) + + def union(self, lists, name=None, description=None, tags=[]): + """Calculate the union of a given set of lists, and return the list representing the result""" + return self._do_operation(self.UNION_PATH, "Union", lists, name, description, tags) + + def xor(self, lists, name=None, description=None, tags=[]): + """Calculate the symmetric difference of a given set of lists, and return the list representing the result""" + return self._do_operation(self.DIFFERENCE_PATH, "Difference", lists, name, description, tags) + + def subtract(self, lefts, rights, name=None, description=None, tags=[]): + """Calculate the subtraction of rights from lefts, and return the list representing the result""" + left_names = self.make_list_names(lefts) + right_names = self.make_list_names(rights) + if description is None: + description = "Subtraction of " + ' and '.join(right_names) + " from " + ' and '.join(left_names) + if name is None: + name = self.get_unused_list_name() + uri = self.service.root + self.SUBTRACTION_PATH + uri += '?' + urllib.urlencode({ + "name": name, + "description": description, + "references": ';'.join(left_names), + "subtract": ';'.join(right_names), + "tags": ";".join(tags) + }) + resp = self.service.opener.open(uri) + data = resp.read() + resp.close() + return self.parse_list_upload_response(data) + + def _do_operation(self, path, operation, lists, name, description, tags): + list_names = self.make_list_names(lists) + if description is None: + description = operation + " of " + ' and '.join(list_names) + if name is None: + name = self.get_unused_list_name() + uri = self.service.root + path + uri += '?' + urllib.urlencode({ + "name": name, + "lists": ';'.join(list_names), + "description": description, + "tags": ";".join(tags) + }) + resp = self.service.opener.open(uri) + data = resp.read() + resp.close() + return self.parse_list_upload_response(data) + + + def make_list_names(self, lists): + """Turn a list of things into a list of list names""" + list_names = [] + for l in lists: + try: + t = l.list_type + list_names.append(l.name) + except AttributeError: + try: + m = l.model + list_names.append(self.create_list(l).name) + except AttributeError: + list_names.append(str(l)) + + return list_names + +class ListServiceError(IOError): + """Errors thrown when something goes wrong with list requests""" + pass diff --git a/intermine/model.py b/intermine/model.py new file mode 100644 index 00000000..b20a74ec --- /dev/null +++ b/intermine/model.py @@ -0,0 +1,958 @@ +from xml.dom import minidom +import weakref +import re + +from intermine.util import openAnything, ReadableException + +""" +Classes representing the data model +=================================== + +Representations of tables and columns, and behaviour +for validating connections between them. + +""" + +__author__ = "Alex Kalderimis" +__organization__ = "InterMine" +__license__ = "LGPL" +__contact__ = "dev@intermine.org" + +class Field(object): + """ + A class representing columns on database tables + =============================================== + + The base class for attributes, references and collections. All + columns in DB tables are represented by fields + + SYNOPSIS + -------- + + >>> service = Service("http://www.flymine.org/query/service") + >>> model = service.model + >>> cd = model.get_class("Gene") + >>> print "Gene has", len(cd.fields), "fields" + >>> for field in gene_cd.fields: + ... print " - ", field + Gene has 45 fields + - CDSs is a group of CDS objects, which link back to this as gene + - GLEANRsymbol is a String + - UTRs is a group of UTR objects, which link back to this as gene + - alleles is a group of Allele objects, which link back to this as gene + - chromosome is a Chromosome + - chromosomeLocation is a Location + - clones is a group of CDNAClone objects, which link back to this as gene + - crossReferences is a group of CrossReference objects, which link back to this as subject + - cytoLocation is a String + - dataSets is a group of DataSet objects, which link back to this as bioEntities + - downstreamIntergenicRegion is a IntergenicRegion + - exons is a group of Exon objects, which link back to this as gene + - flankingRegions is a group of GeneFlankingRegion objects, which link back to this as gene + - goAnnotation is a group of GOAnnotation objects + - homologues is a group of Homologue objects, which link back to this as gene + - id is a Integer + - interactions is a group of Interaction objects, which link back to this as gene + - length is a Integer + ... + + @see: L{Attribute} + @see: L{Reference} + @see: L{Collection} + """ + def __init__(self, name, type_name, class_origin): + """ + Constructor - DO NOT USE + ======================== + + THIS CLASS IS NOT MEANT TO BE INSTANTIATED DIRECTLY + + you are unlikely to need to do + so anyway: it is recommended you access fields + through the classes generated by the model + + @param name: The name of the reference + @param type_name: The name of the model.Class this refers to + @param class_origin: The model.Class this was declared in + + """ + self.name = name + self.type_name = type_name + self.type_class = None + self.declared_in = class_origin + def __repr__(self): + return self.name + " is a " + self.type_name + def __str__(self): + return self.name + +class Attribute(Field): + """ + Attributes represent columns that contain actual data + ===================================================== + + The Attribute class inherits all the behaviour of L{intermine.model.Field} + """ + pass + +class Reference(Field): + """ + References represent columns that refer to records in other tables + ================================================================== + + In addition the the behaviour and properties of Field, references + may also have a reverse reference, if the other record points + back to this one as well. And all references will have their + type upgraded to a type_class during parsing + """ + def __init__(self, name, type_name, class_origin, reverse_ref=None): + """ + Constructor + =========== + + In addition to the a parameters of Field, Reference also + takes an optional reverse reference name (str) + + @param name: The name of the reference + @param type_name: The name of the model.Class this refers to + @param class_origin: The model.Class this was declared in + @param reverse_ref: The name of the reverse reference (default: None) + + """ + self.reverse_reference_name = reverse_ref + super(Reference, self).__init__(name, type_name, class_origin) + self.reverse_reference = None + def __repr__(self): + """ + Return a string representation + ============================== + + @rtype: str + """ + s = super(Reference, self).__repr__() + if self.reverse_reference is None: + return s + else: + return s + ", which links back to this as " + self.reverse_reference.name + +class Collection(Reference): + """ + Collections are references which refer to groups of objects + =========================================================== + + Collections have all the same behaviour and properties as References + """ + def __repr__(self): + """Return a string representation""" + ret = super(Collection, self).__repr__().replace(" is a ", " is a group of ") + if self.reverse_reference is None: + return ret + " objects" + else: + return ret.replace(", which links", " objects, which link") + + +class Class(object): + """ + An abstraction of database tables in the data model + =================================================== + + These objects refer to the table objects in the + InterMine ORM layer. + + SYNOPSIS + -------- + + >>> service = Service("http://www.flymine.org/query/service") + >>> model = service.model + >>> + >>> if "Gene" in model.classes: + ... gene_cd = model.get_class("Gene") + ... print "Gene has", len(gene_cd.fields), "fields" + ... for field in gene_cd.fields: + ... print " - ", field.name + + OVERVIEW + -------- + + Each class can have attributes (columns) of various types, + and can have references to other classes (tables), on either + a one-to-one (references) or one-to-many (collections) basis + + Classes should not be instantiated by hand, but rather used + as part of the model they belong to. + + """ + + + def __init__(self, name, parents, model): + """ + Constructor - Creates a new Class descriptor + ============================================ + + >>> cd = intermine.model.Class("Gene", ["SequenceFeature"]) + + + This constructor is called when deserialising the + model - you should have no need to create Classes by hand + + @param name: The name of this class + @param parents: a list of parental names + + """ + self.name = name + self.parents = parents + self.model = model + self.parent_classes = [] + self.field_dict = {} + self.has_id = "Object" not in parents + if self.has_id: + # All InterMineObject classes have an id attribute. + id_field = Attribute("id", "Integer", self) + self.field_dict["id"] = id_field + + def __repr__(self): + return "<%s.%s %s.%s>" % (self.__module__, self.__class__.__name__, + self.model.package_name if hasattr(self.model, 'package_name') else "__test__", self.name) + + @property + def fields(self): + """ + The fields of this class + ======================== + + The fields are returned sorted by name. Fields + includes all Attributes, References and Collections + + @rtype: list(L{Field}) + """ + return sorted(self.field_dict.values(), key=lambda field: field.name) + + def __iter__(self): + for f in self.field_dict.values(): + yield f + + def __contains__(self, item): + if isinstance(item, Field): + return item in self.field_dict.values() + else: + return str(item) in self.field_dict + + @property + def attributes(self): + """ + The fields of this class which contain data + =========================================== + + @rtype: list(L{Attribute}) + """ + return filter(lambda x: isinstance(x, Attribute), self.fields) + + @property + def references(self): + """ + fields which reference other objects + ==================================== + + @rtype: list(L{Reference}) + """ + def isRef(x): return isinstance(x, Reference) and not isinstance(x, Collection) + return filter(isRef, self.fields) + + @property + def collections(self): + """ + fields which reference many other objects + ========================================= + + @rtype: list(L{Collection}) + """ + return filter(lambda x: isinstance(x, Collection), self.fields) + + def get_field(self, name): + """ + Get a field by name + =================== + + The standard way of retrieving a field + + @raise ModelError: if the Class does not have such a field + + @rtype: subclass of L{intermine.model.Field} + """ + if name in self.field_dict: + return self.field_dict[name] + else: + raise ModelError("There is no field called %s in %s" % (name, self.name)) + + def isa(self, other): + """ + Check if self is, or inherits from other + ======================================== + + This method validates statements about inheritance. + Returns true if the "other" is, or is within the + ancestry of, this class + + Other can be passed as a name (str), or as the class object itself + + @rtype: boolean + """ + if isinstance(other, Class): + other_name = other.name + else: + other_name = other + if self.name == other_name: + return True + if other_name in self.parents: + return True + for p in self.parent_classes: + if p.isa(other): + return True + return False + + +class Path(object): + """ + A class representing a validated dotted string path + =================================================== + + A path represents a connection between records and fields + + SYNOPSIS + -------- + + >>> service = Service("http://www.flymine.org/query/service") + model = service.model + path = model.make_path("Gene.organism.name") + path.is_attribute() + ... True + >>> path2 = model.make_path("Gene.proteins") + path2.is_attribute() + ... False + >>> path2.is_reference() + ... True + >>> path2.get_class() + ... + + OVERVIEW + -------- + + This class is used for performing validation on dotted path strings. + The simple act of parsing it into existence will validate the path + to some extent, but there are additional methods for verifying certain + relationships as well + """ + def __init__(self, path, model, subclasses={}): + """ + Constructor + =========== + + >>> path = Path("Gene.name", model) + + You will not need to use this constructor directly. Instead, + use the "make_path" method on the model to construct paths for you. + + @param path: the dotted path string (eg: Gene.proteins.name) + @type path: str + @param model: the model to validate the path against + @type model: L{Model} + @param subclasses: a dict which maps subclasses (defaults to an empty dict) + @type subclasses: dict + """ + self.model = weakref.proxy(model) + self.subclasses = subclasses + if isinstance(path, Class): + self._string = path.name + self.parts = [path] + else: + self._string = str(path) + self.parts = model.parse_path_string(str(path), subclasses) + + def __str__(self): + return self._string + + def __repr__(self): + return '<' + self.__module__ + "." + self.__class__.__name__ + ": " + self._string + '>' + + def prefix(self): + """ + The path one step above this path. + ================================== + + >>> p1 = Path("Gene.exons.name", model) + >>> p2 = p1.prefix() + >>> print p2 + ... Gene.exons + + """ + parts = list(self.parts) + parts.pop() + if len(parts) < 1: + raise PathParseError(str(self) + " does not have a prefix") + s = ".".join(map(lambda x: x.name, parts)) + return Path(s, self.model._unproxied(), self.subclasses) + + def append(self, *elements): + """ + Construct a new path by adding elements to the end of this one. + =============================================================== + + >>> p1 = Path("Gene.exons", model) + >>> p2 = p1.append("name") + >>> print p2 + ... Gene.exons.name + + This is the inverse of prefix. + """ + s = str(self) + "." + ".".join(elements) + return Path(s, self.model._unproxied(), self.subclasses) + + @property + def root(self): + """ + The descriptor for the first part of the string. This should always a class descriptor. + + @rtype: L{intermine.model.Class} + """ + return self.parts[0] + + @property + def end(self): + """ + The descriptor for the last part of the string. + + @rtype: L{model.Class} or L{model.Field} + """ + return self.parts[-1] + + def get_class(self): + """ + Return the class object for this path, if it refers to a class + or a reference. Attribute paths return None + + @rtype: L{model.Class} + """ + if self.is_class(): + return self.end + elif self.is_reference(): + if str(self) in self.subclasses: + return self.model.get_class(self.subclasses[str(self)]) + return self.end.type_class + else: + return None + + end_class = property(get_class) + + def is_reference(self): + """ + Return true if the path is a reference, eg: Gene.organism or Gene.proteins + Note: Collections are ALSO references + + @rtype: boolean + """ + return isinstance(self.end, Reference) + + def is_class(self): + """ + Return true if the path just refers to a class, eg: Gene + + @rtype: boolean + """ + return isinstance(self.end, Class) + + def is_attribute(self): + """ + Return true if the path refers to an attribute, eg: Gene.length + + @rtype: boolean + """ + return isinstance(self.end, Attribute) + + def __eq__(self, other): + return str(self) == str(other) + + def __hash__(self): + i = hash(str(self)) + return reduce(lambda a, b: a ^ b, [hash(k) ^ hash(v) for k, v in self.subclasses.items()], i) + +class ConstraintTree(object): + + def __init__(self, op, left, right): + self.op = op + self.left = left + self.right = right + + def __and__(self, other): + return ConstraintTree('AND', self, other) + + def __or__(self, other): + return ConstraintTree('OR', self, other) + + def __iter__(self): + for n in [self.left, self.right]: + for subn in n: + yield subn + + def as_logic(self, codes = None, start = 'A'): + if codes is None: + codes = (chr(c) for c in range(ord(start), ord('Z'))) + return "(%s %s %s)" % (self.left.as_logic(codes), self.op, self.right.as_logic(codes)) + +class ConstraintNode(ConstraintTree): + + def __init__(self, *args, **kwargs): + self.vargs = args + self.kwargs = kwargs + + def __iter__(self): + yield self + + def as_logic(self, codes = None, start = 'A'): + if codes is None: + codes = (chr(c) for c in range(ord(start), ord('Z'))) + return codes.next() + +class CodelessNode(ConstraintNode): + + def as_logic(self, code = None, start = 'A'): + return '' + +class Column(object): + """ + A representation of a path in a query that can be constrained + ============================================================= + + Column objects allow constraints to be constructed in something + close to a declarative style + """ + + def __init__(self, path, model, subclasses={}, query=None, parent = None): + self._model = model + self._query = query + self._subclasses = subclasses + self._parent = parent + self.filter = self.where # alias + if isinstance(path, Path): + self._path = path + else: + self._path = model.make_path(path, subclasses) + self._branches = {} + + def select(self, *cols): + """ + Create a new query with this column as the base class, selecting the given fields. + """ + q = self._model.service.new_query(str(self)) + q.select(*cols) + return q + + def where(self, *args, **kwargs): + """ + Create a new query based on this column, filtered with the given constraint. + + also available as "filter" + """ + q = self._model.service.new_query(str(self)) + return q.where(*args, **kwargs) + + def __iter__(self): + """ + Return a query for all objects of this class in the given webservice + """ + q = self.select("*") + return iter(q) + + def __getattr__(self, name): + if name in self._branches: + return self._branches[name] + cld = self._path.get_class() + if cld is not None: + try: + fld = cld.get_field(name) + branch = Column(str(self) + "." + name, self._model, self._subclasses, self._query, self) + self._branches[name] = branch + return branch + except ModelError, e: + raise AttributeError(str(e)) + raise AttributeError("No attribute '" + name + "'") + + def __str__(self): + return str(self._path) + + def __mod__(self, other): + if isinstance(other, tuple): + return ConstraintNode(str(self), 'LOOKUP', *other) + else: + return ConstraintNode(str(self), 'LOOKUP', str(other)) + + def __rshift__(self, other): + return CodelessNode(str(self), str(other)) + + __lshift__ = __rshift__ + + def __eq__(self, other): + if other is None: + return ConstraintNode(str(self), "IS NULL") + elif isinstance(other, Column): + return ConstraintNode(str(self), "IS", str(other)) + elif hasattr(other, "make_list_constraint"): + return other.make_list_constraint(str(self), "IN") + elif isinstance(other, list): + return ConstraintNode(str(self), "ONE OF", other) + else: + return ConstraintNode(str(self), "=", other) + + def __ne__(self, other): + if other is None: + return ConstraintNode(str(self), "IS NOT NULL") + elif isinstance(other, Column): + return ConstraintNode(str(self), "IS NOT", str(other)) + elif hasattr(other, "make_list_constraint"): + return other.make_list_constraint(str(self), "NOT IN") + elif isinstance(other, list): + return ConstraintNode(str(self), "NONE OF", other) + else: + return ConstraintNode(str(self), "!=", other) + + def __xor__(self, other): + if hasattr(other, "make_list_constraint"): + return other.make_list_constraint(str(self), "NOT IN") + elif isinstance(other, list): + return ConstraintNode(str(self), "NONE OF", other) + raise TypeError("Invalid argument for xor: %r" % other) + + def in_(self, other): + if hasattr(other, "make_list_constraint"): + return other.make_list_constraint(str(self), "IN") + elif isinstance(other, list): + return ConstraintNode(str(self), "ONE OF", other) + raise TypeError("Invalid argument for in_: %r" % other) + + def __lt__(self, other): + if isinstance(other, Column): + self._parent._subclasses[str(self)] = str(other) + self._parent._branches = {} + return CodelessNode(str(self), str(other)) + try: + return self.in_(other) + except TypeError: + return ConstraintNode(str(self), "<", other) + + def __le__(self, other): + if isinstance(other, Column): + return CodelessNode(str(self), str(other)) + try: + return self.in_(other) + except TypeError: + return ConstraintNode(str(self), "<=", other) + + def __gt__(self, other): + return ConstraintNode(str(self), ">", other) + + def __ge__(self, other): + return ConstraintNode(str(self), ">=", other) + +class Model(object): + """ + A class for representing the data model of an InterMine datawarehouse + ===================================================================== + + An abstraction of the database schema + + SYNOPSIS + -------- + + >>> service = Service("http://www.flymine.org/query/service") + >>> model = service.model + >>> model.get_class("Gene") + + + OVERVIEW + -------- + + This class represents the data model - ie. an abstraction + of the database schema. It can be used to introspect what + data is available and how it is inter-related + """ + + NUMERIC_TYPES = frozenset(["int", "Integer", "float", "Float", "double", "Double", "long", "Long", "short", "Short"]) + + def __init__(self, source, service=None): + """ + Constructor + =========== + + >>> model = Model(xml) + + You will most like not need to create a model directly, + instead get one from the Service object: + + @see: L{intermine.webservice.Service} + + @param source: the model.xml, as a local file, string, or url + """ + assert source is not None + self.source = source + if service is not None: + self.service = weakref.proxy(service) + else: + self.service = service + self.classes= {} + self.parse_model(source) + self.vivify() + + # Make sugary aliases + self.table = self.column + + def parse_model(self, source): + """ + Create classes, attributes, references and collections from the model.xml + ========================================================================= + + The xml can be provided as a file, url or string. This method + is called during instantiation - it does not need to be called + directly. + + @param source: the model.xml, as a local file, string, or url + @raise ModelParseError: if there is a problem parsing the source + """ + try: + io = openAnything(source) + doc = minidom.parse(io) + for node in doc.getElementsByTagName('model'): + self.name = node.getAttribute('name') + self.package_name = node.getAttribute('package') + assert node.nextSibling is None, "More than one model element" + assert self.name and self.package_name, "No model name or package name" + + for c in doc.getElementsByTagName('class'): + class_name = c.getAttribute('name') + assert class_name, "Name not defined in" + c.toxml() + def strip_java_prefix(x): + return re.sub(r'.*\.', '', x) + parents = map(strip_java_prefix, + c.getAttribute('extends').split(' ')) + cl = Class(class_name, parents, self) + for a in c.getElementsByTagName('attribute'): + name = a.getAttribute('name') + type_name = strip_java_prefix(a.getAttribute('type')) + at = Attribute(name, type_name, cl) + cl.field_dict[name] = at + for r in c.getElementsByTagName('reference'): + name = r.getAttribute('name') + type_name = r.getAttribute('referenced-type') + linked_field_name = r.getAttribute('reverse-reference') + ref = Reference(name, type_name, cl, linked_field_name) + cl.field_dict[name] = ref + for co in c.getElementsByTagName('collection'): + name = co.getAttribute('name') + type_name = co.getAttribute('referenced-type') + linked_field_name = co.getAttribute('reverse-reference') + col = Collection(name, type_name, cl, linked_field_name) + cl.field_dict[name] = col + self.classes[class_name] = cl + except Exception, error: + raise ModelParseError("Error parsing model", source, error) + + def vivify(self): + """ + Make names point to instances and insert inherited fields + ========================================================= + + This method ensures the model is internally consistent. This method + is called during instantiaton. It does not need to be called + directly. + + @raise ModelError: if the names point to non-existent objects + """ + for c in self.classes.values(): + c.parent_classes = self.to_ancestry(c) + for pc in c.parent_classes: + c.field_dict.update(pc.field_dict) + for f in c.fields: + f.type_class = self.classes.get(f.type_name) + if hasattr(f, 'reverse_reference_name') and f.reverse_reference_name != '': + rrn = f.reverse_reference_name + f.reverse_reference = f.type_class.field_dict[rrn] + + def to_ancestry(self, cd): + """ + Returns the lineage of the class + ================================ + + >>> classes = Model.to_ancestry(cd) + + Returns the class' parents, and all the class' parents' parents + + @rtype: list(L{intermine.model.Class}) + """ + parents = cd.parents + def defined(x): return x is not None # weeds out the java classes + def to_class(x): return self.classes.get(x) + ancestry = filter(defined, map(to_class, parents)) + for ancestor in ancestry: + ancestry.extend(self.to_ancestry(ancestor)) + return ancestry + + def to_classes(self, classnames): + """ + take a list of class names and return a list of classes + ======================================================= + + >>> classes = model.to_classes(["Gene", "Protein", "Organism"]) + + This simply maps from a list of strings to a list of + classes in the calling model. + + @raise ModelError: if the list of class names includes ones that don't exist + + @rtype: list(L{intermine.model.Class}) + """ + return map(self.get_class, classnames) + + def column(self, path, *rest): + return Column(path, self, *rest) + + def __getattr__(self, name): + return self.column(name) + + def get_class(self, name): + """ + Get a class by its name, or by a dotted path + ============================================ + + >>> model = Model("http://www.flymine.org/query/service/model") + >>> model.get_class("Gene") + + >>> model.get_class("Gene.proteins") + + + This is the recommended way of retrieving a class from + the model. As well as handling class names, you can also + pass in a path such as "Gene.proteins" and get the + corresponding class back () + + @raise ModelError: if the class name refers to a non-existant object + + @rtype: L{intermine.model.Class} + """ + if name.find(".") != -1: + path = self.make_path(name) + if path.is_attribute(): + raise ModelError("'" + str(path) + "' is not a class") + else: + return path.get_class() + if name in self.classes: + return self.classes[name] + else: + raise ModelError("'" + name + "' is not a class in this model") + + def make_path(self, path, subclasses={}): + """ + Return a path object for the given path string + ============================================== + + >>> path = Model.make_path("Gene.organism.name") + + + This is recommended manner of constructing path objects. + + @type path: str + @type subclasses: dict + + @raise PathParseError: if there is a problem parsing the path string + + @rtype: L{intermine.model.Path} + """ + return Path(path, self, subclasses) + + def validate_path(self, path_string, subclasses={}): + """ + Validate a path + =============== + + >>> try: + ... model.validate_path("Gene.symbol") + ... return "path is valid" + ... except PathParseError: + ... return "path is invalid" + "path is valid" + + When you don't need to interrogate relationships + between paths, simply using this method to validate + a path string is enough. It guarantees that there + is a descriptor for each section of the string, + with the appropriate relationships + + @raise PathParseError: if there is a problem parsing the path string + """ + try: + self.parse_path_string(path_string, subclasses) + return True + except PathParseError, e: + raise PathParseError("Error parsing '%s' (subclasses: %s)" + % ( path_string, str(subclasses) ), e ) + + def parse_path_string(self, path_string, subclasses={}): + """ + Parse a path string into a list of descriptors - one for each section + ===================================================================== + + >>> parts = Model.parse_path_string(string) + + This method is used when making paths from a model, and + when validating path strings. It probably won't need to + be called directly. + + @see: L{intermine.model.Model.make_path} + @see: L{intermine.model.Model.validate_path} + @see: L{intermine.model.Path} + """ + descriptors = [] + names = path_string.split('.') + root_name = names.pop(0) + + root_descriptor = self.get_class(root_name) + descriptors.append(root_descriptor) + + if root_name in subclasses: + current_class = self.get_class(subclasses[root_name]) + else: + current_class = root_descriptor + + for field_name in names: + field = current_class.get_field(field_name) + descriptors.append(field) + + if isinstance(field, Reference): + key = '.'.join(map(lambda x: x.name, descriptors)) + if key in subclasses: + current_class = self.get_class(subclasses[key]) + else: + current_class = field.type_class + else: + current_class = None + + return descriptors + + def _unproxied(self): + return self + +class ModelError(ReadableException): + pass + +class PathParseError(ModelError): + pass + +class ModelParseError(ModelError): + + def __init__(self, message, source, cause=None): + self.source = source + super(ModelParseError, self).__init__(message, cause) + + def __str__(self): + base = repr(self.message) + ":" + repr(self.source) + if self.cause is None: + return base + else: + return base + repr(self.cause) + diff --git a/intermine/pathfeatures.py b/intermine/pathfeatures.py new file mode 100644 index 00000000..26c2cb25 --- /dev/null +++ b/intermine/pathfeatures.py @@ -0,0 +1,113 @@ +import re + +PATTERN_STR = "^(?:\w+\.)*\w+$" +PATH_PATTERN = re.compile(PATTERN_STR) + +class PathFeature(object): + def __init__(self, path): + if not PATH_PATTERN.match(path): + raise TypeError( + "Path '" + path + "' does not match expected pattern" + PATTERN_STR) + self.path = path + def __repr__(self): + return "<" + self.__class__.__name__ + ": " + self.to_string() + ">" + def to_string(self): + return str(self.path) + def to_dict(self): + return { 'path' : self.path } + @property + def child_type(self): + raise AttributeError() + +class Join(PathFeature): + valid_join_styles = ['OUTER', 'INNER'] + INNER = "INNER" + OUTER = "OUTER" + child_type = 'join' + def __init__(self, path, style='OUTER'): + if style.upper() not in Join.valid_join_styles: + raise TypeError("Unknown join style: " + style) + self.style = style.upper() + super(Join, self).__init__(path) + def to_dict(self): + d = super(Join, self).to_dict() + d.update(style=self.style) + return d + def __repr__(self): + return('<' + self.__class__.__name__ + + ' '.join([':', self.path, self.style]) + '>') + +class PathDescription(PathFeature): + child_type = 'pathDescription' + def __init__(self, path, description): + self.description = description + super(PathDescription, self).__init__(path) + def to_dict(self): + d = super(PathDescription, self).to_dict() + d.update(description=self.description) + return d + +class SortOrder(PathFeature): + ASC = "asc" + DESC = "desc" + DIRECTIONS = frozenset(["asc", "desc"]) + def __init__(self, path, order): + try: + order = order.lower() + except: + pass + + if not order in self.DIRECTIONS: + raise TypeError("Order must be one of " + str(self.DIRECTIONS) + + " - not " + order) + self.order = order + super(SortOrder, self).__init__(path) + def __str__(self): + return self.path + " " + self.order + def to_string(self): + return str(self) + +class SortOrderList(object): + """ + A container implementation for holding sort orders + ================================================== + + This class exists to hold the sort order information for a + query. It handles appending elements, and the stringification + of the sort order. + """ + def __init__(self, *sos): + self.sort_orders = [] + self.append(*sos) + def append(self, *sos): + """ + Add sort order elements to the sort order list. + =============================================== + + Elements can be provided as a SortOrder object or + as a tuple of arguments (path, direction). + """ + for so in sos: + if isinstance(so, SortOrder): + self.sort_orders.append(so) + elif isinstance(so, tuple): + self.sort_orders.append(SortOrder(*so)) + else: + raise TypeError( + "Sort orders must be either SortOrder instances," + + " or tuples of arguments: I got:" + so + sos) + def __repr__(self): + return '<' + self.class__.__name__ + ': [' + str(self) + ']>' + def __str__(self): + return " ".join(map(str, self.sort_orders)) + def clear(self): + self.sort_orders = [] + def is_empty(self): + return len(self.sort_orders) == 0 + def __len__(self): + return len(self.sort_orders) + def next(self): + return self.sort_orders.next() + def __iter__(self): + return iter(self.sort_orders) + diff --git a/intermine/query.py b/intermine/query.py new file mode 100644 index 00000000..1df688d9 --- /dev/null +++ b/intermine/query.py @@ -0,0 +1,1871 @@ +import re +from copy import deepcopy +from xml.dom import minidom, getDOMImplementation + +from intermine.util import openAnything, ReadableException +from intermine.pathfeatures import PathDescription, Join, SortOrder, SortOrderList +from intermine.model import Column, Class, Model, Reference, ConstraintNode +import constraints + +""" +Classes representing queries against webservices +================================================ + +Representations of queries, and templates. + +""" + +__author__ = "Alex Kalderimis" +__organization__ = "InterMine" +__license__ = "LGPL" +__contact__ = "dev@intermine.org" + + +class Query(object): + """ + A Class representing a structured database query + ================================================ + + Objects of this class have properties that model the + attributes of the query, and methods for performing + the request. + + SYNOPSIS + -------- + + example: + + >>> service = Service("http://www.flymine.org/query/service") + >>> query = service.new_query() + >>> + >>> query.add_view("Gene.symbol", "Gene.pathways.name", "Gene.proteins.symbol") + >>> query.add_sort_order("Gene.pathways.name") + >>> + >>> query.add_constraint("Gene", "LOOKUP", "eve") + >>> query.add_constraint("Gene.pathways.name", "=", "Phosphate*") + >>> + >>> query.set_logic("A or B") + >>> + >>> for row in query.rows(): + ... handle_row(row) + + OR, using an SQL style DSL: + + >>> s = Service("www.flymine.org/query") + >>> query = s.query("Gene").\\ + ... select("*", "pathways.*").\\ + ... where("symbol", "=", "H").\\ + ... outerjoin("pathways").\\ + ... order_by("symbol") + >>> for row in query.rows(start=10, size=5): + ... handle_row(row) + + OR, for a more SQL-alchemy, ORM style: + + >>> for gene in s.query(s.model.Gene).filter(s.model.Gene.symbol == ["zen", "H", "eve"]).add_columns(s.model.Gene.alleles): + ... handle(gene) + + Query objects represent structured requests for information over the database + housed at the datawarehouse whose webservice you are querying. They utilise + some of the concepts of relational databases, within an object-related + ORM context. If you don't know what that means, don't worry: you + don't need to write SQL, and the queries will be fast. + + To make things slightly more familiar to those with knowledge of SQL, some syntactical + sugar is provided to make constructing queries a bit more recognisable. + + PRINCIPLES + ---------- + + The data model represents tables in the databases as classes, with records + within tables as instances of that class. The columns of the database are the + fields of that object:: + + The Gene table - showing two records/objects + +---------------------------------------------------+ + | id | symbol | length | cyto-location | organism | + +----------------------------------------+----------+ + | 01 | eve | 1539 | 46C10-46C10 | 01 | + +----------------------------------------+----------+ + | 02 | zen | 1331 | 84A5-84A5 | 01 | + +----------------------------------------+----------+ + ... + + The organism table - showing one record/object + +----------------------------------+ + | id | name | taxon id | + +----------------------------------+ + | 01 | D. melanogaster | 7227 | + +----------------------------------+ + + Columns that contain a meaningful value are known as 'attributes' (in the tables above, that is + everything except the id columns). The other columns (such as "organism" in the gene table) + are ones that reference records of other tables (ie. other objects), and are called + references. You can refer to any field in any class, that has a connection, + however tenuous, with a table, by using dotted path notation:: + + Gene.organism.name -> the name column in the organism table, referenced by a record in the gene table + + These paths, and the connections between records and tables they represent, + are the basis for the structure of InterMine queries. + + THE STUCTURE OF A QUERY + ----------------------- + + A query has two principle sets of properties: + - its view: the set of output columns + - its constraints: the set of rules for what to include + + A query must have at least one output column in its view, but constraints + are optional - if you don't include any, you will get back every record + from the table (every object of that type) + + In addition, the query must be coherent: if you have information about + an organism, and you want a list of genes, then the "Gene" table + should be the basis for your query, and as such the Gene class, which + represents this table, should be the root of all the paths that appear in it: + + So, to take a simple example:: + + I have an organism name, and I want a list of genes: + + The view is the list of things I want to know about those genes: + + >>> query.add_view("Gene.name") + >>> query.add_view("Gene.length") + >>> query.add_view("Gene.proteins.sequence.length") + + Note I can freely mix attributes and references, as long as every view ends in + an attribute (a meaningful value). As a short-cut I can also write: + + >>> query.add_views("Gene.name", "Gene.length", "Gene.proteins.sequence.length") + + or: + + >>> query.add_views("Gene.name Gene.length Gene.proteins.sequence.length") + + They are all equivalent. You can also use common SQL style shortcuts such as "*" for all + attribute fields: + + >>> query.add_views("Gene.*") + + You can also use "select" as a synonymn for "add_view" + + Now I can add my constraints. As, we mentioned, I have information about an organism, so: + + >>> query.add_constraint("Gene.organism.name", "=", "D. melanogaster") + + (note, here I can use "where" as a synonymn for "add_constraint") + + If I run this query, I will get literally millions of results - + it needs to be filtered further: + + >>> query.add_constraint("Gene.proteins.sequence.length", "<", 500) + + If that doesn't restrict things enough I can add more filters: + + >>> query.add_constraint("Gene.symbol", "ONE OF", ["eve", "zen", "h"]) + + Now I am guaranteed to get only information on genes I am interested in. + + Note, though, that because I have included the link (or "join") from Gene -> Protein, + this, by default, means that I only want genes that have protein information associated + with them. If in fact I want information on all genes, and just want to know the + protein information if it is available, then I can specify that with: + + >>> query.add_join("Gene.proteins", "OUTER") + + And if perhaps my query is not as simple as a strict cumulative filter, but I want all + D. mel genes that EITHER have a short protein sequence OR come from one of my favourite genes + (as unlikely as that sounds), I can specify the logic for that too: + + >>> query.set_logic("A and (B or C)") + + Each letter refers to one of the constraints - the codes are assigned in the order you add + the constraints. If you want to be absolutely certain about the constraints you mean, you + can use the constraint objects themselves: + + >>> gene_is_eve = query.add_constraint("Gene.symbol", "=", "eve") + >>> gene_is_zen = query.add_constraint("Gene.symbol", "=", "zne") + >>> + >>> query.set_logic(gene_is_eve | gene_is_zen) + + By default the logic is a straight cumulative filter (ie: A and B and C and D and ...) + + Putting it all together: + + >>> query.add_view("Gene.name", "Gene.length", "Gene.proteins.sequence.length") + >>> query.add_constraint("Gene.organism.name", "=", "D. melanogaster") + >>> query.add_constraint("Gene.proteins.sequence.length", "<", 500) + >>> query.add_constraint("Gene.symbol", "ONE OF", ["eve", "zen", "h"]) + >>> query.add_join("Gene.proteins", "OUTER") + >>> query.set_logic("A and (B or C)") + + This can be made more concise and readable with a little DSL sugar: + + >>> query = service.query("Gene") + >>> query.select("name", "length", "proteins.sequence.length").\ + ... where('organism.name' '=', 'D. melanogaster').\ + ... where("proteins.sequence.length", "<", 500).\ + ... where('symbol', 'ONE OF', ['eve', 'h', 'zen']).\ + ... outerjoin('proteins').\ + ... set_logic("A and (B or C)") + + And the query is defined. + + Result Processing: Rows + ----------------------- + + calling ".rows()" on a query will return an iterator of rows, where each row + is a ResultRow object, which can be treated as both a list and a dictionary. + + Which means you can refer to columns by name: + + >>> for row in query.rows(): + ... print "name is %s" % (row["name"]) + ... print "length is %d" % (row["length"]) + + As well as using list indices: + + >>> for row in query.rows(): + ... print "The first column is %s" % (row[0]) + + Iterating over a row iterates over the cell values as a list: + + >>> for row in query.rows(): + ... for column in row: + ... do_something(column) + + Here each row will have a gene name, a gene length, and a sequence length, eg: + + >>> print row.to_l + ["even skipped", "1359", "376"] + + To make that clearer, you can ask for a dictionary instead of a list: + + >>> for row in query.rows() + ... print row.to_d + {"Gene.name":"even skipped","Gene.length":"1359","Gene.proteins.sequence.length":"376"} + + + If you just want the raw results, for printing to a file, or for piping to another program, + you can request strings instead: + + >>> for row in query.result("string") + ... print(row) + + Result Processing: Results + -------------------------- + + Results can also be processing on a record by record basis. If you have a query that + has output columns of "Gene.symbol", "Gene.pathways.name" and "Gene.proteins.proteinDomains.primaryIdentifier", + than processing it by records will return one object per gene, and that gene will have a property + named "pathways" which contains objects which have a name property. Likewise there will be a + proteins property which holds a list of proteinDomains which all have a primaryIdentifier property, and so on. + This allows a more object orientated approach to database records, familiar to users of + other ORMs. + + This is the format used when you choose to iterate over a query directly, or can be explicitly + chosen by invoking L{intermine.query.Query.results}: + + >>> for gene in query: + ... print gene.name, map(lambda x: x.name, gene.pathways) + + The structure of the object and the information it contains depends entirely + on the output columns selected. The values may be None, of course, but also any valid values of an object + (according to the data model) will also be None if they were not selected for output. Attempts + to access invalid properties (such as gene.favourite_colour) will cause exceptions to be thrown. + + Getting us to Generate your Code + -------------------------------- + + Not that you have to actually write any of this! The webapp will happily + generate the code for any query (and template) you can build in it. A good way to get + started is to use the webapp to generate your code, and then run it as scripts + to speed up your queries. You can always tinker with and edit the scripts you download. + + To get generated queries, look for the "python" link at the bottom of query-builder and + template form pages, it looks a bit like this:: + + . +=====================================+============= + | | + | Perl | Python | Java [Help] | + | | + +============================================== + + """ + + SO_SPLIT_PATTERN = re.compile("\s*(asc|desc)\s*", re.I) + LOGIC_SPLIT_PATTERN = re.compile("\s*(?:and|or|\(|\))\s*", re.I) + TRAILING_OP_PATTERN = re.compile("\s*(and|or)\s*$", re.I) + LEADING_OP_PATTERN = re.compile("^\s*(and|or)\s*", re.I) + ORPHANED_OP_PATTERN = re.compile("(?:\(\s*(?:and|or)\s*|\s*(?:and|or)\s*\))", re.I) + LOGIC_OPS = ["and", "or"] + LOGIC_PRODUCT = [(x, y) for x in LOGIC_OPS for y in LOGIC_OPS] + + def __init__(self, model, service=None, validate=True, root=None): + """ + Construct a new Query + ===================== + + Construct a new query for making database queries + against an InterMine data warehouse. + + Normally you would not need to use this constructor + directly, but instead use the factory method on + intermine.webservice.Service, which will handle construction + for you. + + @param model: an instance of L{intermine.model.Model}. Required + @param service: an instance of l{intermine.service.Service}. Optional, + but you will not be able to make requests without one. + @param validate: a boolean - defaults to True. If set to false, the query + will not try and validate itself. You should not set this to false. + + """ + self.model = model + if root is None: + self.root = root + else: + self.root = model.make_path(root).root + + self.name = '' + self.description = '' + self.service = service + self.prefetch_depth = service.prefetch_depth if service is not None else 1 + self.prefetch_id_only = service.prefetch_id_only if service is not None else False + self.do_verification = validate + self.path_descriptions = [] + self.joins = [] + self.constraint_dict = {} + self.uncoded_constraints = [] + self.views = [] + self._sort_order_list = SortOrderList() + self._logic_parser = constraints.LogicParser(self) + self._logic = None + self.constraint_factory = constraints.ConstraintFactory() + + # Set up sugary aliases + self.c = self.column + self.filter = self.where + self.add_column = self.add_view + self.add_columns = self.add_view + self.add_views = self.add_view + self.add_to_select = self.add_view + self.order_by = self.add_sort_order + self.all = self.get_results_list + self.size = self.count + self.summarize = self.summarise + + def __iter__(self): + """Return an iterator over all the objects returned by this query""" + return self.results("jsonobjects") + + def __len__(self): + """Return the number of rows this query will return.""" + return self.count() + + def __sub__(self, other): + """Construct a new list from the symmetric difference of these things""" + return self.service._list_manager.subtract([self], [other]) + + def __xor__(self, other): + """Calculate the symmetric difference of this query and another""" + return self.service._list_manager.xor([self, other]) + + def __and__(self, other): + """ + Intersect this query and another query or list + """ + return self.service._list_manager.intersect([self, other]) + + def __or__(self, other): + """ + Return the union of this query and another query or list. + """ + return self.service._list_manager.union([self, other]) + + def __add__(self, other): + """ + Return the union of this query and another query or list + """ + return self.service._list_manager.union([self, other]) + + @classmethod + def from_xml(cls, xml, *args, **kwargs): + """ + Deserialise a query serialised to XML + ===================================== + + This method is used to instantiate serialised queries. + It is used by intermine.webservice.Service objects + to instantiate Template objects and it can be used + to read in queries you have saved to a file. + + @param xml: The xml as a file name, url, or string + + @raise QueryParseError: if the query cannot be parsed + @raise ModelError: if the query has illegal paths in it + @raise ConstraintError: if the constraints don't make sense + + @rtype: L{Query} + """ + obj = cls(*args, **kwargs) + obj.do_verification = False + f = openAnything(xml) + doc = minidom.parse(f) + f.close() + + queries = doc.getElementsByTagName('query') + if len(queries) != 1: + raise QueryParseError("wrong number of queries in xml. " + + "Only one element is allowed. Found %d" % len(queries)) + q = queries[0] + obj.name = q.getAttribute('name') + obj.description = q.getAttribute('description') + obj.add_view(q.getAttribute('view')) + for p in q.getElementsByTagName('pathDescription'): + path = p.getAttribute('pathString') + description = p.getAttribute('description') + obj.add_path_description(path, description) + for j in q.getElementsByTagName('join'): + path = j.getAttribute('path') + style = j.getAttribute('style') + obj.add_join(path, style) + for c in q.getElementsByTagName('constraint'): + args = {} + args['path'] = c.getAttribute('path') + if args['path'] is None: + if c.parentNode.tagName != "node": + msg = "Constraints must have a path" + raise QueryParseError(msg) + args['path'] = c.parentNode.getAttribute('path') + args['op'] = c.getAttribute('op') + args['value'] = c.getAttribute('value') + args['code'] = c.getAttribute('code') + args['subclass'] = c.getAttribute('type') + args['editable'] = c.getAttribute('editable') + args['optional'] = c.getAttribute('switchable') + args['extra_value'] = c.getAttribute('extraValue') + args['loopPath'] = c.getAttribute('loopPath') + values = [] + for val_e in c.getElementsByTagName('value'): + texts = [] + for node in val_e.childNodes: + if node.nodeType == node.TEXT_NODE: texts.append(node.data) + values.append(' '.join(texts)) + if len(values) > 0: args["values"] = values + for k, v in args.items(): + if v is None or v == '': del args[k] + if "loopPath" in args: + args["op"] = { + "=" : "IS", + "!=": "IS NOT" + }.get(args["op"]) + con = obj.add_constraint(**args) + if not con: + raise ConstraintError("error adding constraint with args: " + args) + + def group(iterator, count): + itr = iter(iterator) + while True: + yield tuple([itr.next() for i in range(count)]) + + if q.getAttribute('sortOrder') is not None: + sos = Query.SO_SPLIT_PATTERN.split(q.getAttribute('sortOrder')) + if len(sos) == 1: + if sos[0] in obj.views: # Be tolerant of irrelevant sort-orders + obj.add_sort_order(sos[0]) + else: + sos.pop() # Get rid of empty string at end + for path, direction in group(sos, 2): + if path in obj.views: # Be tolerant of irrelevant so. + obj.add_sort_order(path, direction) + + if q.getAttribute('constraintLogic') is not None: + obj._set_questionable_logic(q.getAttribute('constraintLogic')) + + obj.verify() + + return obj + + def _set_questionable_logic(self, questionable_logic): + """Attempts to sanity check the logic argument before it is set""" + logic = questionable_logic + used_codes = set(self.constraint_dict.keys()) + logic_codes = set(Query.LOGIC_SPLIT_PATTERN.split(questionable_logic)) + if "" in logic_codes: + logic_codes.remove("") + irrelevant_codes = logic_codes - used_codes + for c in irrelevant_codes: + pattern = re.compile("\\b" + c + "\\b", re.I) + logic = pattern.sub("", logic) + # Remove empty groups + logic = re.sub("\((:?and|or|\s)*\)", "", logic) + # Remove trailing and leading operators + logic = Query.LEADING_OP_PATTERN.sub("", logic) + logic = Query.TRAILING_OP_PATTERN.sub("", logic) + for x in range(2): # repeat, as this process can leave doubles + for left, right in Query.LOGIC_PRODUCT: + if left == right: + repl = left + else: + repl = "and" + pattern = re.compile(left + "\s*" + right, re.I) + logic = pattern.sub(repl, logic) + logic = Query.ORPHANED_OP_PATTERN.sub(lambda x: "(" if "(" in x.group(0) else ")", logic) + logic = logic.strip().lstrip() + logic = Query.LEADING_OP_PATTERN.sub("", logic) + logic = Query.TRAILING_OP_PATTERN.sub("", logic) + try: + if len(logic) > 0 and logic not in ["and", "or"]: + self.set_logic(logic) + except Exception, e: + raise Exception("Error parsing logic string " + + repr(questionable_logic) + + " (which is " + repr(logic) + " after irrelevant codes have been removed)" + + " with available codes: " + repr(list(used_codes)) + + " because: " + e.message) + + def __str__(self): + """Return the XML serialisation of this query""" + return self.to_xml() + + def verify(self): + """ + Validate the query + ================== + + Invalid queries will fail to run, and it is not always + obvious why. The validation routine checks to see that + the query will not cause errors on execution, and tries to + provide informative error messages. + + This method is called immediately after a query is fully + deserialised. + + @raise ModelError: if the paths are invalid + @raise QueryError: if there are errors in query construction + @raise ConstraintError: if there are errors in constraint construction + + """ + self.verify_views() + self.verify_constraint_paths() + self.verify_join_paths() + self.verify_pd_paths() + self.validate_sort_order() + self.do_verification = True + + def select(self, *paths): + """ + Replace the current selection of output columns with this one + ============================================================= + + example:: + + query.select("*", "proteins.name") + + This method is intended to provide an API familiar to those + with experience of SQL or other ORM layers. This method, in + contrast to other view manipulation methods, replaces + the selection of output columns, rather than appending to it. + + Note that any sort orders that are no longer in the view will + be removed. + + @param paths: The output columns to add + """ + self.views = [] + self.add_view(*paths) + so_elems = self._sort_order_list + self._sort_order_list = SortOrderList() + + for so in so_elems: + if so.path in self.views: + self._sort_order_list.append(so) + return self + + def add_view(self, *paths): + """ + Add one or more views to the list of output columns + =================================================== + + example:: + + query.add_view("Gene.name Gene.organism.name") + + This is the main method for adding views to the list + of output columns. As well as appending views, it + will also split a single, space or comma delimited + string into multiple paths, and flatten out lists, or any + combination. It will also immediately try to validate + the views. + + Output columns must be valid paths according to the + data model, and they must represent attributes of tables + + Also available as: + - add_views + - add_column + - add_columns + - add_to_select + + @see: intermine.model.Model + @see: intermine.model.Path + @see: intermine.model.Attribute + """ + views = [] + for p in paths: + if isinstance(p, (set, list)): + views.extend(list(p)) + elif isinstance(p, Class): + views.append(p.name + ".*") + elif isinstance(p, Column): + if p._path.is_attribute(): + views.append(str(p)) + else: + views.append(str(p) + ".*") + elif isinstance(p, Reference): + views.append(p.name + ".*") + else: + views.extend(re.split("(?:,?\s+|,)", str(p))) + + views = map(self.prefix_path, views) + + views_to_add = [] + for view in views: + if view.endswith(".*"): + view = re.sub("\.\*$", "", view) + scd = self.get_subclass_dict() + def expand(p, level, id_only=False): + if level > 0: + path = self.model.make_path(p, scd) + cd = path.end_class + add_f = lambda x: p + "." + x.name + vs = [p + ".id"] if id_only and cd.has_id else map(add_f, cd.attributes) + next_level = level - 1 + rs_and_cs = cd.references + cd.collections + for r in rs_and_cs: + rp = add_f(r) + if next_level: + self.outerjoin(rp) + vs.extend(expand(rp, next_level, self.prefetch_id_only)) + return vs + else: + return [] + depth = self.prefetch_depth + views_to_add.extend(expand(view, depth)) + else: + views_to_add.append(view) + + if self.do_verification: + self.verify_views(views_to_add) + + self.views.extend(views_to_add) + + return self + + def prefix_path(self, path): + if self.root is None: + if self.do_verification: # eg. not when building from XML + if path.endswith(".*"): + trimmed = re.sub("\.\*$", "", path) + else: + trimmed = path + self.root = self.model.make_path(trimmed, self.get_subclass_dict()).root + return path + else: + if path.startswith(self.root.name): + return path + else: + return self.root.name + "." + path + + def clear_view(self): + """ + Clear the output column list + ============================ + + Deletes all entries currently in the view list. + """ + self.views = [] + + def verify_views(self, views=None): + """ + Check to see if the views given are valid + ========================================= + + This method checks to see if the views: + - are valid according to the model + - represent attributes + + @see: L{intermine.model.Attribute} + + @raise intermine.model.ModelError: if the paths are invalid + @raise ConstraintError: if the paths are not attributes + """ + if views is None: views = self.views + for path in views: + path = self.model.make_path(path, self.get_subclass_dict()) + if not path.is_attribute(): + raise ConstraintError("'" + str(path) + + "' does not represent an attribute") + + def add_constraint(self, *args, **kwargs): + """ + Add a constraint (filter on records) + ==================================== + + example:: + + query.add_constraint("Gene.symbol", "=", "zen") + + This method will try to make a constraint from the arguments + given, trying each of the classes it knows of in turn + to see if they accept the arguments. This allows you + to add constraints of different types without having to know + or care what their classes or implementation details are. + All constraints derive from intermine.constraints.Constraint, + and they all have a path attribute, but are otherwise diverse. + + Before adding the constraint to the query, this method + will also try to check that the constraint is valid by + calling Query.verify_constraint_paths() + + @see: L{intermine.constraints} + + @rtype: L{intermine.constraints.Constraint} + """ + if len(args) == 1 and len(kwargs) == 0: + if isinstance(args[0], tuple): + con = self.constraint_factory.make_constraint(*args[0]) + else: + try: + con = self.constraint_factory.make_constraint(*args[0].vargs, **args[0].kwargs) + except AttributeError: + con = args[0] + else: + if len(args) == 0 and len(kwargs) == 1: + k, v = kwargs.items()[0] + d = {"path": k} + if v in constraints.UnaryConstraint.OPS: + d["op"] = v + else: + d["op"] = "=" + d["value"] = v + kwargs = d + + con = self.constraint_factory.make_constraint(*args, **kwargs) + + con.path = self.prefix_path(con.path) + if self.do_verification: self.verify_constraint_paths([con]) + if hasattr(con, "code"): + self.constraint_dict[con.code] = con + else: + self.uncoded_constraints.append(con) + + return con + + def where(self, *cons, **kwargs): + """ + Add a constraint to the query + ============================= + + In contrast to add_constraint, this method returns + a new object with the given comstraint added. + + Also available as Query.filter + """ + c = self.clone() + try: + for conset in cons: + codeds = c.coded_constraints + lstr = str(c.get_logic()) + " AND " if codeds else "" + start_c = chr(ord(codeds[-1].code) + 1) if codeds else 'A' + for con in conset: + c.add_constraint(*con.vargs, **con.kwargs) + try: + c.set_logic(lstr + conset.as_logic(start = start_c)) + except constraints.EmptyLogicError: + pass + for path, value in kwargs.items(): + c.add_constraint(path, "=", value) + except AttributeError: + c.add_constraint(*cons, **kwargs) + return c + + def column(self, col): + """ + Return a Column object suitable for using to construct constraints with + ======================================================================= + + This method is part of the SQLAlchemy style API. + + Also available as Query.c + """ + return self.model.column(self.prefix_path(str(col)), self.get_subclass_dict(), self) + + def verify_constraint_paths(self, cons=None): + """ + Check that the constraints are valid + ==================================== + + This method will check the path attribute of each constraint. + In addition it will: + - Check that BinaryConstraints and MultiConstraints have an Attribute as their path + - Check that TernaryConstraints have a Reference as theirs + - Check that SubClassConstraints have a correct subclass relationship + - Check that LoopConstraints have a valid loopPath, of a compatible type + - Check that ListConstraints refer to an object + - Don't even try to check RangeConstraints: these have variable semantics + + @param cons: The constraints to check (defaults to all constraints on the query) + + @raise ModelError: if the paths are not valid + @raise ConstraintError: if the constraints do not satisfy the above rules + + """ + if cons is None: cons = self.constraints + for con in cons: + pathA = self.model.make_path(con.path, self.get_subclass_dict()) + if isinstance(con, constraints.RangeConstraint): + pass # No verification done on these, beyond checking its path, of course. + elif isinstance(con, constraints.TernaryConstraint): + if pathA.get_class() is None: + raise ConstraintError("'" + str(pathA) + "' does not represent a class, or a reference to a class") + elif isinstance(con, constraints.BinaryConstraint) or isinstance(con, constraints.MultiConstraint): + if not pathA.is_attribute(): + raise ConstraintError("'" + str(pathA) + "' does not represent an attribute") + elif isinstance(con, constraints.SubClassConstraint): + pathB = self.model.make_path(con.subclass, self.get_subclass_dict()) + if not pathB.get_class().isa(pathA.get_class()): + raise ConstraintError("'" + con.subclass + "' is not a subclass of '" + con.path + "'") + elif isinstance(con, constraints.LoopConstraint): + pathB = self.model.make_path(con.loopPath, self.get_subclass_dict()) + for path in [pathA, pathB]: + if not path.get_class(): + raise ConstraintError("'" + str(path) + "' does not refer to an object") + (classA, classB) = (pathA.get_class(), pathB.get_class()) + if not classA.isa(classB) and not classB.isa(classA): + raise ConstraintError("the classes are of incompatible types: " + str(classA) + "," + str(classB)) + elif isinstance(con, constraints.ListConstraint): + if not pathA.get_class(): + raise ConstraintError("'" + str(pathA) + "' does not refer to an object") + + @property + def constraints(self): + """ + Returns the constraints of the query + ==================================== + + Query.constraints S{->} list(intermine.constraints.Constraint) + + Constraints are returned in the order of their code (normally + the order they were added to the query) and with any + subclass contraints at the end. + + @rtype: list(Constraint) + """ + ret = sorted(self.constraint_dict.values(), key=lambda con: con.code) + ret.extend(self.uncoded_constraints) + return ret + + def get_constraint(self, code): + """ + Returns the constraint with the given code + ========================================== + + Returns the constraint with the given code, if if exists. + If no such constraint exists, it throws a ConstraintError + + @return: the constraint corresponding to the given code + @rtype: L{intermine.constraints.CodedConstraint} + """ + if code in self.constraint_dict: + return self.constraint_dict[code] + else: + raise ConstraintError("There is no constraint with the code '" + + code + "' on this query") + + def add_join(self, *args ,**kwargs): + """ + Add a join statement to the query + ================================= + + example:: + + query.add_join("Gene.proteins", "OUTER") + + A join statement is used to determine if references should + restrict the result set by only including those references + exist. For example, if one had a query with the view:: + + "Gene.name", "Gene.proteins.name" + + Then in the normal case (that of an INNER join), we would only + get Genes that also have at least one protein that they reference. + Simply by asking for this output column you are placing a + restriction on the information you get back. + + If in fact you wanted all genes, regardless of whether they had + proteins associated with them or not, but if they did + you would rather like to know _what_ proteins, then you need + to specify this reference to be an OUTER join:: + + query.add_join("Gene.proteins", "OUTER") + + Now you will get many more rows of results, some of which will + have "null" values where the protein name would have been, + + This method will also attempt to validate the join by calling + Query.verify_join_paths(). Joins must have a valid path, the + style can be either INNER or OUTER (defaults to OUTER, + as the user does not need to specify inner joins, since all + references start out as inner joins), and the path + must be a reference. + + @raise ModelError: if the path is invalid + @raise TypeError: if the join style is invalid + + @rtype: L{intermine.pathfeatures.Join} + """ + join = Join(*args, **kwargs) + join.path = self.prefix_path(join.path) + if self.do_verification: self.verify_join_paths([join]) + self.joins.append(join) + return self + + def outerjoin(self, column): + """Alias for add_join(column, "OUTER")""" + return self.add_join(str(column), "OUTER") + + def verify_join_paths(self, joins=None): + """ + Check that the joins are valid + ============================== + + Joins must have valid paths, and they must refer to references. + + @raise ModelError: if the paths are invalid + @raise QueryError: if the paths are not references + """ + if joins is None: joins = self.joins + for join in joins: + path = self.model.make_path(join.path, self.get_subclass_dict()) + if not path.is_reference(): + raise QueryError("'" + join.path + "' is not a reference") + + def add_path_description(self, *args ,**kwargs): + """ + Add a path description to the query + =================================== + + example:: + + query.add_path_description("Gene.proteins.proteinDomains", "Protein Domain") + + This allows you to alias the components of long paths to + improve the way they display column headers in a variety of circumstances. + In the above example, if the view included the unwieldy path + "Gene.proteins.proteinDomains.primaryIdentifier", it would (depending on the + mine) be displayed as "Protein Domain > DB Identifer". These + setting are taken into account by the webservice when generating + column headers for flat-file results with the columnheaders parameter given, and + always supplied when requesting jsontable results. + + @rtype: L{intermine.pathfeatures.PathDescription} + + """ + path_description = PathDescription(*args, **kwargs) + path_description.path = self.prefix_path(path_description.path) + if self.do_verification: self.verify_pd_paths([path_description]) + self.path_descriptions.append(path_description) + return path_description + + def verify_pd_paths(self, pds=None): + """ + Check that the path of the path description is valid + ==================================================== + + Checks for consistency with the data model + + @raise ModelError: if the paths are invalid + """ + if pds is None: pds = self.path_descriptions + for pd in pds: + self.model.validate_path(pd.path, self.get_subclass_dict()) + + @property + def coded_constraints(self): + """ + Returns the list of constraints that have a code + ================================================ + + Query.coded_constraints S{->} list(intermine.constraints.CodedConstraint) + + This returns an up to date list of the constraints that can + be used in a logic expression. The only kind of constraint + that this excludes, at present, is SubClassConstraints + + @rtype: list(L{intermine.constraints.CodedConstraint}) + """ + return sorted(self.constraint_dict.values(), key=lambda con: con.code) + + def get_logic(self): + """ + Returns the logic expression for the query + ========================================== + + This returns the up to date logic expression. The default + value is the representation of all coded constraints and'ed together. + + If the logic is empty and there are no constraints, returns an + empty string. + + The LogicGroup object stringifies to a string that can be parsed to + obtain itself (eg: "A and (B or C or D)"). + + @rtype: L{intermine.constraints.LogicGroup} + """ + if self._logic is None: + if len(self.coded_constraints) > 0: + return reduce(lambda x, y: x+y, self.coded_constraints) + else: + return "" + else: + return self._logic + + def set_logic(self, value): + """ + Sets the Logic given the appropriate input + ========================================== + + example:: + + Query.set_logic("A and (B or C)") + + This sets the logic to the appropriate value. If the value is + already a LogicGroup, it is accepted, otherwise + the string is tokenised and parsed. + + The logic is then validated with a call to validate_logic() + + raise LogicParseError: if there is a syntax error in the logic + """ + if isinstance(value, constraints.LogicGroup): + logic = value + else: + try: + logic = self._logic_parser.parse(value) + except constraints.EmptyLogicError: + if self.coded_constraints: + raise + else: + return self + if self.do_verification: self.validate_logic(logic) + self._logic = logic + return self + + def validate_logic(self, logic=None): + """ + Validates the query logic + ========================= + + Attempts to validate the logic by checking + that every coded_constraint is included + at least once + + @raise QueryError: if not every coded constraint is represented + """ + if logic is None: logic = self._logic + logic_codes = set(logic.get_codes()) + for con in self.coded_constraints: + if con.code not in logic_codes: + raise QueryError("Constraint " + con.code + repr(con) + + " is not mentioned in the logic: " + str(logic)) + + def get_default_sort_order(self): + """ + Gets the sort order when none has been specified + ================================================ + + This method is called to determine the sort order if + none is specified + + @raise QueryError: if the view is empty + + @rtype: L{intermine.pathfeatures.SortOrderList} + """ + try: + v0 = self.views[0] + for j in self.joins: + if j.style == "OUTER": + if v0.startswith(j.path): + return "" + return SortOrderList((self.views[0], SortOrder.ASC)) + except IndexError: + raise QueryError("Query view is empty") + + def get_sort_order(self): + """ + Return a sort order for the query + ================================= + + This method returns the sort order if set, otherwise + it returns the default sort order + + @raise QueryError: if the view is empty + + @rtype: L{intermine.pathfeatures.SortOrderList} + """ + if self._sort_order_list.is_empty(): + return self.get_default_sort_order() + else: + return self._sort_order_list + + def add_sort_order(self, path, direction=SortOrder.ASC): + """ + Adds a sort order to the query + ============================== + + example:: + + Query.add_sort_order("Gene.name", "DESC") + + This method adds a sort order to the query. + A query can have multiple sort orders, which are + assessed in sequence. + + If a query has two sort-orders, for example, + the first being "Gene.organism.name asc", + and the second being "Gene.name desc", you would have + the list of genes grouped by organism, with the + lists within those groupings in reverse alphabetical + order by gene name. + + This method will try to validate the sort order + by calling validate_sort_order() + + Also available as Query.order_by + """ + so = SortOrder(str(path), direction) + so.path = self.prefix_path(so.path) + if self.do_verification: self.validate_sort_order(so) + self._sort_order_list.append(so) + return self + + def validate_sort_order(self, *so_elems): + """ + Check the validity of the sort order + ==================================== + + Checks that the sort order paths are: + - valid paths + - in the view + + @raise QueryError: if the sort order is not in the view + @raise ModelError: if the path is invalid + + """ + if not so_elems: + so_elems = self._sort_order_list + from_paths = self._from_paths() + for so in so_elems: + p = self.model.make_path(so.path, self.get_subclass_dict()) + if p.prefix() not in from_paths: + raise QueryError("Sort order element %s is not in the query" % so.path) + + def _from_paths(self): + scd = self.get_subclass_dict() + froms = set(map(lambda x: self.model.make_path(x, scd).prefix(), self.views)) + for c in self.constraints: + p = self.model.make_path(c.path, scd) + if p.is_attribute(): + froms.add(p.prefix()) + else: + froms.add(p) + return froms + + def get_subclass_dict(self): + """ + Return the current mapping of class to subclass + =============================================== + + This method returns a mapping of classes used + by the model for assessing whether certain paths are valid. For + intance, if you subclass MicroArrayResult to be FlyAtlasResult, + you can refer to the .presentCall attributes of fly atlas results. + MicroArrayResults do not have this attribute, and a path such as:: + + Gene.microArrayResult.presentCall + + would be marked as invalid unless the dictionary is provided. + + Users most likely will not need to ever call this method. + + @rtype: dict(string, string) + """ + subclass_dict = {} + for c in self.constraints: + if isinstance(c, constraints.SubClassConstraint): + subclass_dict[c.path] = c.subclass + return subclass_dict + + def results(self, row="object", start=0, size=None, summary_path=None): + """ + Return an iterator over result rows + =================================== + + Usage:: + + >>> query = service.model.Gene.select("symbol", "length") + >>> total = 0 + >>> for gene in query.results(): + ... print gene.symbol # handle strings + ... total += gene.length # handle numbers + >>> for row in query.results(row="rr"): + ... print row["symbol"] # handle strings by dict index + ... total += row["length"] # handle numbers by dict index + ... print row["Gene.symbol"] # handle strings by full dict index + ... total += row["Gene.length"] # handle numbers by full dict index + ... print row[0] # handle strings by list index + ... total += row[1] # handle numbers by list index + >>> for d in query.results(row="dict"): + ... print row["Gene.symbol"] # handle strings + ... total += row["Gene.length"] # handle numbers + >>> for l in query.results(row="list"): + ... print row[0] # handle strings + ... total += row[1] # handle numbers + >>> import csv + >>> csv_reader = csv.reader(q.results(row="csv"), delimiter=",", quotechar='"') + >>> for row in csv_reader: + ... print row[0] # handle strings + ... length_sum += int(row[1]) # handle numbers + >>> tsv_reader = csv.reader(q.results(row="tsv"), delimiter="\t") + >>> for row in tsv_reader: + ... print row[0] # handle strings + ... length_sum += int(row[1]) # handle numbers + + This is the general method that allows access to any of the available + result formats. The example above shows the ways these differ in terms + of accessing fields of the rows, as well as dealing with different + data types. Results can either be retrieved as typed values (jsonobjects, + rr ['ResultRows'], dict, list), or as lists of strings (csv, tsv) which then require + further parsing. The default format for this method is "objects", where + information is grouped by its relationships. The other main format is + "rr", which stands for 'ResultRows', and can be accessed directly through + the L{rows} method. + + Note that when requesting object based results (the default), if your query + contains any kind of collection, it is highly likely that start and size won't do what + you think, as they operate only on the underlying + rows used to build up the returned objects. If you want rows + back, you are recommeded to use the simpler rows method. + + If no views have been specified, all attributes of the root class + are selected for output. + + @param row: The format for each result. One of "object", "rr", + "dict", "list", "tsv", "csv", "jsonrows", "jsonobjects" + @type row: string + @param start: the index of the first result to return (default = 0) + @type start: int + @param size: The maximum number of results to return (default = all) + @type size: int + @param summary_path: A column name to optionally summarise. Specifying a path + will force "jsonrows" format, and return an iterator over a list + of dictionaries. Use this when you are interested in processing + a summary in order of greatest count to smallest. + @type summary_path: str or L{intermine.model.Path} + + @rtype: L{intermine.webservice.ResultIterator} + + @raise WebserviceError: if the request is unsuccessful + """ + + to_run = self.clone() + + if len(to_run.views) == 0: + to_run.add_view(to_run.root) + + if "object" in row: + for c in self.coded_constraints: + p = to_run.column(c.path)._path + from_p = p if p.end_class is not None else p.prefix() + if not filter(lambda v: v.startswith(str(from_p)), to_run.views): + if p.is_attribute(): + to_run.add_view(p) + else: + to_run.add_view(p.append("id")) + + path = to_run.get_results_path() + params = to_run.to_query_params() + params["start"] = start + if size: + params["size"] = size + if summary_path: + params["summaryPath"] = to_run.prefix_path(summary_path) + row = "jsonrows" + + view = to_run.views + cld = to_run.root + return to_run.service.get_results(path, params, row, view, cld) + + def rows(self, start=0, size=None): + """ + Return the results as rows of data + ================================== + + This is a shortcut for results("rr") + + Usage:: + + >>> for row in query.rows(start=10, size=10): + ... print row["proteins.name"] + + @param start: the index of the first result to return (default = 0) + @type start: int + @param size: The maximum number of results to return (default = all) + @type size: int + @rtype: iterable + """ + return self.results(row="rr", start=start, size=size) + + def summarise(self, summary_path, **kwargs): + """ + Return a summary of the results for this column. + ================================================ + + Usage:: + >>> query = service.select("Gene.*", "organism.*").where("Gene", "IN", "my-list") + >>> print query.summarise("length")["average"] + ... 12345.67890 + >>> print query.summarise("organism.name")["Drosophila simulans"] + ... 98 + + This method allows you to get statistics summarising the information + from just one column of a query. For numerical columns you get dictionary with + four keys ('average', 'stdev', 'max', 'min'), and for non-numerical + columns you get a dictionary where each item is a key and the values + are the number of occurrences of this value in the column. + + Any key word arguments will be passed to the underlying results call - + so you can limit the result size to the top 100 items by passing "size = 100" + as part of the call. + + @see: L{intermine.query.Query.results} + + @param summary_path: The column to summarise (either in long or short form) + @type summary_path: str or L{intermine.model.Path} + + @rtype: dict + This method is sugar for particular combinations of calls to L{results}. + """ + p = self.model.make_path(self.prefix_path(summary_path), self.get_subclass_dict()) + results = self.results(summary_path = summary_path, **kwargs) + if p.end.type_name in Model.NUMERIC_TYPES: + return dict([ (k, float(v)) for k, v in results.next().iteritems()]) + else: + return dict([ (r["item"], r["count"]) for r in results]) + + def one(self, row="jsonobjects"): + """Return one result, and raise an error if the result size is not 1""" + if row == "jsonobjects": + if self.count() == 1: + return self.first(row) + else: + ret = None + for obj in self.results(): + if ret is not None: + raise QueryError("More than one result received") + else: + ret = obj + if ret is None: + raise QueryError("No results received") + + return ret + else: + c = self.count() + if (c != 1): + raise QueryError("Result size is not one: got %d results" % (c)) + else: + return self.first(row) + + def first(self, row="jsonobjects", start=0, **kw): + """Return the first result, or None if the results are empty""" + if row == "jsonobjects": + size = None + else: + size = 1 + try: + return self.results(row, start=start, size=size, **kw).next() + except StopIteration: + return None + + def get_results_list(self, *args, **kwargs): + """ + Get a list of result rows + ========================= + + This method is a shortcut so that you do not have to + do a list comprehension yourself on the iterator that + is normally returned. If you have a very large result + set (and these can get up to 100's of thousands or rows + pretty easily) you will not want to + have the whole list in memory at once, but there may + be other circumstances when you might want to keep the whole + list in one place. + + It takes all the same arguments and parameters as Query.results + + Also available as Query.all + + @see: L{intermine.query.Query.results} + + """ + rows = self.results(*args, **kwargs) + return [r for r in rows] + + def get_row_list(self, start=0, size=None): + return self.get_results_list("rr", start, size) + + def count(self): + """ + Return the total number of rows this query returns + ================================================== + + Obtain the number of rows a particular query will + return, without having to fetch and parse all the + actual data. This method makes a request to the server + to report the count for the query, and is sugar for a + results call. + + Also available as Query.size + + @rtype: int + @raise WebserviceError: if the request is unsuccessful. + """ + count_str = "" + for row in self.results(row = "count"): + count_str += row + try: + return int(count_str) + except ValueError: + raise ResultError("Server returned a non-integer count: " + count_str) + + def get_list_upload_uri(self): + """ + Returns the uri to use to create a list from this query + ======================================================= + + Query.get_list_upload_uri() -> str + + This method is used internally when performing list operations + on queries. + + @rtype: str + """ + return self.service.root + self.service.QUERY_LIST_UPLOAD_PATH + + def get_list_append_uri(self): + """ + Returns the uri to use to create a list from this query + ======================================================= + + Query.get_list_append_uri() -> str + + This method is used internally when performing list operations + on queries. + + @rtype: str + """ + return self.service.root + self.service.QUERY_LIST_APPEND_PATH + + + def get_results_path(self): + """ + Returns the path section pointing to the REST resource + ====================================================== + + Query.get_results_path() -> str + + Internally, this just calls a constant property + in intermine.service.Service + + @rtype: str + """ + return self.service.QUERY_PATH + + + def children(self): + """ + Returns the child objects of the query + ====================================== + + This method is used during the serialisation of queries + to xml. It is unlikely you will need access to this as a whole. + Consider using "path_descriptions", "joins", "constraints" instead + + @see: Query.path_descriptions + @see: Query.joins + @see: Query.constraints + + @return: the child element of this query + @rtype: list + """ + return sum([self.path_descriptions, self.joins, self.constraints], []) + + def to_query(self): + """ + Implementation of trait that allows use of these objects as queries (casting). + """ + return self + + def make_list_constraint(self, path, op): + """ + Implementation of trait that allows use of these objects in list constraints + """ + l = self.service.create_list(self) + return ConstraintNode(path, op, l.name) + + def to_query_params(self): + """ + Returns the parameters to be passed to the webservice + ===================================================== + + The query is responsible for producing its own query + parameters. These consist simply of: + - query: the xml representation of the query + + @rtype: dict + + """ + xml = self.to_xml() + params = {'query' : xml } + return params + + def to_Node(self): + """ + Returns a DOM node representing the query + ========================================= + + This is an intermediate step in the creation of the + xml serialised version of the query. You probably + won't need to call this directly. + + @rtype: xml.minidom.Node + """ + impl = getDOMImplementation() + doc = impl.createDocument(None, "query", None) + query = doc.documentElement + + query.setAttribute('name', self.name) + query.setAttribute('model', self.model.name) + query.setAttribute('view', ' '.join(self.views)) + query.setAttribute('sortOrder', str(self.get_sort_order())) + query.setAttribute('longDescription', self.description) + if len(self.coded_constraints) > 1: + query.setAttribute('constraintLogic', str(self.get_logic())) + + for c in self.children(): + element = doc.createElement(c.child_type) + for name, value in c.to_dict().items(): + if isinstance(value, (set, list)): + for v in value: + subelement = doc.createElement(name) + text = doc.createTextNode(v) + subelement.appendChild(text) + element.appendChild(subelement) + else: + element.setAttribute(name, value) + query.appendChild(element) + return query + + def to_xml(self): + """ + Return an XML serialisation of the query + ======================================== + + This method serialises the current state of the query to an + xml string, suitable for storing, or sending over the + internet to the webservice. + + @return: the serialised xml string + @rtype: string + """ + n = self.to_Node() + return n.toxml() + + def to_formatted_xml(self): + """ + Return a readable XML serialisation of the query + ================================================ + + This method serialises the current state of the query to an + xml string, suitable for storing, or sending over the + internet to the webservice, only more readably. + + @return: the serialised xml string + @rtype: string + """ + n = self.to_Node() + return n.toprettyxml() + + def clone(self): + """ + Performs a deep clone + ===================== + + This method will produce a clone that is independent, + and can be altered without affecting the original, + but starts off with the exact same state as it. + + The only shared elements should be the model + and the service, which are shared by all queries + that refer to the same webservice. + + @return: same class as caller + """ + newobj = self.__class__(self.model) + for attr in ["joins", "views", "_sort_order_list", "_logic", "path_descriptions", "constraint_dict", "uncoded_constraints"]: + setattr(newobj, attr, deepcopy(getattr(self, attr))) + + for attr in ["name", "description", "service", "do_verification", "constraint_factory", "root"]: + setattr(newobj, attr, getattr(self, attr)) + return newobj + +class Template(Query): + """ + A Class representing a predefined query + ======================================= + + Templates are ways of saving queries + and allowing others to run them + simply. They are the main interface + to querying in the webapp + + SYNOPSIS + -------- + + example:: + + service = Service("http://www.flymine.org/query/service") + template = service.get_template("Gene_Pathways") + for row in template.results(A={"value":"eve"}): + process_row(row) + ... + + A template is a subclass of query that comes predefined. They + are typically retrieved from the webservice and run by specifying + the values for their existing constraints. They are a concise + and powerful way of running queries in the webapp. + + Being subclasses of query, everything is true of them that is true + of a query. They are just less work, as you don't have to design each + one. Also, you can store your own templates in the web-app, and then + access them as a private webservice method, from anywhere, making them + a kind of query in the cloud - for this you will need to authenticate + by providing log in details to the service. + + The most significant difference is how constraint values are specified + for each set of results. + + @see: L{Template.results} + + """ + def __init__(self, *args, **kwargs): + """ + Constructor + =========== + + Instantiation is identical that of queries. As with queries, + these are best obtained from the intermine.webservice.Service + factory methods. + + @see: L{intermine.webservice.Service.get_template} + """ + super(Template, self).__init__(*args, **kwargs) + self.constraint_factory = constraints.TemplateConstraintFactory() + @property + def editable_constraints(self): + """ + Return the list of constraints you can edit + =========================================== + + Template.editable_constraints -> list(intermine.constraints.Constraint) + + Templates have a concept of editable constraints, which + is a way of hiding complexity from users. An underlying query may have + five constraints, but only expose the one that is actually + interesting. This property returns this subset of constraints + that have the editable flag set to true. + """ + isEditable = lambda x: x.editable + return filter(isEditable, self.constraints) + + def to_query_params(self): + """ + Returns the query parameters needed for the webservice + ====================================================== + + Template.to_query_params() -> dict(string, string) + + Overrides the method of the same name in query to provide the + parameters needed by the templates results service. These + are slightly more complex: + - name: The template's name + - for each constraint: (where [i] is an integer incremented for each constraint) + - constraint[i]: the path + - op[i]: the operator + - value[i]: the value + - code[i]: the code + - extra[i]: the extra value for ternary constraints (optional) + + + @rtype: dict + """ + p = {'name' : self.name} + i = 1 + for c in self.editable_constraints: + if not c.switched_on: next + for k, v in c.to_dict().items(): + if k == "extraValue": k = "extra" + if k == "path": k = "constraint" + p[k + str(i)] = v + i += 1 + return p + + def get_results_path(self): + """ + Returns the path section pointing to the REST resource + ====================================================== + + Template.get_results_path() S{->} str + + Internally, this just calls a constant property + in intermine.service.Service + + This overrides the method of the same name in Query + + @return: the path to the REST resource + @rtype: string + """ + return self.service.TEMPLATEQUERY_PATH + + def get_adjusted_template(self, con_values): + """ + Gets a template to run + ====================== + + Template.get_adjusted_template(con_values) S{->} Template + + When templates are run, they are first cloned, and their + values are changed to those desired. This leaves the original + template unchanged so it can be run again with different + values. This method does the cloning and changing of constraint + values + + @raise ConstraintError: if the constraint values specify values for a non-editable constraint. + + @rtype: L{Template} + """ + clone = self.clone() + for code, options in con_values.items(): + con = clone.get_constraint(code) + if not con.editable: + raise ConstraintError("There is a constraint '" + code + + "' on this query, but it is not editable") + try: + for key, value in options.items(): + setattr(con, key, value) + except AttributeError: + setattr(con, "value", options) + return clone + + def results(self, row="object", start=0, size=None, **con_values): + """ + Get an iterator over result rows + ================================ + + This method returns the same values with the + same options as the method of the same name in + Query (see intermine.query.Query). The main difference in in the + arguments. + + The template result methods also accept a key-word pair + set of arguments that are used to supply values + to the editable constraints. eg:: + + template.results( + A = {"value": "eve"}, + B = {"op": ">", "value": 5000} + ) + + The keys should be codes for editable constraints (you can inspect these + with Template.editable_constraints) and the values should be a dictionary + of constraint properties to replace. You can replace the values for + "op" (operator), "value", and "extra_value" and "values" in the case of + ternary and multi constraints. + + @rtype: L{intermine.webservice.ResultIterator} + """ + clone = self.get_adjusted_template(con_values) + return super(Template, clone).results(row, start, size) + + def get_results_list(self, row="object", start=0, size=None, **con_values): + """ + Get a list of result rows + ========================= + + This method performs the same as the method of the + same name in Query, and it shares the semantics of + Template.results(). + + @see: L{intermine.query.Query.get_results_list} + @see: L{intermine.query.Template.results} + + @rtype: list + + """ + clone = self.get_adjusted_template(con_values) + return super(Template, clone).get_results_list(row, start, size) + + def get_row_list(self, start=0, size=None, **con_values): + """Return a list of the rows returned by this query""" + clone = self.get_adjusted_template(con_values) + return super(Template, clone).get_row_list(start, size) + + def rows(self, start=0, size=None, **con_values): + """Get an iterator over the rows returned by this query""" + clone = self.get_adjusted_template(con_values) + return super(Template, clone).rows(start, size) + + def count(self, **con_values): + """ + Return the total number of rows this template returns + ===================================================== + + Obtain the number of rows a particular query will + return, without having to fetch and parse all the + actual data. This method makes a request to the server + to report the count for the query, and is sugar for a + results call. + + @rtype: int + @raise WebserviceError: if the request is unsuccessful. + """ + clone = self.get_adjusted_template(con_values) + return super(Template, clone).count() + + +class QueryError(ReadableException): + pass + +class ConstraintError(QueryError): + pass + +class QueryParseError(QueryError): + pass + +class ResultError(ReadableException): + pass + diff --git a/intermine/results.py b/intermine/results.py new file mode 100644 index 00000000..afb07fef --- /dev/null +++ b/intermine/results.py @@ -0,0 +1,683 @@ +try: + import simplejson as json # Prefer this as it is faster +except ImportError: # pragma: no cover + try: + import json + except ImportError: + raise ImportError("Could not find any JSON module to import - " + + "please install simplejson or jsonlib to continue") + +import urllib +import httplib +import re +import copy +import base64 +from urlparse import urlparse +from itertools import groupby +import UserDict + +from intermine.errors import WebserviceError +from intermine.model import Attribute, Reference, Collection + +USER_AGENT = 'WebserviceInterMinePerlAPIClient' + +class EnrichmentLine(UserDict.UserDict): + """ + An object that represents a result returned from the enrichment service. + ======================================================================== + + These objects operate as dictionaries as well as objects with predefined + properties. + """ + + def __str__(self): + return str(self.data) + + def __repr__(self): + return "EnrichmentLine(%s)" % self.data + + def __getattr__(self, name): + if name is not None: + key_name = name.replace('_', '-') + if key_name in self.keys(): + return self.data[key_name] + raise AttributeError(name) + +class ResultObject(object): + """ + An object used to represent result records as returned in jsonobjects format + ============================================================================ + + These objects are backed by a row of data and the class descriptor that + describes the object. They allow access in standard object style: + + >>> for gene in query.results(): + ... print gene.symbol + ... print map(lambda x: x.name, gene.pathways) + + All objects will have "id" and "type" properties. The type refers to the + actual type of this object: if it is a subclass of the one requested, the + subclass name will be returned. The "id" refers to the internal database id + of the object, and is a guarantor of object identity. + + """ + + def __init__(self, data, cld, view=[]): + stripped = [v[v.find(".") + 1:] for v in view] + self.selected_attributes = [v for v in stripped if "." not in v] + self.reference_paths = dict(((k, list(i)) for k, i in groupby(stripped, lambda x: x[:x.find(".") + 1]))) + self._data = data + self._cld = cld if "class" not in data or cld.name == data["class"] else cld.model.get_class(data["class"]) + self._attr_cache = {} + + def __str__(self): + dont_show = set(["objectId", "class"]) + return "%s(%s)" % (self._cld.name, ", ".join("%s = %r" % (k, v) for k, v in self._data.items() + if not isinstance(v, dict) and not isinstance(v, list) and k not in dont_show)) + + def __repr__(self): + dont_show = set(["objectId", "class"]) + return "%s(%s)" % (self._cld.name, ", ".join("%s = %r" % (k, getattr(self, k)) for k in self._data.keys() + if k not in dont_show)) + + def __getattr__(self, name): + if name in self._attr_cache: + return self._attr_cache[name] + + if name == "type": + return self._data["class"] + + fld = self._cld.get_field(name) + attr = None + if isinstance(fld, Attribute): + if name in self._data: + attr = self._data[name] + if attr is None: + attr = self._fetch_attr(fld) + elif isinstance(fld, Reference): + ref_paths = self._get_ref_paths(fld) + if name in self._data: + data = self._data[name] + else: + data = self._fetch_reference(fld) + if isinstance(fld, Collection): + if data is None: + attr = [] + else: + attr = map(lambda x: ResultObject(x, fld.type_class, ref_paths), data) + else: + if data is None: + attr = None + else: + attr = ResultObject(data, fld.type_class, ref_paths) + else: + raise WebserviceError("Inconsistent model - This should never happen") + self._attr_cache[name] = attr + return attr + + def _get_ref_paths(self, fld): + if fld.name + "." in self.reference_paths: + return self.reference_paths[fld.name + "."] + else: + return [] + + @property + def id(self): + """Return the internal DB identifier of this object. Or None if this is not an InterMine object""" + return self._data.get('objectId') + + def _fetch_attr(self, fld): + if fld.name in self.selected_attributes: + return None # Was originally selected - no point asking twice + c = self._cld + if "id" not in c: + return None # Cannot reliably fetch anything without access to the objectId. + q = c.model.service.query(c, fld).where(id = self.id) + r = q.first() + return r._data[fld.name] if fld.name in r._data else None + + def _fetch_reference(self, ref): + if ref.name + "." in self.reference_paths: + return None # Was originally selected - no point asking twice. + c = self._cld + if "id" not in c: + return None # Cannot reliably fetch anything without access to the objectId. + q = c.model.service.query(ref).outerjoin(ref).where(id = self.id) + r = q.first() + return r._data[ref.name] if ref.name in r._data else None + +class ResultRow(object): + """ + An object for representing a row of data received back from the server. + ======================================================================= + + ResultRows provide access to the fields of the row through index lookup. However, + for convenience both list indexes and dictionary keys can be used. So the + following all work: + + >>> # Assuming the view is "Gene.symbol", "Gene.organism.name": + >>> row[0] == row["symbol"] == row["Gene.symbol"] + ... True + + """ + + def __init__(self, data, views): + self.data = data + self.views = views + self.index_map = None + + def __len__(self): + """Return the number of cells in this row""" + return len(self.data) + + def __iter__(self): + """Return the list view of the row, so each cell can be processed""" + return iter(self.to_l()) + + def _get_index_for(self, key): + if self.index_map is None: + self.index_map = {} + for i in range(len(self.views)): + view = self.views[i] + headless_view = re.sub("^[^.]+.", "", view) + self.index_map[view] = i + self.index_map[headless_view] = i + + return self.index_map[key] + + def __str__(self): + root = re.sub("\..*$", "", self.views[0]) + parts = [root + ":"] + for view in self.views: + short_form = re.sub("^[^.]+.", "", view) + value = self[view] + parts.append(short_form + "=" + repr(value)) + return " ".join(parts) + + def __getitem__(self, key): + if isinstance(key, int): + return self.data[key] + elif isinstance(key, slice): + return self.data[key] + else: + index = self._get_index_for(key) + return self.data[index] + + def to_l(self): + """Return a list view of this row""" + return [x for x in self.data] + + + def to_d(self): + """Return a dictionary view of this row""" + d = {} + for view in self.views: + d[view] = self[view] + + return d + + def items(self): + return [(view, self[view]) for view in self.views] + + def iteritems(self): + for view in self.views: + yield (view, self[view]) + + def keys(self): + return copy.copy(self.views) + + def values(self): + return self.to_l() + + def itervalues(self): + return iter(self.to_l()) + + def iterkeys(self): + return iter(self.views) + + def has_key(self, key): + try: + self._get_index_for(key) + return True + except KeyError: + return False + +class TableResultRow(ResultRow): + """ + A class for parsing results from the jsonrows data format. + """ + + def __getitem__(self, key): + if isinstance(key, int): + return self.data[key]["value"] + elif isinstance(key, slice): + vals = map(lambda x: x["value"], self.data[key]) + return vals + else: + index = self._get_index_for(key) + return self.data[index]["value"] + + def to_l(self): + """Return a list view of this row""" + return map(lambda x: x["value"], self.data) + +class ResultIterator(object): + """ + A facade over the internal iterator object + ========================================== + + These objects handle the iteration over results + in the formats requested by the user. They are responsible + for generating an appropriate parser, + connecting the parser to the results, and delegating + iteration appropriately. + """ + + PARSED_FORMATS = frozenset(["rr", "list", "dict"]) + STRING_FORMATS = frozenset(["tsv", "csv", "count"]) + JSON_FORMATS = frozenset(["jsonrows", "jsonobjects", "json"]) + ROW_FORMATS = PARSED_FORMATS | STRING_FORMATS | JSON_FORMATS + + def __init__(self, service, path, params, rowformat, view, cld=None): + """ + Constructor + =========== + + Services are responsible for getting result iterators. You will + not need to create one manually. + + @param root: The root path (eg: "http://www.flymine.org/query/service") + @type root: string + @param path: The resource path (eg: "/query/results") + @type path: string + @param params: The query parameters for this request + @type params: dict + @param rowformat: One of "rr", "object", "count", "dict", "list", "tsv", "csv", "jsonrows", "jsonobjects", "json" + @type rowformat: string + @param view: The output columns + @type view: list + @param opener: A url opener (user-agent) + @type opener: urllib.URLopener + + @raise ValueError: if the row format is incorrect + @raise WebserviceError: if the request is unsuccessful + """ + if rowformat.startswith("object"): # Accept "object", "objects", "objectformat", etc... + rowformat = "jsonobjects" # these are synonymous + if rowformat not in self.ROW_FORMATS: + raise ValueError("'%s' is not one of the valid row formats (%s)" + % (rowformat, repr(list(self.ROW_FORMATS)))) + + self.row = ResultRow if service.version >= 8 else TableResultRow + + if rowformat in self.PARSED_FORMATS: + if service.version >= 8: + params.update({"format": "json"}) + else: + params.update({"format" : "jsonrows"}) + else: + params.update({"format" : rowformat}) + + self.url = service.root + path + self.data = urllib.urlencode(params) + self.view = view + self.opener = service.opener + self.cld = cld + self.rowformat = rowformat + self._it = None + + def __len__(self): + """ + Return the number of items in this iterator + =========================================== + + Note that this requires iterating over the full result set. + """ + c = 0 + for x in self: + c += 1 + return c + + def __iter__(self): + """ + Return an iterator over the results + =================================== + + Returns the internal iterator object. + """ + con = self.opener.open(self.url, self.data) + identity = lambda x: x + flat_file_parser = lambda: FlatFileIterator(con, identity) + simple_json_parser = lambda: JSONIterator(con, identity) + + try: + reader = { + "tsv" : flat_file_parser, + "csv" : flat_file_parser, + "count" : flat_file_parser, + "json" : simple_json_parser, + "jsonrows" : simple_json_parser, + "list" : lambda: JSONIterator(con, lambda x: self.row(x, self.view).to_l()), + "rr" : lambda: JSONIterator(con, lambda x: self.row(x, self.view)), + "dict" : lambda: JSONIterator(con, lambda x: self.row(x, self.view).to_d()), + "jsonobjects" : lambda: JSONIterator(con, lambda x: ResultObject(x, self.cld, self.view)) + }.get(self.rowformat)() + except Exception, e: + raise Exception("Couldn't get iterator for " + self.rowformat + str(e)) + return reader + + def next(self): + """ + Returns the next row, in the appropriate format + + @rtype: whatever the rowformat was determined to be + """ + if self._it is None: + self._it = iter(self) + try: + return self._it.next() + except StopIteration: + self._it = None + raise StopIteration + +class FlatFileIterator(object): + """ + An iterator for handling results returned as a flat file (TSV/CSV). + =================================================================== + + This iterator can be used as the sub iterator in a ResultIterator + """ + + def __init__(self, connection, parser): + """ + Constructor + =========== + + @param connection: The source of data + @type connection: socket.socket + @param parser: a handler for each row of data + @type parser: Parser + """ + self.connection = connection + self.parser = parser + + def __iter__(self): + return self + + def next(self): + """Return a parsed line of data""" + line = self.connection.next().strip() + if line.startswith("[ERROR]"): + raise WebserviceError(line) + return self.parser(line) + +class JSONIterator(object): + """ + An iterator for handling results returned in the JSONRows format + ================================================================ + + This iterator can be used as the sub iterator in a ResultIterator + """ + + def __init__(self, connection, parser): + """ + Constructor + =========== + + @param connection: The source of data + @type connection: socket.socket + @param parser: a handler for each row of data + @type parser: Parser + """ + self.connection = connection + self.parser = parser + self.header = "" + self.footer = "" + self.parse_header() + self._is_finished = False + + def __iter__(self): + return self + + def next(self): + """Returns a parsed row of data""" + if self._is_finished: + raise StopIteration + return self.get_next_row_from_connection() + + def parse_header(self): + """Reads out the header information from the connection""" + try: + line = self.connection.next().strip() + self.header += line + if not line.endswith('"results":['): + self.parse_header() + except StopIteration: + raise WebserviceError("The connection returned a bad header" + self.header) + + def check_return_status(self): + """ + Perform status checks + ===================== + + The footer containts information as to whether the result + set was successfully transferred in its entirety. This + method makes sure we don't silently accept an + incomplete result set. + + @raise WebserviceError: if the footer indicates there was an error + """ + container = self.header + self.footer + info = None + try: + info = json.loads(container) + except: + raise WebserviceError("Error parsing JSON container: " + container) + + if not info["wasSuccessful"]: + raise WebserviceError(info["statusCode"], info["error"]) + + def get_next_row_from_connection(self): + """ + Reads the connection to get the next row, and sends it to the parser + + @raise WebserviceError: if the connection is interrupted + """ + next_row = None + try: + line = self.connection.next() + if line.startswith("]"): + self.footer += line; + for otherline in self.connection: + self.footer += line + self.check_return_status() + else: + line = line.strip().strip(',') + if len(line) > 0: + try: + row = json.loads(line) + except json.decoder.JSONDecodeError, e: + raise WebserviceError("Error parsing line from results: '" + + line + "' - " + str(e)) + next_row = self.parser(row) + except StopIteration: + raise WebserviceError("Connection interrupted") + + if next_row is None: + self._is_finished = True + raise StopIteration + else: + return next_row + +class InterMineURLOpener(urllib.FancyURLopener): + """ + Specific implementation of urllib.FancyURLOpener for this client + ================================================================ + + Provides user agent and authentication headers, and handling of errors + """ + version = "InterMine-Python-Client-0.96.00" + + def __init__(self, credentials=None, token=None): + """ + Constructor + =========== + + InterMineURLOpener((username, password)) S{->} InterMineURLOpener + + Return a new url-opener with the appropriate credentials + """ + urllib.FancyURLopener.__init__(self) + self.token = token + self.plain_post_header = { + "Content-Type": "text/plain; charset=utf-8", + "UserAgent": USER_AGENT + } + if credentials and len(credentials) == 2: + base64string = base64.encodestring('%s:%s' % credentials)[:-1] + self.addheader("Authorization", base64string) + self.plain_post_header["Authorization"] = base64string + self.using_authentication = True + else: + self.using_authentication = False + + def post_plain_text(self, url, body): + url = self.prepare_url(url) + o = urlparse(url) + con = httplib.HTTPConnection(o.hostname, o.port) + con.request('POST', url, body, self.plain_post_header) + resp = con.getresponse() + content = resp.read() + con.close() + if resp.status != 200: + raise WebserviceError(resp.status, resp.reason, content) + return content + + def open(self, url, data=None): + url = self.prepare_url(url) + return urllib.FancyURLopener.open(self, url, data) + + def prepare_url(self, url): + if self.token: + token_param = "token=" + self.token + o = urlparse(url) + if o.query: + url += "&" + token_param + else: + url += "?" + token_param + + return url + + def delete(self, url): + url = self.prepare_url(url) + o = urlparse(url) + con = httplib.HTTPConnection(o.hostname, o.port) + con.request('DELETE', url, None, self.plain_post_header) + resp = con.getresponse() + content = resp.read() + con.close() + if resp.status != 200: + raise WebserviceError(resp.status, resp.reason, content) + return content + + def http_error_default(self, url, fp, errcode, errmsg, headers): + """Re-implementation of http_error_default, with content now supplied by default""" + content = fp.read() + fp.close() + raise WebserviceError(errcode, errmsg, content) + + def http_error_400(self, url, fp, errcode, errmsg, headers, data=None): + """ + Handle 400 HTTP errors, attempting to return informative error messages + ======================================================================= + + 400 errors indicate that something about our request was incorrect + + @raise WebserviceError: in all circumstances + + """ + content = fp.read() + fp.close() + try: + message = json.loads(content)["error"] + except: + message = content + raise WebserviceError("There was a problem with our request", errcode, errmsg, message) + + def http_error_401(self, url, fp, errcode, errmsg, headers, data=None): + """ + Handle 401 HTTP errors, attempting to return informative error messages + ======================================================================= + + 401 errors indicate we don't have sufficient permission for the resource + we requested - usually a list or a tempate + + @raise WebserviceError: in all circumstances + + """ + content = fp.read() + fp.close() + if self.using_authentication: + raise WebserviceError("Insufficient permissions", errcode, errmsg, content) + else: + raise WebserviceError("No permissions - not logged in", errcode, errmsg, content) + + def http_error_403(self, url, fp, errcode, errmsg, headers, data=None): + """ + Handle 403 HTTP errors, attempting to return informative error messages + ======================================================================= + + 401 errors indicate we don't have sufficient permission for the resource + we requested - usually a list or a tempate + + @raise WebserviceError: in all circumstances + + """ + content = fp.read() + fp.close() + try: + message = json.loads(content)["error"] + except: + message = content + if self.using_authentication: + raise WebserviceError("Insufficient permissions", errcode, errmsg, message) + else: + raise WebserviceError("No permissions - not logged in", errcode, errmsg, message) + + def http_error_404(self, url, fp, errcode, errmsg, headers, data=None): + """ + Handle 404 HTTP errors, attempting to return informative error messages + ======================================================================= + + 404 errors indicate that the requested resource does not exist - usually + a template that is not longer available. + + @raise WebserviceError: in all circumstances + + """ + content = fp.read() + fp.close() + try: + message = json.loads(content)["error"] + except: + message = content + raise WebserviceError("Missing resource", errcode, errmsg, message) + def http_error_500(self, url, fp, errcode, errmsg, headers, data=None): + """ + Handle 500 HTTP errors, attempting to return informative error messages + ======================================================================= + + 500 errors indicate that the server borked during the request - ie: it wasn't + our fault. + + @raise WebserviceError: in all circumstances + + """ + content = fp.read() + fp.close() + try: + message = json.loads(content)["error"] + except: + message = content + raise WebserviceError("Internal server error", errcode, errmsg, message) + diff --git a/intermine/util.py b/intermine/util.py new file mode 100644 index 00000000..5c63b50c --- /dev/null +++ b/intermine/util.py @@ -0,0 +1,26 @@ +def openAnything(source): + # Try to open with urllib (http, ftp, file url) + import urllib + try: + return urllib.urlopen(source) + except (IOError, OSError): + pass + + try: + return open(source) + except (IOError, OSError): + pass + + import StringIO + return StringIO.StringIO(str(source)) + +class ReadableException(Exception): + def __init__(self, message, cause=None): + self.message = message + self.cause = cause + + def __str__(self): + if self.cause is None: + return repr(self.message) + else: + return repr(self.message) + repr(self.cause) diff --git a/intermine/webservice.py b/intermine/webservice.py new file mode 100644 index 00000000..5ff8d9a6 --- /dev/null +++ b/intermine/webservice.py @@ -0,0 +1,488 @@ +from xml.dom import minidom +import urllib +from urlparse import urlparse +import base64 +import UserDict + +#class UJsonLibDecoder(object): # pragma: no cover +# def __init__(self): +# self.loads = ujson.decode +# +# Use core json for 2.6+, simplejson for <=2.5 +#try: +# import ujson +# json = UJsonLibDecoder() +#except ImportError: # pragma: no cover +try: + import simplejson as json # Prefer this as it is faster +except ImportError: # pragma: no cover + try: + import json + except ImportError: + raise ImportError("Could not find any JSON module to import - " + + "please install simplejson or jsonlib to continue") + +# Local intermine imports +from intermine.query import Query, Template +from intermine.model import Model, Attribute, Reference, Collection, Column +from intermine.lists.listmanager import ListManager +from intermine.errors import ServiceError, WebserviceError +from intermine.results import InterMineURLOpener, ResultIterator + +""" +Webservice Interaction Routines for InterMine Webservices +========================================================= + +Classes for dealing with communication with an InterMine +RESTful webservice. + +""" + +__author__ = "Alex Kalderimis" +__organization__ = "InterMine" +__license__ = "LGPL" +__contact__ = "dev@intermine.org" + +class Registry(object, UserDict.DictMixin): + """ + A Class representing an InterMine registry. + =========================================== + + Registries are web-services that mines can automatically register themselves + with, and thus enable service discovery by clients. + + SYNOPSIS + -------- + + example:: + + from intermine.webservice import Registry + + # Connect to the default registry service + # at www.intermine.org/registry + registry = Registry() + + # Find all the available mines: + for name, mine in registry.items(): + print name, mine.version + + # Dict-like interface for accessing mines. + flymine = registry["flymine"] + + # The mine object is a Service + for gene in flymine.select("Gene.*").results(): + process(gene) + + This class is meant to aid with interoperation between + mines by allowing them to discover one-another, and + allow users to always have correct connection information. + """ + + MINES_PATH = "/mines.json" + + def __init__(self, registry_url="http://www.intermine.org/registry"): + self.registry_url = registry_url + opener = InterMineURLOpener() + data = opener.open(registry_url + Registry.MINES_PATH).read() + mine_data = json.loads(data) + self.__mine_dict = dict(( (mine["name"], mine) for mine in mine_data["mines"])) + self.__synonyms = dict(( (name.lower(), name) for name in self.__mine_dict.keys() )) + self.__mine_cache = {} + + def __contains__(self, name): + return name.lower() in self.__synonyms + + def __getitem__(self, name): + lc = name.lower() + if lc in self.__synonyms: + if lc not in self.__mine_cache: + self.__mine_cache[lc] = Service(self.__mine_dict[self.__synonyms[lc]]["webServiceRoot"]) + return self.__mine_cache[lc] + else: + raise KeyError("Unknown mine: " + name) + + def __setitem__(self, name, item): + raise NotImplementedError("You cannot add items to a registry") + + def __delitem__(self, name): + raise NotImplementedError("You cannot remove items from a registry") + + def keys(self): + return self.__mine_dict.keys() + +class Service(object): + """ + A class representing connections to different InterMine WebServices + =================================================================== + + The intermine.webservice.Service class is the main interface for the user. + It will provide access to queries and templates, as well as doing the + background task of fetching the data model, and actually requesting + the query results. + + SYNOPSIS + -------- + + example:: + + from intermine.webservice import Service + service = Service("http://www.flymine.org/query/service") + + template = service.get_template("Gene_Pathways") + for row in template.results(A={"value":"zen"}): + do_something_with(row) + ... + + query = service.new_query() + query.add_view("Gene.symbol", "Gene.pathway.name") + query.add_constraint("Gene", "LOOKUP", "zen") + for row in query.results(): + do_something_with(row) + ... + + new_list = service.create_list("some/file/with.ids", "Gene") + list_on_server = service.get_list("On server") + in_both = new_list & list_on_server + in_both.name = "Intersection of these lists" + for row in in_both: + do_something_with(row) + ... + + OVERVIEW + -------- + The two methods the user will be most concerned with are: + - L{Service.new_query}: constructs a new query to query a service with + - L{Service.get_template}: gets a template from the service + - L{ListManager.create_list}: creates a new list on the service + + For list management information, see L{ListManager}. + + TERMINOLOGY + ----------- + X{Query} is the term for an arbitrarily complex structured request for + data from the webservice. The user is responsible for specifying the + structure that determines what records are returned, and what information + about each record is provided. + + X{Template} is the term for a predefined "Query", ie: one that has been + written and saved on the webservice you will access. The definition + of the query is already done, but the user may want to specify the + values of the constraints that exist on the template. Templates are accessed + by name, and while you can easily introspect templates, it is assumed + you know what they do when you use them + + X{List} is a saved result set containing a set of objects previously identified + in the database. Lists can be created and managed using this client library. + + @see: L{intermine.query} + """ + QUERY_PATH = '/query/results' + LIST_ENRICHMENT_PATH = '/list/enrichment' + QUERY_LIST_UPLOAD_PATH = '/query/tolist/json' + QUERY_LIST_APPEND_PATH = '/query/append/tolist/json' + MODEL_PATH = '/model' + TEMPLATES_PATH = '/templates/xml' + TEMPLATEQUERY_PATH = '/template/results' + LIST_PATH = '/lists/json' + LIST_CREATION_PATH = '/lists/json' + LIST_RENAME_PATH = '/lists/rename/json' + LIST_APPENDING_PATH = '/lists/append/json' + LIST_TAG_PATH = '/list/tags/json' + SAVEDQUERY_PATH = '/savedqueries/xml' + VERSION_PATH = '/version/ws' + RELEASE_PATH = '/version/release' + SCHEME = 'http://' + SERVICE_RESOLUTION_PATH = "/check/" + + def __init__(self, root, + username=None, password=None, token=None, + prefetch_depth=1, prefetch_id_only=False): + """ + Constructor + =========== + + Construct a connection to a webservice:: + + url = "http://www.flymine.org/query/service" + + # An unauthenticated connection - access to all public data + service = Service(url) + + # An authenticated connection - access to private and public data + service = Service(url, token="ABC123456") + + + @param root: the root url of the webservice (required) + @param username: your login name (optional) + @param password: your password (required if a username is given) + @param token: your API access token(optional - used in preference to username and password) + + @raise ServiceError: if the version cannot be fetched and parsed + @raise ValueError: if a username is supplied, but no password + + There are two alternative authentication systems supported by InterMine + webservices. The first is username and password authentication, which + is supported by all webservices. Newer webservices (version 6+) + also support API access token authentication, which is the recommended + system to use. Token access is more secure as you will never have + to transmit your username or password, and the token can be easily changed + or disabled without changing your webapp login details. + + """ + o = urlparse(root) + if not o.scheme: root = "http://" + root + if not root.endswith("/service"): root = root + "/service" + + self.root = root + self.prefetch_depth = prefetch_depth + self.prefetch_id_only = prefetch_id_only + self._templates = None + self._model = None + self._version = None + self._release = None + self._list_manager = ListManager(self) + self.__missing_method_name = None + if token: + self.opener = InterMineURLOpener(token=token) + elif username: + if token: + raise ValueError("Both username and token credentials supplied") + + if not password: + raise ValueError("Username given, but no password supplied") + + self.opener = InterMineURLOpener((username, password)) + else: + self.opener = InterMineURLOpener() + + try: + self.version + except WebserviceError, e: + raise ServiceError("Could not validate service - is the root url (%s) correct? %s" % (root, e)) + + if token and self.version < 6: + raise ServiceError("This service does not support API access token authentication") + + # Set up sugary aliases + self.query = self.new_query + + + # Delegated list methods + + LIST_MANAGER_METHODS = frozenset(["get_list", "get_all_lists", + "get_all_list_names", + "create_list", "get_list_count", "delete_lists", "l"]) + + def __getattribute__(self, name): + return object.__getattribute__(self, name) + + def __getattr__(self, name): + if name in self.LIST_MANAGER_METHODS: + method = getattr(self._list_manager, name) + return method + raise AttributeError("Could not find " + name) + + def __del__(self): + try: + self._list_manager.delete_temporary_lists() + except ReferenceError: + pass + + @property + def version(self): + """ + Returns the webservice version + ============================== + + The version specifies what capabilities a + specific webservice provides. The most current + version is 3 + + may raise ServiceError: if the version cannot be fetched + + @rtype: int + """ + if self._version is None: + try: + url = self.root + self.VERSION_PATH + self._version = int(self.opener.open(url).read()) + except ValueError, e: + raise ServiceError("Could not parse a valid webservice version: " + str(e)) + return self._version + + def resolve_service_path(self, variant): + """Resolve the path to optional services""" + url = self.root + self.SERVICE_RESOLUTION_PATH + variant + return self.opener.open(url).read() + + @property + def release(self): + """ + Returns the datawarehouse release + ================================= + + Service.release S{->} string + + The release is an arbitrary string used to distinguish + releases of the datawarehouse. This usually coincides + with updates to the data contained within. While a string, + releases usually sort in ascending order of recentness + (eg: "release-26", "release-27", "release-28"). They can also + have less machine readable meanings (eg: "beta") + + @rtype: string + """ + if self._release is None: + self._release = urllib.urlopen(self.root + self.RELEASE_PATH).read() + return self._release + + def load_query(self, xml, root=None): + """ + Construct a new Query object for the given webservice + ===================================================== + + This is the standard method for instantiating new Query + objects. Queries require access to the data model, as well + as the service itself, so it is easiest to access them through + this factory method. + + @return: L{intermine.query.Query} + """ + return Query.from_xml(xml, self.model, root=root) + + def select(self, *columns, **kwargs): + """ + Construct a new Query object with the given columns selected. + ============================================================= + + As new_query, except that instead of a root class, a list of + output column expressions are passed instead. + """ + if "xml" in kwargs: + return self.load_query(kwargs["xml"]) + if len(columns) == 1: + view = columns[0] + if isinstance(view, Attribute): + return Query(self.model, self).select("%s.%s" % (view.declared_in.name, view)) + if isinstance(view, Reference): + return Query(self.model, self).select("%s.%s.*" % (view.declared_in.name, view)) + elif not isinstance(view, Column) and not str(view).endswith("*"): + path = self.model.make_path(view) + if not path.is_attribute(): + return Query(self.model, self).select(str(view) + ".*") + return Query(self.model, self).select(*columns) + + new_query = select + + def get_template(self, name): + """ + Returns a template of the given name + ==================================== + + Tries to retrieve a template of the given name + from the webservice. If you are trying to fetch + a private template (ie. one you made yourself + and is not available to others) then you may need to authenticate + + @see: L{intermine.webservice.Service.__init__} + + @param name: the template's name + @type name: string + + @raise ServiceError: if the template does not exist + @raise QueryParseError: if the template cannot be parsed + + @return: L{intermine.query.Template} + """ + try: + t = self.templates[name] + except KeyError: + raise ServiceError("There is no template called '" + + name + "' at this service") + if not isinstance(t, Template): + t = Template.from_xml(t, self.model, self) + self.templates[name] = t + return t + + @property + def templates(self): + """ + The dictionary of templates from the webservice + =============================================== + + Service.templates S{->} dict(intermine.query.Template|string) + + For efficiency's sake, Templates are not parsed until + they are required, and until then they are stored as XML + strings. It is recommended that in most cases you would want + to use L{Service.get_template}. + + You can use this property however to test for template existence though:: + + if name in service.templates: + template = service.get_template(name) + + @rtype: dict + + """ + if self._templates is None: + sock = self.opener.open(self.root + self.TEMPLATES_PATH) + dom = minidom.parse(sock) + sock.close() + templates = {} + for e in dom.getElementsByTagName('template'): + name = e.getAttribute('name') + if name in templates: + raise ServiceError("Two templates with same name: " + name) + else: + templates[name] = e.toxml() + self._templates = templates + return self._templates + + @property + def model(self): + """ + The data model for the webservice you are querying + ================================================== + + Service.model S{->} L{intermine.model.Model} + + This is used when constructing queries to provide them + with information on the structure of the data model + they are accessing. You are very unlikely to want to + access this object directly. + + raises ModelParseError: if the model cannot be read + + @rtype: L{intermine.model.Model} + + """ + if self._model is None: + model_url = self.root + self.MODEL_PATH + self._model = Model(model_url, self) + return self._model + + def get_results(self, path, params, rowformat, view, cld=None): + """ + Return an Iterator over the rows of the results + =============================================== + + This method is called internally by the query objects + when they are called to get results. You will not + normally need to call it directly + + @param path: The resource path (eg: "/query/results") + @type path: string + @param params: The query parameters for this request as a dictionary + @type params: dict + @param rowformat: One of "rr", "object", "count", "dict", "list", "tsv", "csv", "jsonrows", "jsonobjects" + @type rowformat: string + @param view: The output columns + @type view: list + + @raise WebserviceError: for failed requests + + @return: L{intermine.webservice.ResultIterator} + """ + return ResultIterator(self, path, params, rowformat, view, cld) + diff --git a/samples/alleles.py b/samples/alleles.py new file mode 100644 index 00000000..b78f9bf6 --- /dev/null +++ b/samples/alleles.py @@ -0,0 +1,54 @@ +from intermine.webservice import Service +import itertools + +def lines_of(x): + return chunker(0, x) + +def chunker(i, x): + d = {"accum": i} + def func(y): + i = d["accum"] + grp = i / x + i += 1 + d["accum"] = i + return grp + return func + +col_width = 15 +cols = 8 +sep = '| ' +ellipsis = '...' +line_width = col_width * cols + (cols - 1) * len(sep) +fit_to_cell = lambda a: a.ljust(col_width) if len(a) <= col_width else a[:col_width - len(ellipsis)] + ellipsis +hrule = "-" * line_width +summary = "\n%s: %d Alleles" + +s = Service("www.flymine.org/query") + +Gene = s.model.Gene + +q = s.query(Gene).\ + add_columns("name", "symbol", "alleles.*").\ + filter(Gene.symbol == ["zen", "eve", "bib", "h"]).\ + filter(Gene.alleles.symbol == "*hs*").\ + outerjoin(Gene.alleles).\ + order_by("symbol") + +for row in q.rows(): + print row + +for gene in s.query(s.model.Gene).filter(s.model.Gene.symbol == ["zen", "eve", "bib", "h"]).add_columns(s.model.Gene.alleles): + + print summary % (gene.symbol, len(gene.alleles)) + print hrule + + for k, line_of_alleles in itertools.groupby(sorted(map(lambda a: a.symbol, gene.alleles)), lines_of(cols)): + print sep.join(map(fit_to_cell, line_of_alleles)) + + print "\nAllele Classes:" + allele_classes = [(key, len(list(group))) for key, group in itertools.groupby(sorted(map(lambda x: x.alleleClass, gene.alleles)))] + for pair in reversed(sorted(allele_classes, key=lambda g: g[1])): + print "%s (%d)" % pair + + print hrule + diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..c33aa706 --- /dev/null +++ b/setup.py @@ -0,0 +1,164 @@ +""" +The test and clean code is shamelessly stolen from +http://da44en.wordpress.com/2002/11/22/using-distutils/ +""" + +import os +import sys +import time +from distutils.core import Command, setup +from distutils import log +from distutils.fancy_getopt import fancy_getopt +from unittest import TextTestRunner, TestLoader +from glob import glob +from os.path import splitext, basename, join as pjoin +from warnings import warn + + +class TestCommand(Command): + user_options = [('verbose', 'v', "produce verbose output", 1)] + + def initialize_options(self): + self._dir = os.getcwd() + self.test_prefix = 'test' + + def finalize_options(self): + args, obj = fancy_getopt(self.user_options, {}, None, None) + # Ugly as sin, but distutils forced me to do it :( + # All I wanted was this command to default to quiet... + if "--verbose" not in args and "-v" not in args: + self.verbose = 0 + + def run(self): + ''' + Finds all the tests modules in tests/, and runs them, exiting after they are all done + ''' + from tests.testserver import TestServer + from tests.test import WebserviceTest + + log.set_verbosity(self.verbose) + + server = TestServer() + server.start() + WebserviceTest.TEST_PORT = server.port + + self.announce("Waiting for test server to start on port " + str(server.port), level=2) + time.sleep(1) + + testfiles = [ ] + for t in glob(pjoin(self._dir, 'tests', self.test_prefix + '*.py')): + if not t.endswith('__init__.py'): + testfiles.append('.'.join( + ['tests', splitext(basename(t))[0]]) + ) + + self.announce("Test files:" + str(testfiles), level=2) + tests = TestLoader().loadTestsFromNames(testfiles) + t = TextTestRunner(verbosity = self.verbose) + t.run(tests) + exit() + +class LiveTestCommand(TestCommand): + + def initialize_options(self): + TestCommand.initialize_options(self) + self.test_prefix = 'live' + +class CleanCommand(Command): + """ + Remove all build files and all compiled files + ============================================= + + Remove everything from build, including that + directory, and all .pyc files + """ + user_options = [('verbose', 'v', "produce verbose output", 1)] + + def initialize_options(self): + self._files_to_delete = [ ] + self._dirs_to_delete = [ ] + + for root, dirs, files in os.walk('.'): + for f in files: + if f.endswith('.pyc'): + self._files_to_delete.append(pjoin(root, f)) + for root, dirs, files in os.walk(pjoin('build')): + for f in files: + self._files_to_delete.append(pjoin(root, f)) + for d in dirs: + self._dirs_to_delete.append(pjoin(root, d)) + # reverse dir list to only get empty dirs + self._dirs_to_delete.reverse() + self._dirs_to_delete.append('build') + + def finalize_options(self): + args, obj = fancy_getopt(self.user_options, {}, None, None) + # Ugly as sin, but distutils forced me to do it :( + # All I wanted was this command to default to quiet... + if "--verbose" not in args and "-v" not in args: + self.verbose = 0 + + def run(self): + for clean_me in self._files_to_delete: + if self.dry_run: + log.info("Would have unlinked " + clean_me) + else: + try: + self.announce("Deleting " + clean_me, level=2) + os.unlink(clean_me) + except: + message = " ".join(["Failed to delete file", clean_me]) + log.warn(message) + for clean_me in self._dirs_to_delete: + if self.dry_run: + log.info("Would have rmdir'ed " + clean_me) + else: + if os.path.exists(clean_me): + try: + self.announce("Going to remove " + clean_me, level=2) + os.rmdir(clean_me) + except: + message = " ".join(["Failed to delete dir", clean_me]) + log.warn(message) + elif clean_me != "build": + log.warn(clean_me + " does not exist") + +extra = {} +if sys.version_info >= (3,): + extra['use_2to3'] = True + +setup( + name = "intermine", + packages = ["intermine", "intermine.lists"], + provides = ["intermine"], + cmdclass = { 'clean': CleanCommand, 'test': TestCommand, 'livetest': LiveTestCommand }, + version = "1.00.01", + description = "InterMine WebService client", + author = "Alex Kalderimis", + author_email = "dev@intermine.org", + url = "http://www.intermine.org", + download_url = "http://www.intermine.org/lib/python-webservice-client-0.98.02.tar.gz", + keywords = ["webservice", "genomic", "bioinformatics"], + classifiers = [ + "Programming Language :: Python", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Scientific/Engineering :: Bio-Informatics", + "Topic :: Scientific/Engineering :: Information Analysis", + "Operating System :: OS Independent", + ], + license = "LGPL", + long_description = """\ +InterMine Webservice Client +---------------------------- + +Provides access routines to datawarehouses powered +by InterMine technology. + +""", + **extra +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/acceptance.py b/tests/acceptance.py new file mode 100644 index 00000000..d85123de --- /dev/null +++ b/tests/acceptance.py @@ -0,0 +1,17 @@ +from intermine.query import Query +from intermine.model import Model + +m = Model('http://www.flymine.org/query/service/model') +q = Query(m) + +q.name = 'Foo' +q.description = 'a query made out of pythons' +q.add_view("Gene.name Gene.symbol") +q.add_constraint('Gene', 'LOOKUP', 'eve') +q.add_constraint('Gene.length', '>', 50000) +q.add_constraint('Gene', 'CDNAClone') +q.add_constraint('Gene.symbol', 'ONE OF', ['eve', 'zen']) +q.add_join('Gene.alleles') +q.add_path_description('Gene', 'One of those gene-y things') +print q.to_xml() +print q.to_formatted_xml() diff --git a/tests/data/test-identifiers.list b/tests/data/test-identifiers.list new file mode 100644 index 00000000..d94e32df --- /dev/null +++ b/tests/data/test-identifiers.list @@ -0,0 +1,6 @@ +Karim +"Not a good id" +"David Brent" +Jean-Marc +"Jennifer Schirrmann" +"Frank Möllers" diff --git a/tests/live_lists.py b/tests/live_lists.py new file mode 100644 index 00000000..bcc67cd1 --- /dev/null +++ b/tests/live_lists.py @@ -0,0 +1,373 @@ +import os +import sys +sys.path.insert(0, os.getcwd()) + +import unittest +from intermine.webservice import Service + +def emp_rows_without_ids(bag): + return [row[:3] + row[4:] for row in bag.to_query().rows()] + + +# This is coded all as one enormous test so that we can do +# a universal clean-up at the end. +class LiveListTest(unittest.TestCase): + + TEST_ROOT = "http://localhost/intermine-test/service" + TEST_USER = "intermine-test-user" + TEST_PASS = "intermine-test-user-password" + + # Expected rows + KARIM = [37, '4', False, 'Karim'] + JENNIFER_SCHIRRMANN = [55, '9', False, 'Jennifer Schirrmann'] + JENNIFER = [45, '8', True, 'Jennifer'] + JEAN_MARC = [53, '0', True, 'Jean-Marc'] + VINCENT = [29, '3', True, 'Vincent'] + INA = [39, '8', True, 'Ina'] + ALEX = [43, '0', True, 'Alex'] + DELPHINE = [47, '9', False, 'Delphine'] + BRENDA = [54, '2', False, 'Brenda'] + KEITH = [56, None, False, 'Keith Bishop'] + CAROL = [62, '3', True, 'Carol'] + GARETH = [61, '8', True, 'Gareth Keenan'] + DAVID = [41, None, False, 'David Brent'] + FRANK = [44, None, False, u'Frank M\xf6llers'] + JULIETTE = [71, None, False, 'Juliette Lebrac'] + BWAH_HA = [74, None, False, "Bwa'h Ha Ha"] + + SERVICE = Service(TEST_ROOT, TEST_USER, TEST_PASS) + + LADIES_NAMES = ["Brenda", "Zop", "Carol", "Quux", "Jennifer", "Delphine", "Ina"] + GUYS_NAMES = 'Alex Karim "Gareth Keenan" Foo Bar "Keith Bishop" Vincent Baz' + EMPLOYEE_FILE = "tests/data/test-identifiers.list" + TYPE = 'Employee' + + maxDiff = None + + def __init__(self, name): + unittest.TestCase.__init__(self, name) + self.initialListCount = self.SERVICE.get_list_count() + + # Disabled due to bug in FlyMine 34.0. + # def testListsFromFlyMine(self): + # s = Service("www.flymine.org/query") + # all_lists = s.get_all_lists() + # possible_statuses = set(["CURRENT", "TO_UPGRADE", "NOT_CURRENT"]) + # got = set((l.status for l in all_lists)) + # self.assertTrue(got <= possible_statuses) + + def testListTagAdding(self): + s = self.SERVICE + t = self.TYPE; + l = s.create_list(self.GUYS_NAMES, t, description="Id string") + self.assertEqual(set(), l.tags) + l.add_tags("a-tag", "b-tag") + self.assertEqual(set(["a-tag", "b-tag"]), l.tags) + + def testListTagRemoval(self): + s = self.SERVICE + t = self.TYPE; + tags = ["a-tag", "b-tag", "c-tag"] + l = s.create_list(self.GUYS_NAMES, t, description="Id string", tags = tags) + self.assertEqual(set(tags), l.tags) + l.remove_tags("a-tag", "c-tag") + self.assertEqual(set(["b-tag"]), l.tags) + l.remove_tags("b-tag", "d-tag") + self.assertEqual(set(), l.tags) + + def testListTagUpdating(self): + s = self.SERVICE + t = self.TYPE; + l = s.create_list(self.GUYS_NAMES, t, description="Id string") + self.assertEqual(set(), l.tags) + self.assertEqual(set(["a-tag", "b-tag"]), set(map(str, s._list_manager.add_tags(l, ["a-tag", "b-tag"])))) + self.assertEqual(set(), l.tags) + l.update_tags() + self.assertEqual(set(["a-tag", "b-tag"]), set(map(str, l.tags))) + + def testLists(self): + t = self.TYPE; + s = self.SERVICE + self.assertTrue(s.get_list_count() > 0) + + l = s.create_list(self.LADIES_NAMES, t, description="Id list") + self.assertEqual(l.unmatched_identifiers, set(["Zop", "Quux"])) + self.assertEqual(l.size, 5) + self.assertEqual(l.list_type, t) + + l = s.get_list(l.name) + self.assertEqual(l.size, 5) + self.assertEqual(l.list_type, t) + + l = s.create_list(self.GUYS_NAMES, t, description="Id string") + self.assertEqual(l.unmatched_identifiers, set(["Foo", "Bar", "Baz"])) + self.assertEqual(l.size, 5) + self.assertEqual(l.list_type, "Employee") + + l = s.create_list(self.EMPLOYEE_FILE, t, description="Id file", tags=["Foo", "Bar"]) + self.assertEqual(l.unmatched_identifiers, set(["Not a good id"])) + self.assertEqual(l.size, 5) + self.assertEqual(l.list_type, "Employee") + self.assertEqual(l.tags, set(["Foo", "Bar"])) + + q = s.new_query() + q.add_view("Employee.id") + q.add_constraint("Employee.department.name", '=', "Sales") + l = s.create_list(q, description="Id query") + self.assertEqual(l.unmatched_identifiers, set()) + self.assertEqual(l.size, 18) + self.assertEqual(l.list_type, t) + + l.name = "renamed query" + + l2 = s.get_list("renamed query") + self.assertEqual(str(l), str(l2)) + + l3 = s.create_list(l) + self.assertEqual(l3.size, l2.size) + + l.delete() + self.assertTrue(s.get_list("renamed query") is None) + + l = s.create_list(self.EMPLOYEE_FILE, t) + expected = [ + LiveListTest.KARIM, LiveListTest.DAVID, LiveListTest.FRANK, + LiveListTest.JEAN_MARC, LiveListTest.JENNIFER_SCHIRRMANN + ] + + got = [row[:3] + row[4:] for row in l.to_query().rows()] + self.assertEqual(got, expected) + + # Test iteration: + got = set([x.age for x in l]) + expected_ages = set([37, 41, 44, 53, 55]) + self.assertEqual(expected_ages, got) + + self.assertTrue(l[0].age in expected_ages) + self.assertTrue(l[-1].age in expected_ages) + self.assertTrue(l[2].age in expected_ages) + self.assertRaises(IndexError, lambda: l[5]) + self.assertRaises(IndexError, lambda: l[-6]) + self.assertRaises(IndexError, lambda: l["foo"]) + + # Test intersections + + listA = s.create_list(self.GUYS_NAMES, t) + listB = s.create_list(self.EMPLOYEE_FILE, t) + + intersection = listA & listB + self.assertEqual(intersection.size, 1) + expected = [LiveListTest.KARIM] + self.assertEqual(emp_rows_without_ids(intersection), expected) + + q = s.new_query("Employee").where("age", ">", 50) + intersection = listB & q + self.assertEqual(intersection.size, 2) + expected = [LiveListTest.JEAN_MARC, LiveListTest.JENNIFER_SCHIRRMANN] + self.assertEqual(emp_rows_without_ids(intersection), expected) + + prev_name = listA.name + prev_desc = listA.description + listA &= listB + self.assertEqual(listA.size, 1) + got = emp_rows_without_ids(listA) + expected = [LiveListTest.KARIM] + self.assertEqual(got, expected) + self.assertEqual(prev_name, listA.name) + self.assertEqual(prev_desc, listA.description) + + # Test unions + listA = s.create_list(self.GUYS_NAMES, t, tags=["tagA", "tagB"]) + listB = s.create_list(self.LADIES_NAMES, t) + + union = listA | listB + self.assertEqual(union.size, 10) + expected = [ + LiveListTest.VINCENT, LiveListTest.KARIM, LiveListTest.INA, + LiveListTest.ALEX, LiveListTest.JENNIFER, LiveListTest.DELPHINE, + LiveListTest.BRENDA, LiveListTest.KEITH, LiveListTest.GARETH, + LiveListTest.CAROL + ] + got = [row[:3] + row[4:] for row in union.to_query().rows()] + self.assertEqual(got, expected) + + union = listA + listB + self.assertEqual(union.size, 10) + self.assertEqual(emp_rows_without_ids(union), expected) + + # Test appending + + prev_name = listA.name + prev_desc = listA.description + listA += listB + self.assertEqual(listA.size, 10) + self.assertEqual(listA.tags, set(["tagA", "tagB"])) + fromService = s.get_list(listA.name) + self.assertEqual(listA.tags, fromService.tags) + self.assertEqual(emp_rows_without_ids(listA), expected) + self.assertEqual(prev_name, listA.name) + self.assertEqual(prev_desc, listA.description) + + listA = s.create_list(self.GUYS_NAMES, t, description="testing appending") + prev_name = listA.name + prev_desc = listA.description + listA += self.LADIES_NAMES + self.assertEqual(listA.size, 10) + self.assertEqual(emp_rows_without_ids(listA), expected) + self.assertEqual(prev_name, listA.name) + self.assertEqual(prev_desc, listA.description) + self.assertEqual(len(listA.unmatched_identifiers), 5) + + listA = s.create_list(self.GUYS_NAMES, t, description="testing appending") + prev_name = listA.name + prev_desc = listA.description + listA += self.EMPLOYEE_FILE + self.assertEqual(listA.size, 9) + expected = [ + LiveListTest.VINCENT, + LiveListTest.KARIM, + LiveListTest.DAVID, + LiveListTest.ALEX, + LiveListTest.FRANK, + LiveListTest.JEAN_MARC, + LiveListTest.JENNIFER_SCHIRRMANN, + LiveListTest.KEITH, + LiveListTest.GARETH + ] + self.assertEqual(emp_rows_without_ids(listA), expected) + self.assertEqual(prev_name, listA.name) + self.assertEqual(prev_desc, listA.description) + + listA = s.create_list(self.GUYS_NAMES, t) + listB = s.create_list(self.EMPLOYEE_FILE, t) + listC = s.create_list(self.LADIES_NAMES, t) + + prev_name = listA.name + prev_desc = listA.description + listA += [listA, listB, listC] + self.assertEqual(listA.size, 14) + expected = [ + LiveListTest.VINCENT, LiveListTest.KARIM, LiveListTest.INA, + LiveListTest.DAVID, LiveListTest.ALEX, + LiveListTest.FRANK, LiveListTest.JENNIFER, LiveListTest.DELPHINE, + LiveListTest.JEAN_MARC, LiveListTest.BRENDA, LiveListTest.JENNIFER_SCHIRRMANN, + LiveListTest.KEITH, LiveListTest.GARETH, LiveListTest.CAROL + ] + self.assertEqual(emp_rows_without_ids(listA), expected) + self.assertEqual(prev_name, listA.name) + self.assertEqual(prev_desc, listA.description) + + listA = s.create_list(self.GUYS_NAMES, t) + q = s.new_query() + q.add_view("Employee.id") + q.add_constraint("Employee.age", '>', 65) + + prev_name = listA.name + prev_desc = listA.description + listA += [listA, listB, listC, q] + self.assertEqual(listA.size, 16) + expected = [ + LiveListTest.VINCENT, LiveListTest.KARIM, LiveListTest.INA, + LiveListTest.DAVID, LiveListTest.ALEX, + LiveListTest.FRANK, LiveListTest.JENNIFER, LiveListTest.DELPHINE, + LiveListTest.JEAN_MARC, LiveListTest.BRENDA, LiveListTest.JENNIFER_SCHIRRMANN, + LiveListTest.KEITH, LiveListTest.GARETH, LiveListTest.CAROL, + LiveListTest.JULIETTE, LiveListTest.BWAH_HA + ] + self.assertEqual(emp_rows_without_ids(listA), expected) + self.assertEqual(prev_name, listA.name) + self.assertEqual(prev_desc, listA.description) + + # Test diffing + listA = s.create_list(self.GUYS_NAMES, t) + listB = s.create_list(self.EMPLOYEE_FILE, t) + + diff = listA ^ listB + self.assertEqual(diff.size, 8) + expected = [ + LiveListTest.VINCENT, + LiveListTest.DAVID, LiveListTest.ALEX, + LiveListTest.FRANK, + LiveListTest.JEAN_MARC, LiveListTest.JENNIFER_SCHIRRMANN, + LiveListTest.KEITH, LiveListTest.GARETH + ] + self.assertEqual(emp_rows_without_ids(diff), expected) + + prev_name = listA.name + prev_desc = listA.description + listA ^= listB + self.assertEqual(listA.size, 8) + self.assertEqual(emp_rows_without_ids(listA), expected) + self.assertEqual(prev_name, listA.name) + self.assertEqual(prev_desc, listA.description) + + # Test subtraction + listA = s.create_list(self.GUYS_NAMES, t, tags=["subtr-a", "subtr-b"]) + listB = s.create_list(self.EMPLOYEE_FILE, t) + + subtr = listA - listB + self.assertEqual(subtr.size, 4) + expected = [ + LiveListTest.VINCENT, LiveListTest.ALEX, LiveListTest.KEITH,LiveListTest.GARETH + ] + got = [row[:3] + row[4:] for row in subtr.to_query().rows()] + self.assertEqual(got, expected) + + prev_name = listA.name + prev_desc = listA.description + listA -= listB + self.assertEqual(listA.size, 4) + self.assertEqual(listA.tags, set(["subtr-a", "subtr-b"])) + self.assertEqual(emp_rows_without_ids(listA), expected) + self.assertEqual(prev_name, listA.name) + self.assertEqual(prev_desc, listA.description) + + # Test subqueries + with_cc_q = s.model.Bank.where("corporateCustomers.id", "IS NOT NULL") + with_cc_l = s.create_list(with_cc_q) + + self.assertEqual(2, s.model.Bank.where(s.model.Bank ^ with_cc_q).count()) + self.assertEqual(2, s.model.Bank.where(s.model.Bank ^ with_cc_l).count()) + + self.assertEqual(3, s.model.Bank.where(s.model.Bank < with_cc_q).count()) + self.assertEqual(3, s.model.Bank.where(s.model.Bank < with_cc_l).count()) + + boring_q = s.new_query("Bank") + boring_q.add_constraint("Bank", "NOT IN", with_cc_q) + self.assertEqual(2, boring_q.count()) + + boring_q = s.new_query("Bank") + boring_q.add_constraint("Bank", "NOT IN", with_cc_l) + self.assertEqual(2, boring_q.count()) + + # Test query overloading + + no_comps = s.new_query("Bank") - with_cc_q + self.assertEqual(2, no_comps.size) + + no_comps = s.new_query("Bank") - with_cc_l + self.assertEqual(2, no_comps.size) + + all_b = s.new_query("Bank") | with_cc_q + self.assertEqual(5, all_b.size) + + all_b = s.new_query("Bank") | with_cc_l + self.assertEqual(5, all_b.size) + + # Test enrichment + + favs = s.l("My-Favourite-Employees") + enriched_contractors = map(lambda x: x.identifier, favs.calculate_enrichment('contractor_enrichment', maxp = 1.0)) + self.assertEqual(enriched_contractors, ['Vikram']) + + def tearDown(self): + s = self.SERVICE + s.__del__() + self.assertEqual(self.SERVICE.get_list_count(), self.initialListCount) + +class LiveListTestWithTokens(LiveListTest): + SERVICE = Service(LiveListTest.TEST_ROOT, token="test-user-token") + +if __name__ == '__main__': + unittest.main() + diff --git a/tests/live_registry.py b/tests/live_registry.py new file mode 100644 index 00000000..17e90614 --- /dev/null +++ b/tests/live_registry.py @@ -0,0 +1,20 @@ +import sys +import os +sys.path.insert(0, os.getcwd()) + +import unittest + +from intermine.webservice import Registry + +class LiveRegistryTest(unittest.TestCase): + + def testAccessRegistry(self): + pass + # Registry is deprecated for the time-being. + #r = Registry() + #self.assertTrue("flymine" in r) + #self.assertTrue(r["flymine"].version > 5) + +if __name__ == '__main__': + unittest.main() + diff --git a/tests/live_results.py b/tests/live_results.py new file mode 100644 index 00000000..e6c7df93 --- /dev/null +++ b/tests/live_results.py @@ -0,0 +1,101 @@ +import sys +import os +sys.path.insert(0, os.getcwd()) + +import unittest +from intermine.webservice import Service + +class LiveListTest(unittest.TestCase): + + TEST_ROOT = "http://localhost/intermine-test/service" + + SERVICE = Service(TEST_ROOT) + + def testLazyReferenceFetching(self): + results = self.SERVICE.select("Department.*").results() + managers = map(lambda x: x.manager.name, results) + expected = [ + 'EmployeeA1', + 'EmployeeB1', + 'EmployeeB3', + 'Jennifer Taylor-Clarke', + 'David Brent', + 'Keith Bishop', + 'Glynn Williams', + 'Neil Godwin', + 'Tatjana Berkel', + u'Sinan Tur\xe7ulu', + 'Bernd Stromberg', + 'Timo Becker', + 'Dr. Stefan Heinemann', + 'Burkhardt Wutke', + u'Frank M\xf6llers', + 'Charles Miner', + 'Michael Scott', + 'Angela', + 'Lonnis Collins', + 'Meredith Palmer', + 'Juliette Lebrac', + 'Gilles Triquet', + 'Jacques Plagnol Jacques', + u'Didier Legu\xe9lec', + 'Joel Liotard', + "Bwa'h Ha Ha", + 'Quote Leader', + 'Separator Leader', + 'Slash Leader', + 'XML Leader'] + + self.assertEqual(expected, managers) + + def testLazyReferenceFetching(self): + dave = self.SERVICE.select("Employee.*").where(name = "David Brent").one() + self.assertEqual("Sales", dave.department.name) + self.assertIsNotNone(dave.address) + + # Can handle null references. + b1 = self.SERVICE.select("Employee.*").where(name = "EmployeeB1").one(); + self.assertIsNone(b1.address) + + def testLazyCollectionFetching(self): + results = self.SERVICE.select("Department.*").results() + age_sum = reduce(lambda x, y: x + reduce(lambda a, b: a + b.age, y.employees, 0), results, 0) + self.assertEqual(5924, age_sum) + + # Can handle empty collections as well as populated ones. + banks = self.SERVICE.select("Bank.*").results() + self.assertEqual([1, 0, 0, 2, 2], [len(bank.corporateCustomers) for bank in banks]) + + def testAllFormats(self): + q = self.SERVICE.select("Manager.age") + + expected_sum = 1383 + + self.assertEqual(expected_sum, sum(map(lambda x: x.age, q.results(row="object")))) + self.assertEqual(expected_sum, sum(map(lambda x: x.age, q.results(row="objects")))) + self.assertEqual(expected_sum, sum(map(lambda x: x.age, q.results(row="jsonobjects")))) + + self.assertEqual(expected_sum, sum(map(lambda x: x["age"], q.results(row="rr")))) + self.assertEqual(expected_sum, sum(map(lambda x: x[0], q.results(row="rr")))) + + self.assertEqual(expected_sum, sum(map(lambda x: x["Manager.age"], q.results(row="dict")))) + self.assertEqual(expected_sum, sum(map(lambda x: x[0], q.results(row="list")))) + + self.assertEqual(expected_sum, sum(map(lambda x: x[0]["value"], q.results(row="jsonrows")))) + + import csv + csvReader = csv.reader(q.results(row="csv"), delimiter=",", quotechar='"') + self.assertEqual(expected_sum, sum(map(lambda x: int(x[0]), csvReader))) + tsvReader = csv.reader(q.results(row="tsv"), delimiter="\t") + self.assertEqual(expected_sum, sum(map(lambda x: int(x[0]), tsvReader))) + + def testModelClassAutoloading(self): + q = self.SERVICE.model.Manager.select("name", "age") + expected_sum = 1383 + + self.assertEqual(expected_sum, sum(map(lambda x: x.age, q.results(row="object")))) + + +if __name__ == '__main__': + unittest.main() + diff --git a/tests/live_summary_test.py b/tests/live_summary_test.py new file mode 100644 index 00000000..a69435ae --- /dev/null +++ b/tests/live_summary_test.py @@ -0,0 +1,46 @@ +import sys +import os +sys.path.insert(0, os.getcwd()) + +import unittest +from intermine.webservice import Service + +class LiveSummaryTest(unittest.TestCase): + + TEST_ROOT = "localhost/intermine-test" + SERVICE = Service(TEST_ROOT) + + QUERY = SERVICE.select("Employee.*", "department.name") + + def testNumericSummary(self): + summary = self.QUERY.summarise("age") + self.assertEqual(10, summary["min"]) + self.assertEqual(74, summary["max"]) + self.assertEqual(44.878787878787875, summary["average"]) + self.assertEqual(12.075481627447155, summary["stdev"]) + + def testNonNumericSummary(self): + summary = self.QUERY.summarise("fullTime") + self.assertEqual(56, summary[True]) + self.assertEqual(76, summary[False]) + + summary = self.QUERY.summarise("department.name") + self.assertEqual(18, summary["Sales"]) + + def testSummaryAsIterator(self): + path = "department.name" + q = self.QUERY + results = q.results(summary_path = path) + top = results.next() + self.assertEqual("Accounting", top["item"]) + self.assertEqual(18, top["count"]) + + self.assertEqual(top, q.first(summary_path = path)) + + def testAliasing(self): + q = self.QUERY + self.assertEqual(q.summarise("age"), q.summarize("age")) + +if __name__ == '__main__': + unittest.main() + diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 00000000..71524669 --- /dev/null +++ b/tests/test.py @@ -0,0 +1,1263 @@ +import time +import unittest +import sys + +from intermine.model import * +from intermine.webservice import * +from intermine.query import * +from intermine.constraints import * +from intermine.lists.list import List + +from testserver import TestServer + +class WebserviceTest(unittest.TestCase): # pragma: no cover + TEST_PORT = 8000 + MAX_ATTEMPTS = 50 + maxDiff = None + + def assertIsNotNone(self, expr, msg = None): + try: + return unittest.TestCase.assertIsNotNone(self, expr, msg) + except AttributeError: + return self.assertTrue(expr is not None, msg) + + def get_test_root(self): + return "http://localhost:" + str(WebserviceTest.TEST_PORT) + "/testservice/service" + + def do_unpredictable_test(self, test, attempts=0, error=None): + if attempts < WebserviceTest.MAX_ATTEMPTS: + try: + test() + except IOError, e: + self.do_unpredictable_test(test, attempts + 1, e) + except: + e, t = sys.exc_info()[:2] + if 104 in t: # Handle connection reset errors + self.do_unpredictable_test(test, attempts + 1, t) + else: + raise + else: + raise RuntimeError("Max error count reached - last error: " + str(error)) + +class TestInstantiation(WebserviceTest): # pragma: no cover + + def testMakeModel(self): + """Should be about to make a model, or fail with an appropriate message""" + m = Model(self.get_test_root() + "/model") + self.assertTrue(isinstance(m, Model), "Can make a model") + try: + bad_m = Model("foo") + self.fail("No ModelParseError thrown at bad model xml") + except ModelParseError, ex: + self.assertEqual(ex.message, "Error parsing model") + + def testMakeService(self): + """Should be able to make a Service""" + s = Service(self.get_test_root()) + self.assertTrue(isinstance(s, Service), "Can make a service") + +class TestModel(WebserviceTest):# pragma: no cover + + model = None + + def setUp(self): + if self.model is None: + self.__class__.model = Model(self.get_test_root() + "/model") + + def testModelClasses(self): + '''The model should have the correct number of classes, which behave correctly''' + self.assertEqual(len(self.model.classes.items()), 19) + for good_class in ["Employee", "Company", "Department"]: + cd = self.model.get_class(good_class) + self.assertEqual(cd.name, good_class) + ceo = self.model.get_class("Employee.department.company.CEO") + self.assertEqual(ceo.name, "CEO") + self.assertTrue(ceo.isa("Employee")) + emp = self.model.get_class("Employee") + self.assertTrue(ceo.isa(emp)) + + try: + self.model.get_class("Foo") + self.fail("No ModelError thrown at non existent class") + except ModelError, ex: + self.assertEqual(ex.message, + "'Foo' is not a class in this model") + try: + self.model.get_class("Employee.name") + self.fail("No ModelError thrown at bad class retrieval") + except ModelError, ex: + self.assertEqual(ex.message, "'Employee.name' is not a class") + + def testClassFields(self): + '''The classes should have the correct fields''' + ceo = self.model.get_class("CEO") + for f in ["name", "age", "seniority", "address", "department"]: + fd = ceo.get_field(f) + self.assertEqual(fd.name, f) + self.assertTrue(isinstance(fd, Field)) + + try: + ceo.get_field("foo") + self.fail("No ModelError thrown at non existent field") + except ModelError, ex: + self.assertEqual(ex.message, + "There is no field called foo in CEO") + + def testFieldTypes(self): + '''The fields should be of the appropriate type''' + dep = self.model.get_class("Department") + self.assertTrue(isinstance(dep.get_field("name"), Attribute)) + self.assertTrue(isinstance(dep.get_field("employees"), Collection)) + self.assertTrue(isinstance(dep.get_field("company"), Reference)) + +class TestService(WebserviceTest): # pragma: no cover + + def setUp(self): + self.s = Service(self.get_test_root()) + + def testRoot(self): + """The service should have the right root""" + self.assertEqual(self.get_test_root(), self.s.root, "it has the right root") + + def testVersion(self): + """The service should have a version""" + from numbers import Number + v = self.s.version + self.assertTrue(isinstance(v, Number)) + + def testRelease(self): + """The service should have a release""" + self.assertEqual(self.s.release, "FOO\n") + + def testQueryMaking(self): + """The service should be able to make a query""" + q = self.s.new_query() + self.assertTrue(isinstance(q, Query), "Can make a query") + self.assertEqual(q.model.name, "testmodel", "and it has the right model") + +class TestQuery(WebserviceTest): # pragma: no cover + + model = None + service = None + expected_unary = '[, , ]' + expected_binary = '[ 50000>, , ]' + expected_multi = "[, ]" + expected_range = "[]" + expected_list = '[, ]' + expected_loop = '[, ]' + expected_ternary = '[, ]' + expected_subclass = "[]" + + XML_1 = """ + + + """ + EXPECTED_VIEWS_1 = ["Employee.name", "Employee.age"] + + XML_2 = """ + + + + """ + + EXPECTED_VIEWS_2 = ["Department.employees.name", + "Department.employees.age", "Department.employees.seniority"] + + XML_3 = """ + + + + + """ + + XML_4 = """ + + + + + """ + + XML_5 = """ + + + + + + """ + + XML_6 = """ + + + + + + + """ + + EXPECTED_VIEWS_6 = ["Department.employees.name", + "Department.employees.age", "Department.employees.salary"] + + XML_7 = """ + + + + + + + """ + + EXPECTED_VIEWS_7 = ["Department.employees.seniority", + "Department.employees.age", "Department.employees.salary"] + + CONSTRAINTS_COUNT_3 = 2 + + XML_8 = ''' + + + + + + ''' + + XML_9 = ''' + + + 1..10 + 30..35 + + + ''' + + XML_10 = ''' + + + foo + bar + + + ''' + + def setUp(self): + if self.service is None: + self.__class__.service = Service(self.get_test_root()) + if self.model is None: + self.__class__.model = Model(self.get_test_root() + "/model", self.service) + self.q = Query(self.model, self.service) + class DummyManager: + pass + list_dict = {"service": None, "manager": DummyManager(), "name": "my-list", "title": None, "type": "Employee", "size": 10} + self.l = List(**list_dict) + + def testFromXML(self): + q1 = self.service.new_query(xml = TestQuery.XML_1) + self.assertEqual(q1.views, TestQuery.EXPECTED_VIEWS_1) + + q2 = self.service.new_query(xml = TestQuery.XML_2) + self.assertEqual(q2.views, TestQuery.EXPECTED_VIEWS_2) + + q3 = self.service.new_query(xml = TestQuery.XML_3) + self.assertEqual(q3.views, TestQuery.EXPECTED_VIEWS_2) + self.assertEqual(len(q3.constraints), TestQuery.CONSTRAINTS_COUNT_3) + + q4 = self.service.new_query(xml = TestQuery.XML_4) + self.assertEqual(q4.views, TestQuery.EXPECTED_VIEWS_2) + self.assertEqual(len(q4.constraints), TestQuery.CONSTRAINTS_COUNT_3) + self.assertEqual(str(q4.get_sort_order()), "Department.employees.age asc") + + q5 = self.service.new_query(xml = TestQuery.XML_5) + self.assertEqual(q5.views, TestQuery.EXPECTED_VIEWS_2) + self.assertEqual(len(q5.constraints), TestQuery.CONSTRAINTS_COUNT_3) + self.assertEqual(str(q5.get_sort_order()), "Department.employees.age asc") + self.assertEqual(len(q5.path_descriptions), 1) + + q6 = self.service.new_query(xml = TestQuery.XML_6) + self.assertEqual(q6.views, TestQuery.EXPECTED_VIEWS_6) + self.assertEqual(len(q6.constraints), TestQuery.CONSTRAINTS_COUNT_3) + self.assertEqual(str(q6.get_sort_order()), "Department.employees.salary asc") + self.assertEqual(len(q6.path_descriptions), 2) + + q7 = self.service.new_query(xml = TestQuery.XML_7) + self.assertEqual(q7.views, TestQuery.EXPECTED_VIEWS_7) + self.assertEqual(len(q7.constraints), TestQuery.CONSTRAINTS_COUNT_3) + self.assertEqual(str(q7.get_sort_order()), "Department.employees.salary asc") + self.assertEqual(len(q7.path_descriptions), 2) + + q8 = self.service.new_query(xml = TestQuery.XML_8) + v = None + try: + v = q8.get_constraint("X").value + except: + pass + + self.assertIsNotNone(v, "query should have a constraint with the code 'X', but it only has the codes: %s" % map(lambda x: x.code, q8.constraints)) + self.assertEqual("foo", v, "And it has the right value") + + q9 = self.service.new_query(xml = TestQuery.XML_9) + self.assertEqual(len(q9.constraints), 1) + self.assertEqual(q9.get_constraint('A').op, 'OVERLAPS') + self.assertEqual(q9.get_constraint('A').values, ['1..10', '30..35']) + + q10 = self.service.new_query(xml = TestQuery.XML_10) + self.assertEqual(len(q10.constraints), 1) + self.assertEqual(q10.get_constraint('A').op, 'ONE OF') + self.assertEqual(q10.get_constraint('A').values, ['foo', 'bar']) + + def testAddViews(self): + """Queries should be able to add legal views, and complain about illegal ones""" + self.q.add_view("Employee.age") + self.q.add_view("Employee.name", "Employee.department.company.name") + self.q.add_view("Employee.department.name Employee.department.company.vatNumber") + self.q.add_view("Employee.department.manager.name,Employee.department.company.CEO.name") + self.q.add_view("Employee.department.manager.name, Employee.department.company.CEO.name") + self.q.add_view("department.*") + expected = [ + "Employee.age", "Employee.name", "Employee.department.company.name", + "Employee.department.name", "Employee.department.company.vatNumber", + "Employee.department.manager.name", "Employee.department.company.CEO.name", + "Employee.department.manager.name", "Employee.department.company.CEO.name", + "Employee.department.id", "Employee.department.name"] + self.assertEqual(self.q.views, expected) + try: + self.q.add_view("Employee.name", "Employee.age", "Employee.department") + self.fail("No ConstraintError thrown at non attribute view") + except ConstraintError, ex: + self.assertEqual(ex.message, "'Employee.department' does not represent an attribute") + + def testIDOnlyExpansion(self): + """Should be able to expand to a given depth, selecting only the id attribute for IM Object classes""" + self.q.prefetch_depth = 3 + self.q.prefetch_id_only = True + + self.q.add_view("Employee.*") + expected = [ + 'Employee.age', + 'Employee.end', + 'Employee.fullTime', + 'Employee.id', + 'Employee.name', + 'Employee.address.id', + 'Employee.department.id', + 'Employee.department.company.id', + 'Employee.department.manager.id', + 'Employee.department.employees.id', + 'Employee.department.rejectedEmployees.id', + 'Employee.departmentThatRejectedMe.id', + 'Employee.departmentThatRejectedMe.company.id', + 'Employee.departmentThatRejectedMe.manager.id', + 'Employee.departmentThatRejectedMe.employees.id', + 'Employee.departmentThatRejectedMe.rejectedEmployees.id', + 'Employee.simpleObjects.name', + 'Employee.simpleObjects.employee.id' + ] + self.assertEqual(self.q.views, expected) + exp_joins = ['Employee.address', + 'Employee.department', + 'Employee.department.company', + 'Employee.department.manager', + 'Employee.department.employees', + 'Employee.department.rejectedEmployees', + 'Employee.departmentThatRejectedMe', + 'Employee.departmentThatRejectedMe.company', + 'Employee.departmentThatRejectedMe.manager', + 'Employee.departmentThatRejectedMe.employees', + 'Employee.departmentThatRejectedMe.rejectedEmployees', + 'Employee.simpleObjects', + 'Employee.simpleObjects.employee'] + self.assertEqual([j.path for j in self.q.joins], exp_joins) + + def testDeepViews(self): + """Should be able to expand to a given prefetch depth""" + self.q.prefetch_depth = 3 + self.q.add_view("Employee.*") + expected = [ + 'Employee.age', + 'Employee.end', + 'Employee.fullTime', + 'Employee.id', + 'Employee.name', + 'Employee.address.address', + 'Employee.address.id', + 'Employee.department.id', + 'Employee.department.name', + 'Employee.department.company.id', + 'Employee.department.company.name', + 'Employee.department.company.vatNumber', + 'Employee.department.manager.age', + 'Employee.department.manager.end', + 'Employee.department.manager.fullTime', + 'Employee.department.manager.id', + 'Employee.department.manager.name', + 'Employee.department.manager.seniority', + 'Employee.department.manager.title', + 'Employee.department.employees.age', + 'Employee.department.employees.end', + 'Employee.department.employees.fullTime', + 'Employee.department.employees.id', + 'Employee.department.employees.name', + 'Employee.department.rejectedEmployees.age', + 'Employee.department.rejectedEmployees.end', + 'Employee.department.rejectedEmployees.fullTime', + 'Employee.department.rejectedEmployees.id', + 'Employee.department.rejectedEmployees.name', + 'Employee.departmentThatRejectedMe.id', + 'Employee.departmentThatRejectedMe.name', + 'Employee.departmentThatRejectedMe.company.id', + 'Employee.departmentThatRejectedMe.company.name', + 'Employee.departmentThatRejectedMe.company.vatNumber', + 'Employee.departmentThatRejectedMe.manager.age', + 'Employee.departmentThatRejectedMe.manager.end', + 'Employee.departmentThatRejectedMe.manager.fullTime', + 'Employee.departmentThatRejectedMe.manager.id', + 'Employee.departmentThatRejectedMe.manager.name', + 'Employee.departmentThatRejectedMe.manager.seniority', + 'Employee.departmentThatRejectedMe.manager.title', + 'Employee.departmentThatRejectedMe.employees.age', + 'Employee.departmentThatRejectedMe.employees.end', + 'Employee.departmentThatRejectedMe.employees.fullTime', + 'Employee.departmentThatRejectedMe.employees.id', + 'Employee.departmentThatRejectedMe.employees.name', + 'Employee.departmentThatRejectedMe.rejectedEmployees.age', + 'Employee.departmentThatRejectedMe.rejectedEmployees.end', + 'Employee.departmentThatRejectedMe.rejectedEmployees.fullTime', + 'Employee.departmentThatRejectedMe.rejectedEmployees.id', + 'Employee.departmentThatRejectedMe.rejectedEmployees.name', + 'Employee.simpleObjects.name', + 'Employee.simpleObjects.employee.age', + 'Employee.simpleObjects.employee.end', + 'Employee.simpleObjects.employee.fullTime', + 'Employee.simpleObjects.employee.id', + 'Employee.simpleObjects.employee.name' + ] + self.assertEqual(self.q.views, expected) + exp_joins = ['Employee.address', + 'Employee.department', + 'Employee.department.company', + 'Employee.department.manager', + 'Employee.department.employees', + 'Employee.department.rejectedEmployees', + 'Employee.departmentThatRejectedMe', + 'Employee.departmentThatRejectedMe.company', + 'Employee.departmentThatRejectedMe.manager', + 'Employee.departmentThatRejectedMe.employees', + 'Employee.departmentThatRejectedMe.rejectedEmployees', + 'Employee.simpleObjects', + 'Employee.simpleObjects.employee'] + self.assertEqual([j.path for j in self.q.joins], exp_joins) + + def testViewAlias(self): + """The aliases for add_view should work as well""" + self.q.select("Employee.age") + self.q.add_to_select("name") + self.q.add_column("department.name") + self.q.add_columns("department.manager.name") + self.q.add_views("department.company.CEO.name") + expected = [ + "Employee.age", "Employee.name", + "Employee.department.name", + "Employee.department.manager.name", "Employee.department.company.CEO.name"] + self.assertEqual(self.q.views, expected) + self.q.add_sort_order("name") + self.q.add_sort_order("department.name") + self.q.select("department.*") + expected = ["Employee.department.id", "Employee.department.name"] + self.assertEqual(self.q.views, expected) + self.assertEqual(len(self.q._sort_order_list), 1) + + def testSortOrder(self): + """Queries should be able to add sort orders, and complain appropriately""" + self.q.add_view("Employee.name", "Employee.age", "Employee.fullTime") + self.assertEqual(str(self.q.get_sort_order()), "Employee.name asc") + self.q.add_sort_order("Employee.fullTime", "desc") + self.assertEqual(str(self.q.get_sort_order()), "Employee.fullTime desc") + self.q.add_sort_order("Employee.age", "asc") + self.assertEqual(str(self.q.get_sort_order()), "Employee.fullTime desc Employee.age asc") + self.q.add_sort_order("Employee.end", "desc") + self.assertEqual(str(self.q.get_sort_order()), "Employee.fullTime desc Employee.age asc Employee.end desc") + self.assertRaises(ModelError, self.q.add_sort_order, "Foo", "asc") + self.assertRaises(TypeError, self.q.add_sort_order, "Employee.name", "up") + self.assertRaises(QueryError, self.q.add_sort_order, "Employee.department.name", "desc") + + def testUCSortOrder(self): + """Sort-Order directions should be accepted in upper case""" + self.q.add_view("Employee.name", "Employee.age", "Employee.fullTime") + self.q.add_sort_order("Employee.age", "ASC") + self.assertEqual(str(self.q.get_sort_order()), "Employee.age asc") + self.q.add_sort_order("Employee.fullTime", "DESC") + self.assertEqual(str(self.q.get_sort_order()), "Employee.age asc Employee.fullTime desc") + + def testConstraintPathProblems(self): + """Queries should not add constraints with bad paths to themselves""" + try: + self.q.add_constraint('Foo', 'IS NULL') + self.fail("No ModelError thrown at bad path name") + except ModelError, ex: + self.assertEqual(ex.message, "'Foo' is not a class in this model") + + def testUnaryConstraints(self): + """Queries should be fine with NULL/NOT NULL constraints""" + self.q.add_constraint('Employee.age', 'IS NULL') + self.q.add_constraint('Employee.name', 'IS NOT NULL') + self.q.add_constraint('Employee.address', 'IS NULL') + self.assertEqual(self.q.constraints.__repr__(), self.expected_unary) + + def testUnaryConstraintsSugar(self): + """Queries should be fine with NULL/NOT NULL constraints""" + Employee = self.q.model.table("Employee") + + self.q.add_constraint(Employee.age == None) + self.q.add_constraint(Employee.name != None) + self.q.add_constraint(Employee.address == None) + self.assertEqual(self.q.constraints.__repr__(), self.expected_unary) + + def testAddBinaryConstraints(self): + """Queries should be able to handle constraints on attribute values""" + self.q.add_constraint('Employee.age', '>', 50000) + self.q.add_constraint('Employee.name', '=', 'John') + self.q.add_constraint('Employee.end', '!=', 0) + self.assertEqual(self.q.constraints.__repr__(), self.expected_binary) + try: + self.q.add_constraint('Employee.department', '=', "foo") + self.fail("No ConstraintError thrown for non attribute BinaryConstraint") + except ConstraintError, ex: + self.assertEqual(ex.message, "'Employee.department' does not represent an attribute") + + def testAddBinaryConstraintsSugar(self): + """Queries should be able to handle constraints on attribute values""" + Employee = self.q.model.table("Employee") + + self.q.add_constraint(Employee.age > 50000) + self.q.add_constraint(Employee.name == 'John') + self.q.add_constraint(Employee.end != 0) + self.assertEqual(self.q.constraints.__repr__(), self.expected_binary) + try: + self.q.add_constraint(Employee.department == "foo") + self.fail("No ConstraintError thrown for non attribute BinaryConstraint") + except ConstraintError, ex: + self.assertEqual(ex.message, "'Employee.department' does not represent an attribute") + + def testTernaryConstraint(self): + """Queries should be able to add constraints for LOOKUPs""" + self.q.add_constraint('Employee', 'LOOKUP', 'Susan') + self.q.add_constraint('Employee.department.manager', 'LOOKUP', 'John', 'Wernham-Hogg') + self.assertEqual(self.q.constraints.__repr__(), self.expected_ternary) + try: + self.q.add_constraint('Employee.department.name', 'LOOKUP', "foo") + self.fail("No ConstraintError thrown for non object TernaryConstraint") + except ConstraintError, ex: + self.assertEqual(ex.message, "'Employee.department.name' does not represent a class, or a reference to a class") + + def testMultiConstraint(self): + """Queries should be ok with multi-value constraints""" + self.q.add_constraint('Employee.name', 'ONE OF', ['Tom', 'Dick', 'Harry']) + self.q.add_constraint('Employee.name', 'NONE OF', ['Sue', 'Jane', 'Helen']) + self.assertEqual(self.q.constraints.__repr__(), self.expected_multi) + self.assertRaises(TypeError, self.q.add_constraint, "Employee.name", "ONE OF", "Tom, Dick, Harry") + self.assertRaises(ConstraintError, self.q.add_constraint, "Employee", "ONE OF", ["Tom", "Dick", "Harry"]) + + def testMultiConstraintSugar(self): + """Queries should be ok with multi-value constraints""" + Employee = self.q.model.table("Employee") + + self.q.add_constraint(Employee.name == ['Tom', 'Dick', 'Harry']) + self.q.add_constraint(Employee.name != ['Sue', 'Jane', 'Helen']) + self.assertEqual(self.q.constraints.__repr__(), self.expected_multi) + self.q.add_constraint(Employee.name == "Tom, Dick, Harry") # This method does not throw an error in this form!! + self.assertRaises(ConstraintError, self.q.add_constraint, Employee == ["Tom", "Dick", "Harry"]) + + def testRangeConstraint(self): + """Queries should be OK with range constraints""" + self.q.add_constraint('Employee.age', 'OVERLAPS', ['1..10', '30..35']) + self.assertEqual(self.q.constraints.__repr__(), self.expected_range) + self.assertRaises(TypeError, self.q.add_constraint, "Employee.age", "OVERLAPS", "Tom, Dick, Harry") + self.q.add_constraint('Employee', 'OVERLAPS', ['1..10', '30..35']) + + def testListConstraint(self): + """Queries should be ok with list constraints""" + self.q.add_constraint('Employee', 'IN', 'my-list') + self.q.add_constraint('Employee.department.manager', 'NOT IN', 'my-list') + self.assertEqual(self.q.constraints.__repr__(), self.expected_list) + self.assertRaises(ConstraintError, self.q.add_constraint, "Employee.name", "IN", "some list") + + def testListConstraintSugar(self): + """Queries should be ok with list constraints""" + Employee = self.q.model.table("Employee") + + self.q.add_constraint(Employee == self.l) + self.q.add_constraint(Employee.department.manager != self.l) + self.assertEqual(self.q.constraints.__repr__(), self.expected_list) + self.assertRaises(ConstraintError, self.q.add_constraint, Employee.name == self.l) + + def testLoopConstraint(self): + """Queries should be ok with loop constraints""" + self.q.add_constraint('Employee', 'IS', 'Employee.department.manager') + self.q.add_constraint('Employee.department.manager', 'IS NOT', 'Employee.department.company.CEO') + self.assertEqual(self.q.constraints.__repr__(), self.expected_loop) + self.assertRaises(ConstraintError, self.q.add_constraint, "Employee", "IS", "Employee.department") + + def testLoopConstraintSugar(self): + """Queries should be ok with loop constraints made with alchemical sugar""" + Employee = self.q.model.table("Employee") + + self.q.add_constraint(Employee == Employee.department.manager) + self.q.add_constraint(Employee.department.manager != Employee.department.company.CEO) + self.assertEqual(self.q.constraints.__repr__(), self.expected_loop) + self.assertRaises(ConstraintError, self.q.add_constraint, Employee == Employee.department) + + def testSubclassConstraints(self): + """Queries should be ok with sub class constraints""" + self.q.add_constraint('Department.employees', 'Manager') + self.assertEqual(self.q.constraints.__repr__(), self.expected_subclass) + try: + self.q.add_constraint('Department.company.CEO', 'Foo') + self.fail("No ModelError raised by bad sub class") + except ModelError, ex: + self.assertEqual(ex.message, "'Foo' is not a class in this model") + try: + self.q.add_constraint('Department.company.CEO', 'Manager') + self.fail("No ConstraintError raised by bad subclass relationship") + except ConstraintError, ex: + self.assertEqual(ex.message, "'Manager' is not a subclass of 'Department.company.CEO'") + + def testStringLogic(self): + """Queries should be able to parse good logic strings""" + + a = self.q.add_constraint("Employee.name", "IS NOT NULL") + b = self.q.add_constraint("Employee.age", ">", 10) + c = self.q.add_constraint("Employee.department", "LOOKUP", "Sales", "Wernham-Hogg") + d = self.q.add_constraint("Employee.department.employees.name", "ONE OF", + ["John", "Paul", "Mary"]) + self.q.add_constraint("Employee.department.employees", "Manager") + + self.assertEqual(str(self.q.get_logic()), "A and B and C and D") + self.q.set_logic("(B or C) and (A or D)") + self.assertEqual(str(self.q.get_logic()), "(B or C) and (A or D)") + self.q.set_logic("B and C or A and D") + self.assertEqual(str(self.q.get_logic()), "B and (C or A) and D") + self.q.set_logic("(A and B) or (A and C and D)") + self.assertEqual(str(self.q.get_logic()), "(A and B) or (A and C and D)") + + def testIrrelevantCodeStripping(self): + """Should be able to recover from queries that have irrelevant codes in their logic""" + + a = self.q.add_constraint("Employee.name", "IS NOT NULL") + b = self.q.add_constraint("Employee.age", ">", 10) + c = self.q.add_constraint("Employee.department", "LOOKUP", "Sales", "Wernham-Hogg") + d = self.q.add_constraint("Employee.department.employees.name", "ONE OF", + ["John", "Paul", "Mary"]) + self.q.add_constraint("Employee.department.employees", "Manager") + + self.q._set_questionable_logic("A and B or C or D and E") + self.assertEqual(str(self.q.get_logic()), "A and (B or C or D)") + self.q._set_questionable_logic("E and A and B or C or D") + self.assertEqual(str(self.q.get_logic()), "A and (B or C or D)") + self.q._set_questionable_logic("A and B and E or C or D") + self.assertEqual(str(self.q.get_logic()), "A and B and (C or D)") + self.q._set_questionable_logic("A and B or X and J or Z and E or C or D") + self.assertEqual(str(self.q.get_logic()), "A and B and (C or D)") + self.q._set_questionable_logic("A and B or X and (J or Z and E or C) or D") + self.assertEqual(str(self.q.get_logic()), "(A and B and C) or D") + self.q._set_questionable_logic("A or (B or X and J or Z and E or C) or D") + self.assertEqual(str(self.q.get_logic()), "A or (B and C) or D") + + def testObjectLogic(self): + """Queries should be able to set logic from object methods""" + + a = self.q.add_constraint("Employee.name", "IS NOT NULL") + b = self.q.add_constraint("Employee.age", ">", 10) + c = self.q.add_constraint("Employee.department", "LOOKUP", "Sales", "Wernham-Hogg") + d = self.q.add_constraint("Employee.department.employees.name", "ONE OF", + ["John", "Paul", "Mary"]) + self.q.add_constraint("Employee.department.employees", "Manager") + + self.q.set_logic(a + b + c + d) + self.assertEqual(str(self.q.get_logic()), "A and B and C and D") + self.q.set_logic(a & b & c & d) + self.assertEqual(str(self.q.get_logic()), "A and B and C and D") + self.q.set_logic(a | b | c | d) + self.assertEqual(str(self.q.get_logic()), "A or B or C or D") + self.q.set_logic(a + b & c | d) + self.assertEqual(str(self.q.get_logic()), "(A and B and C) or D") + + self.assertEqual(repr(self.q.get_logic()), '') + + self.assertRaises(ConstraintError, self.q.set_logic, "E and C or A and D") + self.assertRaises(QueryError, self.q.set_logic, "A and B and C") + self.assertRaises(LogicParseError, self.q.set_logic, "A and B and C not D") + self.assertRaises(LogicParseError, self.q.set_logic, "A and ((B and C and D)") + self.assertRaises(LogicParseError, self.q.set_logic, "A and ((B and C) and D))") + self.assertRaises(LogicParseError, self.q.set_logic, "A and B( and C and D)") + self.assertRaises(LogicParseError, self.q.set_logic, "A and (B and C and )D") + self.assertRaises(LogicParseError, self.q.set_logic, "A and (B and C) D") + self.assertRaises(LogicParseError, self.q.set_logic, "A and (B and C) (D and E)") + self.assertRaises(TypeError, lambda: self.q.get_logic() + 1) + self.assertRaises(TypeError, lambda: self.q.get_logic() & 1) + self.assertRaises(TypeError, lambda: self.q.get_logic() | 1) + self.assertRaises(TypeError, lambda: LogicGroup(a, "bar", b)) + + def testJoins(self): + """Queries should be able to add joins""" + self.assertRaises(TypeError, self.q.add_join, 'Employee.department', 'foo') + self.assertRaises(QueryError, self.q.add_join, 'Employee.age', 'inner') + self.assertRaises(ModelError, self.q.add_join, 'Employee.foo', 'inner') + self.q.add_join('Employee.department', 'inner') + self.q.add_join('Employee.department.company', 'outer') + expected = "[, ]" + self.assertEqual(expected, self.q.joins.__repr__()) + + def testXML(self): + """Queries should be able to serialise themselves to XML""" + self.q.add_view("Employee.name", "Employee.age", "Employee.department.name") + self.q.add_constraint("Employee.name", "IS NOT NULL") + self.q.add_constraint("Employee.age", ">", 10) + self.q.add_constraint("Employee.department", "LOOKUP", "Sales", "Wernham-Hogg") + self.q.add_constraint("Employee.department.employees.name", "ONE OF", + ["John", "Paul", "Mary"]) + self.q.add_constraint("Employee.department.manager", "IS", "Employee") + self.q.add_constraint("Employee", "IN", "some list of employees") + self.q.add_constraint("Employee.age", "OVERLAPS", ["1..10", "30..35"]) + self.q.add_constraint("Employee.department.employees", "Manager") + self.q.add_join("Employee.department", "outer") + self.q.add_sort_order("Employee.age") + self.q.set_logic("(A and B) or (A and C and D) and (E or F or G)") + expected ='JohnPaulMary1..1030..35' + self.assertEqual(expected, self.q.to_xml()) + self.assertEqual(expected, self.q.clone().to_xml()) # Clones must produce identical XML + + def testSugaryQueryConstruction(self): + """Test use of operation coercion which is similar to SQLAlchemy""" + model = self.q.model + + Employee = model.table("Employee") + Manager = model.table("Manager") + + expected = 'JohnPaulMary' + + # SQL style + q = Employee.\ + select("name", "age", "department.name").\ + where(Employee.name != None).\ + where(Employee.age > 10).\ + where(Employee.department % ("Sales", "Wernham-Hogg")).\ + where(Employee.department.employees.name == ["John", "Paul", "Mary"]).\ + where(Employee.department.manager == Employee).\ + where(Employee == self.l).\ + where(Employee.department.employees >> Manager).\ + outerjoin(Employee.department).\ + order_by(Employee.age).\ + set_logic("(A and B) or (A and C and D) and (E or F)") + + self.assertEqual(expected, q.to_xml()) + + # SQLAlchemy style + q = self.service.query(Employee).\ + select("name", "age", "department.name").\ + filter(Employee.name != None).\ + filter(Employee.age > 10).\ + filter(Employee.department % ("Sales", "Wernham-Hogg")).\ + filter(Employee.department.employees.name == ["John", "Paul", "Mary"]).\ + filter(Employee.department.manager == Employee).\ + filter(Employee == self.l).\ + filter(Employee.department.employees >> Manager).\ + outerjoin(Employee.department).\ + order_by(Employee.age).\ + set_logic("(A and B) or (A and C and D) and (E or F)") + + self.assertEqual(expected, q.to_xml()) + + def testKWCons(self): + """Test use of constraints provided in kwargs""" + + model = self.q.model + + expected = '' + + q = model.Employee.select("name").where(age = 10) + + self.assertEqual(expected, q.to_xml()) + + def testLogicConstraintTrees(self): + # Actually SQL-Alchemy-esque + expected = """ + + + + + + + + + """ + e = self.service.model.Employee + CEO = self.service.model.CEO + q = self.service.query(e, e.department.name).\ + filter( + e.department.manager < CEO, + ( + ((e.name != None) & (e.age > 10)) + | (e.in_(self.l) & (e.department.manager % "David")) + ) + ).\ + outerjoin(e.department).\ + order_by(e.age) + + expected = re.sub(r'\s+', ' ', expected) + expected = re.sub(r'>\s+<', '><', expected) + expected = expected.strip() + self.assertEqual(expected, q.to_xml()) + +class TestTemplate(TestQuery): # pragma: no cover + + expected_unary = '[, , ]' + expected_binary = '[ 50000 (editable, locked)>, , ]' + expected_multi = "[, ]" + expected_range = "[]" + expected_ternary = '[, ]' + expected_subclass = '[]' + expected_list = '[, ]' + expected_loop = '[, ]' + + def setUp(self): + super(TestTemplate, self).setUp() + self.q = Template(self.model) + +class TestQueryResults(WebserviceTest): # pragma: no cover + + model = None + service = None + + class MockService(object): + + QUERY_PATH = '/QUERY-PATH' + TEMPLATEQUERY_PATH = '/TEMPLATE-PATH' + root = 'ROOT' + prefetch_depth = 1 + prefetch_id_only = False + + def get_results(self, *args): + return args + + def setUp(self): + if self.service is None: + self.__class__.service = Service(self.get_test_root()) + if self.model is None: + self.__class__.model = Model(self.get_test_root() + "/model") + + q = Query(self.model, self.service) + q.add_view("Employee.name", "Employee.age", "Employee.id") + self.query = q + t = Template(self.model, self.service) + t.add_view("Employee.name", "Employee.age", "Employee.id") + t.add_constraint("Employee.name", '=', "Fred") + t.add_constraint("Employee.age", ">", 25) + self.template = t + + def testURLs(self): + """Should be able to produce the right information for opening urls""" + q = Query(self.model, self.MockService()) + q.add_view("Employee.name", "Employee.age", "Employee.id") + q.add_constraint("Employee.name", '=', "Fred") + q.add_constraint("Employee.age", ">", 25) + + t = Template(self.model, self.MockService()) + t.name = "TEST-TEMPLATE" + t.add_view("Employee.name", "Employee.age", "Employee.id") + t.add_constraint("Employee.name", '=', "Fred") + t.add_constraint("Employee.age", ">", 25) + + expectedQ = ( + '/QUERY-PATH', + { + 'query': '', + 'start': 0 + }, + 'object', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + self.assertEqual(expectedQ, q.results()) + self.assertEqual(list(expectedQ), q.get_results_list()) + + expectedQ = ( + '/QUERY-PATH', + { + 'query': '', + 'start': 0 + }, + 'rr', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + self.assertEqual(expectedQ, q.rows()) + self.assertEqual(list(expectedQ), q.get_row_list()) + + expectedQ = ( + '/QUERY-PATH', + { + 'query': '', + 'start': 10, + 'size': 200 + }, + 'object', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + self.assertEqual(expectedQ, q.results(start=10, size=200)) + self.assertEqual(list(expectedQ), q.get_results_list(start=10, size=200)) + + expected1 = ( + '/TEMPLATE-PATH', + { + 'name': 'TEST-TEMPLATE', + 'code1': 'A', + 'code2': 'B', + 'constraint1': 'Employee.name', + 'constraint2': 'Employee.age', + 'op1': '=', + 'op2': '>', + 'value1': 'Fred', + 'value2': '25', + 'start': 0 + }, + 'object', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + self.assertEqual(expected1, t.results()) + self.assertEqual(list(expected1), t.get_results_list()) + + expected1 = ( + '/TEMPLATE-PATH', + { + 'name': 'TEST-TEMPLATE', + 'code1': 'A', + 'code2': 'B', + 'constraint1': 'Employee.name', + 'constraint2': 'Employee.age', + 'op1': '=', + 'op2': '>', + 'value1': 'Fred', + 'value2': '25', + 'start': 0 + }, + 'rr', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + self.assertEqual(expected1, t.rows()) + self.assertEqual(list(expected1), t.get_row_list()) + + expected2 = ( + '/TEMPLATE-PATH', + { + 'name': 'TEST-TEMPLATE', + 'code1': 'A', + 'code2': 'B', + 'constraint1': 'Employee.name', + 'constraint2': 'Employee.age', + 'op1': '<', + 'op2': '>', + 'value1': 'Tom', + 'value2': '55', + 'start': 0 + }, + 'object', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + self.assertEqual(expected2, t.results( + A = {"op": "<", "value": "Tom"}, + B = {"value": 55} + )) + + expected2 = ( + '/TEMPLATE-PATH', + { + 'name': 'TEST-TEMPLATE', + 'code1': 'A', + 'code2': 'B', + 'constraint1': 'Employee.name', + 'constraint2': 'Employee.age', + 'op1': '<', + 'op2': '>', + 'value1': 'Tom', + 'value2': '55', + 'start': 10, + 'size': 200 + }, + 'object', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + self.assertEqual(expected2, t.results( + start = 10, + size = 200, + A = {"op": "<", "value": "Tom"}, + B = {"value": 55} + )) + self.assertEqual(list(expected2), t.get_results_list( + start = 10, + size = 200, + A = {"op": "<", "value": "Tom"}, + B = {"value": 55} + )) + + # Check that we can just use strings for simple value replacement. + expected3 = ( + '/TEMPLATE-PATH', + { + 'name': 'TEST-TEMPLATE', + 'code1': 'A', + 'code2': 'B', + 'constraint1': 'Employee.name', + 'constraint2': 'Employee.age', + 'op1': '=', + 'op2': '>', + 'value1': 'Foo', + 'value2': 'Bar', + 'start': 10, + 'size': 200 + }, + 'object', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + self.assertEqual(list(expected3), t.get_results_list( + start = 10, + size = 200, + A = "Foo", + B = "Bar" + )) + # Check that these contraint values have not been applied to the actual template + self.assertEqual(expected1, t.rows()) + expected1 = ( + '/TEMPLATE-PATH', + { + 'name': 'TEST-TEMPLATE', + 'code1': 'A', + 'code2': 'B', + 'constraint1': 'Employee.name', + 'constraint2': 'Employee.age', + 'op1': '=', + 'op2': '>', + 'value1': 'Fred', + 'value2': '25', + 'start': 0 + }, + 'object', + ['Employee.name', 'Employee.age', 'Employee.id'], + self.model.get_class("Employee") + ) + + self.assertEqual(expected1, t.results()) + + def testResultsList(self): + """Should be able to get results as one list per row""" + def logic(): + expected = [['foo', 'bar', 'baz'], [123, 1.23, -1.23], [True, False, None]] + self.assertEqual(self.query.get_results_list("list"), expected) + self.assertEqual(self.template.get_results_list("list"), expected) + + self.do_unpredictable_test(logic) + + def testResultRows(self): + """Should be able to get results as result rows""" + def logic(): + assertEqual = self.assertEqual + q_res = self.query.all("rr") + t_res = self.template.all("rr") + for results in [q_res, t_res]: + assertEqual(results[0]["age"], 'bar') # Can index by short path + assertEqual(results[1]["Employee.age"], 1.23) # or by full path + assertEqual(results[2][0], True) # or by numerical index + assertEqual(len(results), 3) + for row in results: + assertEqual(len(row), 3) + self.do_unpredictable_test(logic) + + def testResultRowIterability(self): + """Result rows should iterate as lists""" + def logic(): + q_res = self.query.all("rr") + t_res = self.template.all("rr") + for results in [q_res, t_res]: + # 'in' as test of iterability + self.assertTrue("bar" in results[0]) + self.assertTrue(1.23 in results[1]) + self.assertTrue(True in results[2]) + # Should be able to actually iterate + count = 0 + for val in results[0]: + count += 1 + self.assertTrue(0 < count < 4) + + self.do_unpredictable_test(logic) + + def testResultRowDictBehaviour(self): + """Result rows should allow iteration using items() and iterkeys()""" + def logic(): + q_res = self.query.all("rr") + t_res = self.template.all("rr") + for results in [q_res, t_res]: + r = results[0] + count = 0 + for (k, v) in r.items(): + count += 1 + self.assertTrue(0 < count < 4) + count = 0 + self.assertEqual([("Employee.name", 'foo'), ("Employee.age", 'bar'), ("Employee.id", 'baz')], r.items()) + for (k, v) in r.iteritems(): + count += 1 + self.assertTrue(0 < count < 4) + self.assertEqual([pair for pair in r.iteritems()], r.items()) + self.assertEqual(r.keys(), self.query.views) + self.assertEqual(r.values(), r.to_l()) + self.assertEqual(zip(r.values(), r.keys()), zip(r.itervalues(), r.iterkeys())) + self.assertTrue(r.has_key("age")) + self.assertTrue(r.has_key("Employee.age")) + self.assertTrue(not r.has_key("Employee.foo")) + + self.do_unpredictable_test(logic) + + def testResultsDict(self): + """Should be able to get results as one dictionary per row""" + expected = [ + {'Employee.age': u'bar', 'Employee.id': u'baz', 'Employee.name': u'foo'}, + {'Employee.age': 1.23, 'Employee.id': -1.23, 'Employee.name': 123}, + {'Employee.age': False, 'Employee.id': None, 'Employee.name': True} + ] + def logic(): + self.assertEqual(self.query.get_results_list("dict"), expected) + self.assertEqual(self.template.get_results_list("dict"), expected) + + self.do_unpredictable_test(logic) + +class MinimalResultsTest(TestQueryResults): + + PATH = "/testservice/legacyjsonrows" + +class TestTSVResults(WebserviceTest): # pragma: no cover + + model = None + service = None + PATH = "/testservice/tsvservice" + FORMAT = "tsv" + EXPECTED_RESULTS = ['foo\tbar\tbaz', '123\t1.23\t-1.23'] + + def get_test_root(self): + return "http://localhost:" + str(self.TEST_PORT) + self.PATH + + def setUp(self): + if self.service is None: + self.__class__.service = Service(self.get_test_root()) + if self.model is None: + self.__class__.model = Model(self.get_test_root() + "/service/model") + + q = Query(self.model, self.service) + q.add_view("Employee.name", "Employee.age", "Employee.id") + self.query = q + t = Template(self.model, self.service) + t.add_view("Employee.name", "Employee.age", "Employee.id") + t.add_constraint("Employee.name", '=', "Fred") + t.add_constraint("Employee.age", ">", 25) + self.template = t + + def testResults(self): + """Should be able to get results as one string per row""" + def logic(): + self.assertEqual(self.query.get_results_list(self.FORMAT), self.EXPECTED_RESULTS) + self.assertEqual(self.template.get_results_list(self.FORMAT), self.EXPECTED_RESULTS) + self.do_unpredictable_test(logic) + + +class TestCSVResults(TestTSVResults): # pragma: no cover + + PATH = "/testservice/csvservice" + FORMAT = "csv" + EXPECTED_RESULTS = ['"foo","bar","baz"', '"123","1.23","-1.23"'] + +class TestResultObjects(WebserviceTest): # pragma: no cover + model = None + service = None + + def get_test_root(self): + return "http://localhost:" + str(self.TEST_PORT) + "/testservice/testresultobjs" + + def setUp(self): + if self.service is None: + self.__class__.service = Service(self.get_test_root()) + if self.model is None: + self.__class__.model = self.service.model + + q = Query(self.model, self.service) + q.add_view("Department.name", "Department.employees.name", "Department.employees.age", "Department.company.vatNumber") + self.query = q + t = Template(self.model, self.service) + q.add_view("Department.name", "Department.employees.name", "Department.employees.age", "Department.company.vatNumber") + t.add_constraint("Department.manager.name", '=', "Fred") + self.template = t + + def testResultObjs(self): + """Should be able to get results as result objects""" + def logic(): + assertEqual = self.assertEqual + q_res = self.query.all("jsonobjects") + t_res = self.template.all("jsonobjects") + for departments in [q_res, t_res]: + assertEqual(departments[0].name, 'Sales') + assertEqual(departments[0].company.vatNumber, 665261) + assertEqual(departments[0].employees[2].name, "Tim Canterbury") + assertEqual(departments[0].employees[3].age, 58) + assertEqual(len(departments[0].employees), 6) + + assertEqual(departments[-1].name, 'Slashes') + assertEqual(departments[-1].company.vatNumber, 764575) + assertEqual(departments[-1].employees[2].name, "Double forward Slash //") + assertEqual(departments[-1].employees[2].age, 62) + assertEqual(len(departments[-1].employees), 5) + + for idx in [0, -1]: + self.assertRaises(ModelError, lambda: departments[idx].foo) # Model errors are thrown for illegal field access + self.assertRaises(ModelError, lambda: departments[idx].company.foo) + + assertEqual(len(departments), 8) + + self.do_unpredictable_test(logic) + +class TestCountResults(TestTSVResults): # pragma: no cover + + PATH = "/testservice/countservice" + FORMAT = "count" + EXPECTED_RESULTS = ['25'] + EXPECTED_COUNT = 25 + + def testCount(self): + """Should be able to get count as an integer""" + def logic(): + self.assertEqual(self.query.count(), self.EXPECTED_COUNT) + self.assertEqual(self.template.count(), self.EXPECTED_COUNT) + self.do_unpredictable_test(logic) + +if __name__ == '__main__': # pragma: no cover + server = TestServer() + server.start() + time.sleep(0.1) # Avoid race conditions with the server + unittest.main() + server.shutdown() diff --git a/tests/test_lists.py b/tests/test_lists.py new file mode 100644 index 00000000..3fad97f7 --- /dev/null +++ b/tests/test_lists.py @@ -0,0 +1,53 @@ +import unittest +from test import WebserviceTest + +from intermine.webservice import * +from intermine.lists.list import List + +class TestLists(WebserviceTest): # pragma: no cover + + def setUp(self): + self.service = Service(self.get_test_root()) + + def testGetLists(self): + """Should be able to get lists from a service""" + self.assertEqual(self.service.get_list_count(), 3) + + list_a = self.service.get_list("test-list-1") + self.assertTrue(list_a.description, "An example test list") + self.assertEqual(list_a.size, 42) + self.assertEqual(list_a.count, 42) + self.assertEqual(len(list_a), 42) + self.assertEqual(list_a.title, "test1") + self.assertTrue(list_a.is_authorized) + self.assertEqual(list_a.list_type, "Employee") + self.assertEqual(list_a.tags, frozenset(["tag1", "tag2", "tag3"])) + + list_a = self.service.get_list("test-list-2") + self.assertTrue(list_a.description, "Another example test list") + self.assertEqual(list_a.size, 7) + self.assertEqual(len(list_a), 7) + self.assertEqual(list_a.count, 7) + self.assertTrue(not list_a.is_authorized) + self.assertEqual(list_a.tags, frozenset([])) + + list_c = self.service.get_list("test-list-3") + self.assertTrue(list_c.description, "Yet Another example test list") + self.assertEqual(list_c.size, 8) + self.assertTrue(list_c.is_authorized) + + def alter_size(): + list_a.size = 10 + def alter_type(): + list_a.list_type = "foo" + self.assertRaises(AttributeError, alter_size) + self.assertRaises(AttributeError, alter_type) + + def testBadListConstruction(self): + args = {} + self.assertRaises(ValueError, lambda: List(**args)) + + def tearDown(self): + s = self.service + s.__del__() + diff --git a/tests/test_templates.py b/tests/test_templates.py new file mode 100644 index 00000000..56579fbe --- /dev/null +++ b/tests/test_templates.py @@ -0,0 +1,155 @@ +import unittest +from test import WebserviceTest + +from intermine.webservice import * +from intermine.query import Template +from intermine.constraints import TemplateConstraint + +class TestTemplates(WebserviceTest): # pragma: no cover + + def setUp(self): + self.service = Service(self.get_test_root()) + + def testGetTemplate(self): + """Should be able to get a template from the webservice, if it exists, and get its results""" + self.assertEqual(len(self.service.templates), 12) + t = self.service.get_template("MultiValueConstraints") + self.assertTrue(isinstance(t, Template)) + expected = "[]" + self.assertEqual(t.editable_constraints.__repr__(), expected) + expected = [[u'foo', u'bar', u'baz'], [123, 1.23, -1.23], [True, False, None]] + attempts = 0 + def do_tests(error=None): + if attempts < 5: + try: + self.assertEqual(t.get_results_list("list"), expected) + except IOError, e: + do_tests(e) + else: + raise RuntimeError("Error connecting to " + self.query.service.root, error) + + do_tests() + try: + self.service.get_template("Non_Existant") + self.fail("No ServiceError raised by non-existant template") + except ServiceError, ex: + self.assertEqual(ex.message, "There is no template called 'Non_Existant' at this service") + + def testIrrelevantSO(self): + """Should fix up bad sort orders and logic when parsing from xml""" + model = self.service.model + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_sort_order()), "Employee.name asc") + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_sort_order()), "Employee.name asc") + + def testCodesInOrder(self): + """Should associate the right constraints with the right codes""" + model = self.service.model + + xml = ''' + + ''' + t = Template.from_xml(xml, model) + v = None + try: + v = t.get_constraint("X").value + except: + pass + + self.assertIsNotNone(v, msg = "Query (%s) should have a constraint with the code 'X'" % t) + self.assertEqual("foo", v, msg = "should be the correct constraint") + + def testIrrelevantConstraintLogic(self): + """Should fix up bad logic""" + model = self.service.model + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_logic()), "") + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_logic()), "") + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_logic()), "A or B") + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_logic()), "(A or B) and C") + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_logic()), "A or B or C") + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_logic()), "(A or B) and C") + + xml = '''''' + t = Template.from_xml(xml, model) + self.assertEqual(str(t.get_logic()), "(A or B or D) and C") + + def testTemplateConstraintParsing(self): + """Should be able to parse template constraints""" + t = self.service.get_template("UneditableConstraints") + self.assertEqual(len(t.constraints), 2) + self.assertEqual(len(t.editable_constraints), 1) + expected = '[]' + self.assertEqual(expected, repr(t.editable_constraints)) + self.assertEqual('', repr(t.get_constraint("B"))) + + t2 = self.service.get_template("SwitchableConstraints") + self.assertEqual(len(t2.editable_constraints), 3) + con = t2.get_constraint("A") + self.assertTrue(con.editable and con.required and con.switched_on) + con = t2.get_constraint("B") + self.assertTrue(con.editable and con.optional and con.switched_on) + self.assertEqual('', repr(con)) + con.switch_off() + self.assertTrue(con.editable and con.optional and con.switched_off) + self.assertEqual('', repr(con)) + con.switch_on() + self.assertTrue(con.editable and con.optional and con.switched_on) + con = t2.get_constraint("C") + self.assertTrue(con.editable and con.optional and con.switched_off) + + self.assertRaises(ValueError, lambda: t2.get_constraint("A").switch_off()) + self.assertRaises(ValueError, lambda: t2.get_constraint("A").switch_on()) + + def testBadTemplateConstraint(self): + self.assertRaises(TypeError, lambda: TemplateConstraint(True, "BAD_VALUE")) + + diff --git a/tests/testserver.py b/tests/testserver.py new file mode 100644 index 00000000..c3808f8d --- /dev/null +++ b/tests/testserver.py @@ -0,0 +1,67 @@ +import threading +import time +import os +import posixpath +import urllib +from socket import socket +from SimpleHTTPServer import SimpleHTTPRequestHandler +from BaseHTTPServer import HTTPServer + +class SilentRequestHandler(SimpleHTTPRequestHandler): # pragma: no cover + + silent = True + + def translate_path(self, path): + """Use the file's location instead of cwd""" + # abandon query parameters + path = path.split('?',1)[0] + path = path.split('#',1)[0] + path = posixpath.normpath(urllib.unquote(path)) + words = path.split('/') + words = filter(None, words) + path = os.path.dirname(__file__) + for word in words: + drive, word = os.path.splitdrive(word) + head, word = os.path.split(word) + if word in (os.curdir, os.pardir): continue + path = os.path.join(path, word) + return path + + def log_message(self, *args): + """Don't log anything, unless you say so""" + if not self.silent: + SimpleHTTPRequestHandler.log_message(self, *args) + + def do_POST(self): + self.do_GET() + +class TestServer( threading.Thread ): # pragma: no cover + def __init__(self, daemonise=True, silent=True): + super(TestServer, self).__init__() + self.daemon = daemonise + self.silent = silent + self.http = None + # Try and get a free port number + sock = socket() + sock.bind(('', 0)) + self.port = sock.getsockname()[1] + sock.close() + def run(self): + protocol="HTTP/1.0" + server_address = ('', self.port) + + SilentRequestHandler.protocol_version = protocol + SilentRequestHandler.silent = self.silent + #if not self.silent: + # print "Starting", protocol, "server on port", self.port + self.http = HTTPServer(server_address, SilentRequestHandler) + self.http.serve_forever() + + def shutdown(self): + self.join() + +if __name__ == '__main__': # pragma: no cover + server = TestServer(silent=False) + server.start() + for number in range(1, 20): + time.sleep(2) diff --git a/tests/testservice/countservice/service/model b/tests/testservice/countservice/service/model new file mode 100644 index 00000000..81257f49 --- /dev/null +++ b/tests/testservice/countservice/service/model @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/countservice/service/query/results b/tests/testservice/countservice/service/query/results new file mode 100644 index 00000000..7273c0fa --- /dev/null +++ b/tests/testservice/countservice/service/query/results @@ -0,0 +1 @@ +25 diff --git a/tests/testservice/countservice/service/template/results b/tests/testservice/countservice/service/template/results new file mode 100644 index 00000000..7273c0fa --- /dev/null +++ b/tests/testservice/countservice/service/template/results @@ -0,0 +1 @@ +25 diff --git a/tests/testservice/countservice/service/templates/xml b/tests/testservice/countservice/service/templates/xml new file mode 100644 index 00000000..e348cc77 --- /dev/null +++ b/tests/testservice/countservice/service/templates/xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/countservice/service/version/release b/tests/testservice/countservice/service/version/release new file mode 100644 index 00000000..b7d6715e --- /dev/null +++ b/tests/testservice/countservice/service/version/release @@ -0,0 +1 @@ +FOO diff --git a/tests/testservice/countservice/service/version/ws b/tests/testservice/countservice/service/version/ws new file mode 100644 index 00000000..29d6383b --- /dev/null +++ b/tests/testservice/countservice/service/version/ws @@ -0,0 +1 @@ +100 diff --git a/tests/testservice/csvservice/service/model b/tests/testservice/csvservice/service/model new file mode 100644 index 00000000..81257f49 --- /dev/null +++ b/tests/testservice/csvservice/service/model @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/csvservice/service/query/results b/tests/testservice/csvservice/service/query/results new file mode 100644 index 00000000..339c7add --- /dev/null +++ b/tests/testservice/csvservice/service/query/results @@ -0,0 +1,2 @@ +"foo","bar","baz" +"123","1.23","-1.23" diff --git a/tests/testservice/csvservice/service/template/results b/tests/testservice/csvservice/service/template/results new file mode 100644 index 00000000..339c7add --- /dev/null +++ b/tests/testservice/csvservice/service/template/results @@ -0,0 +1,2 @@ +"foo","bar","baz" +"123","1.23","-1.23" diff --git a/tests/testservice/csvservice/service/templates/xml b/tests/testservice/csvservice/service/templates/xml new file mode 100644 index 00000000..e348cc77 --- /dev/null +++ b/tests/testservice/csvservice/service/templates/xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/csvservice/service/version/release b/tests/testservice/csvservice/service/version/release new file mode 100644 index 00000000..b7d6715e --- /dev/null +++ b/tests/testservice/csvservice/service/version/release @@ -0,0 +1 @@ +FOO diff --git a/tests/testservice/csvservice/service/version/ws b/tests/testservice/csvservice/service/version/ws new file mode 100644 index 00000000..29d6383b --- /dev/null +++ b/tests/testservice/csvservice/service/version/ws @@ -0,0 +1 @@ +100 diff --git a/tests/testservice/legacyjsonrows/lists/json b/tests/testservice/legacyjsonrows/lists/json new file mode 100644 index 00000000..f265e144 --- /dev/null +++ b/tests/testservice/legacyjsonrows/lists/json @@ -0,0 +1,29 @@ +{"lists":[ + { + "name":"test-list-1", + "title":"test1", + "description":"An example test list", + "type": "Employee", + "size": 42, + "dateCreated": "2011-05-07T19:52:03", + "authorized": true, + "tags": ["tag1", "tag2", "tag3"] + }, + { + "name":"test-list-2", + "title":"test2", + "description":"Another example test list", + "type": "Manager", + "size": 7, + "dateCreated": "2011-05-07T10:11:12", + "authorized": false + }, + { + "name":"test-list-3", + "title":"test3", + "description":"Yet Another example test list", + "type": "CEO", + "size": 8, + "dateCreated": "2011-05-07T10:11:12" + } +],"wasSuccessful":true,"error":null,"statusCode":200} diff --git a/tests/testservice/legacyjsonrows/model b/tests/testservice/legacyjsonrows/model new file mode 100644 index 00000000..81257f49 --- /dev/null +++ b/tests/testservice/legacyjsonrows/model @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/legacyjsonrows/query/results b/tests/testservice/legacyjsonrows/query/results new file mode 100644 index 00000000..9a02bb5d --- /dev/null +++ b/tests/testservice/legacyjsonrows/query/results @@ -0,0 +1,5 @@ +{"results":[ +[{"value":"foo","url":"/some/path/foo"},{"value":"bar","url":"/some/path/bar"},{"value":"baz","url":"/some/path/baz"}], +[{"value":123,"url":"/some/path/123"},{"value":1.23,"url":"/some/path/1.23"},{"value":-1.23,"url":"/some/path/-1.23"}], +[{"value":true,"url":"/some/path/123"},{"value":false,"url":"/some/path/1.23"},{"value":null,"url":"/some/path/-1.23"}] +],"executionTime":"2011.04.02 14:41::10","wasSuccessful":true,"error":null,"statusCode":200} diff --git a/tests/testservice/legacyjsonrows/template/results b/tests/testservice/legacyjsonrows/template/results new file mode 100644 index 00000000..9a02bb5d --- /dev/null +++ b/tests/testservice/legacyjsonrows/template/results @@ -0,0 +1,5 @@ +{"results":[ +[{"value":"foo","url":"/some/path/foo"},{"value":"bar","url":"/some/path/bar"},{"value":"baz","url":"/some/path/baz"}], +[{"value":123,"url":"/some/path/123"},{"value":1.23,"url":"/some/path/1.23"},{"value":-1.23,"url":"/some/path/-1.23"}], +[{"value":true,"url":"/some/path/123"},{"value":false,"url":"/some/path/1.23"},{"value":null,"url":"/some/path/-1.23"}] +],"executionTime":"2011.04.02 14:41::10","wasSuccessful":true,"error":null,"statusCode":200} diff --git a/tests/testservice/legacyjsonrows/templates/xml b/tests/testservice/legacyjsonrows/templates/xml new file mode 100644 index 00000000..e348cc77 --- /dev/null +++ b/tests/testservice/legacyjsonrows/templates/xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/legacyjsonrows/version/release b/tests/testservice/legacyjsonrows/version/release new file mode 100644 index 00000000..b7d6715e --- /dev/null +++ b/tests/testservice/legacyjsonrows/version/release @@ -0,0 +1 @@ +FOO diff --git a/tests/testservice/legacyjsonrows/version/ws b/tests/testservice/legacyjsonrows/version/ws new file mode 100644 index 00000000..7f8f011e --- /dev/null +++ b/tests/testservice/legacyjsonrows/version/ws @@ -0,0 +1 @@ +7 diff --git a/tests/testservice/service/lists/json b/tests/testservice/service/lists/json new file mode 100644 index 00000000..f265e144 --- /dev/null +++ b/tests/testservice/service/lists/json @@ -0,0 +1,29 @@ +{"lists":[ + { + "name":"test-list-1", + "title":"test1", + "description":"An example test list", + "type": "Employee", + "size": 42, + "dateCreated": "2011-05-07T19:52:03", + "authorized": true, + "tags": ["tag1", "tag2", "tag3"] + }, + { + "name":"test-list-2", + "title":"test2", + "description":"Another example test list", + "type": "Manager", + "size": 7, + "dateCreated": "2011-05-07T10:11:12", + "authorized": false + }, + { + "name":"test-list-3", + "title":"test3", + "description":"Yet Another example test list", + "type": "CEO", + "size": 8, + "dateCreated": "2011-05-07T10:11:12" + } +],"wasSuccessful":true,"error":null,"statusCode":200} diff --git a/tests/testservice/service/model b/tests/testservice/service/model new file mode 100644 index 00000000..81257f49 --- /dev/null +++ b/tests/testservice/service/model @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/service/query/results b/tests/testservice/service/query/results new file mode 100644 index 00000000..ab232140 --- /dev/null +++ b/tests/testservice/service/query/results @@ -0,0 +1,5 @@ +{"results":[ +["foo","bar","baz"], +[123,1.23,-1.23], +[true,false,null] +],"executionTime":"2011.04.02 14:41::10","wasSuccessful":true,"error":null,"statusCode":200} diff --git a/tests/testservice/service/template/results b/tests/testservice/service/template/results new file mode 100644 index 00000000..ab232140 --- /dev/null +++ b/tests/testservice/service/template/results @@ -0,0 +1,5 @@ +{"results":[ +["foo","bar","baz"], +[123,1.23,-1.23], +[true,false,null] +],"executionTime":"2011.04.02 14:41::10","wasSuccessful":true,"error":null,"statusCode":200} diff --git a/tests/testservice/service/templates/xml b/tests/testservice/service/templates/xml new file mode 100644 index 00000000..4cb92d41 --- /dev/null +++ b/tests/testservice/service/templates/xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/service/version/release b/tests/testservice/service/version/release new file mode 100644 index 00000000..b7d6715e --- /dev/null +++ b/tests/testservice/service/version/release @@ -0,0 +1 @@ +FOO diff --git a/tests/testservice/service/version/ws b/tests/testservice/service/version/ws new file mode 100644 index 00000000..45a4fb75 --- /dev/null +++ b/tests/testservice/service/version/ws @@ -0,0 +1 @@ +8 diff --git a/tests/testservice/testresultobjs/service/model b/tests/testservice/testresultobjs/service/model new file mode 100644 index 00000000..81257f49 --- /dev/null +++ b/tests/testservice/testresultobjs/service/model @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/testresultobjs/service/query/results b/tests/testservice/testresultobjs/service/query/results new file mode 100644 index 00000000..e6262286 --- /dev/null +++ b/tests/testservice/testresultobjs/service/query/results @@ -0,0 +1,10 @@ +{"rootClass":"Department","modelName":"testmodel","start":0,"views":["Department.name","Department.company.vatNumber","Department.employees.age","Department.employees.name"],"results":[ +{"objectId":3000008,"name":"Sales","company":{"objectId":3000004,"class":"Company","vatNumber":665261},"class":"Department","employees":[{"objectId":3000027,"age":35,"name":"Rachel","class":"Employee"},{"objectId":3000019,"age":36,"name":"Ricky","class":"Employee"},{"objectId":3000035,"age":46,"name":"Tim Canterbury","class":"Employee"},{"objectId":3000045,"age":58,"name":"Malcolm","class":"Employee"},{"objectId":3000007,"age":62,"name":"David Brent","class":"Manager"},{"objectId":3000041,"age":64,"name":"Gareth Keenan","class":"Employee"}]}, +{"objectId":3000143,"name":"Sales","company":{"objectId":3000139,"class":"Company","vatNumber":701669},"class":"Department","employees":[{"objectId":3000170,"age":26,"name":"Andy Bernard","class":"Employee"},{"objectId":3000162,"age":30,"name":"Dwight Schrute","class":"Employee"},{"objectId":3000154,"age":34,"name":"Jim Halpert","class":"Employee"},{"objectId":3000178,"age":34,"name":"Stanley Hudson","class":"Employee"},{"objectId":3000142,"age":36,"name":"Michael Scott","class":"Manager"},{"objectId":3000186,"age":58,"name":"Phyllis Lapin-Vance","class":"Employee"}]}, +{"objectId":3000202,"name":"Sales","company":{"objectId":3000198,"class":"Company","vatNumber":831757},"class":"Department","employees":[{"objectId":3000213,"age":31,"name":"Jennifer","class":"Employee"},{"objectId":3000229,"age":32,"name":"Patrick","class":"Employee"},{"objectId":3000201,"age":38,"name":"Gilles Triquet","class":"Manager"},{"objectId":3000221,"age":51,"name":"Delphine","class":"Employee"},{"objectId":3000237,"age":54,"name":"Nicole","class":"Employee"},{"objectId":3000243,"age":64,"name":"Fatou","class":"Employee"}]}, +{"objectId":3000063,"name":"Schadensregulierung","company":{"objectId":3000053,"class":"Company","vatNumber":321941},"class":"Department","employees":[{"objectId":3000102,"age":37,"name":"Josef Müller","class":"Employee"},{"objectId":3000062,"age":45,"name":"Timo Becker","class":"Manager"},{"objectId":3000114,"age":55,"name":"Rita Klüver","class":"Employee"},{"objectId":3000090,"age":58,"name":"Suzanne Landsfried","class":"Employee"},{"objectId":3000078,"age":61,"name":"Maja Decker","class":"Employee"},{"objectId":3000126,"age":63,"name":"Andreas Hermann","class":"Employee"}]}, +{"objectId":3000057,"name":"Schadensregulierung A-L","company":{"objectId":3000053,"class":"Company","vatNumber":321941},"class":"Department","employees":[{"objectId":3000098,"age":31,"name":"Ulf Steinke","class":"Employee"},{"objectId":3000074,"age":33,"name":"Jennifer Schirrmann","class":"Employee"},{"objectId":3000122,"age":35,"name":"Erika Burstedt","class":"Employee"},{"objectId":3000110,"age":45,"name":"Berthold Heisterkamp","class":"Employee"},{"objectId":3000086,"age":48,"name":"Tanja Seifert","class":"Employee"},{"objectId":3000056,"age":59,"name":"Sinan Turçulu","class":"Manager"}]}, +{"objectId":3000060,"name":"Schadensregulierung M-Z","company":{"objectId":3000053,"class":"Company","vatNumber":321941},"class":"Department","employees":[{"objectId":3000124,"age":28,"name":"Frank Montenbruck","class":"Employee"},{"objectId":3000112,"age":28,"name":"Nicole Rückert","class":"Employee"},{"objectId":3000088,"age":29,"name":"Herr Pötsch","class":"Employee"},{"objectId":3000059,"age":42,"name":"Bernd Stromberg","class":"Manager"},{"objectId":3000100,"age":46,"name":"Lars Lehnhoff","class":"Employee"},{"objectId":3000076,"age":48,"name":"Sabine Buhrer","class":"Employee"}]}, +{"objectId":3000258,"name":"Separators","company":{"objectId":3000251,"class":"Company","vatNumber":764575},"class":"Department","employees":[{"objectId":3000268,"age":28,"name":"Comma , here","class":"Employee"},{"objectId":3000284,"age":32,"name":"New line\nhere","class":"Employee"},{"objectId":3000292,"age":41,"name":"Double backwards slash \\","class":"Employee"},{"objectId":3000276,"age":54,"name":"Tab\there","class":"Employee"},{"objectId":3000257,"age":55,"name":"Separator Leader","class":"Manager"}]}, +{"objectId":3000261,"name":"Slashes","company":{"objectId":3000251,"class":"Company","vatNumber":764575},"class":"Department","employees":[{"objectId":3000294,"age":49,"name":"Quot \"","class":"Employee"},{"objectId":3000270,"age":51,"name":"Forward Slash /","class":"Employee"},{"objectId":3000278,"age":62,"name":"Double forward Slash //","class":"Employee"},{"objectId":3000286,"age":63,"name":"Backwards Slash \\'","class":"Employee"},{"objectId":3000260,"age":64,"name":"Slash Leader","class":"Manager"}]} +],"executionTime":"2011.07.19 13:04::35","wasSuccessful":true,"error":null,"statusCode":200} \ No newline at end of file diff --git a/tests/testservice/testresultobjs/service/template/results b/tests/testservice/testresultobjs/service/template/results new file mode 100644 index 00000000..e6262286 --- /dev/null +++ b/tests/testservice/testresultobjs/service/template/results @@ -0,0 +1,10 @@ +{"rootClass":"Department","modelName":"testmodel","start":0,"views":["Department.name","Department.company.vatNumber","Department.employees.age","Department.employees.name"],"results":[ +{"objectId":3000008,"name":"Sales","company":{"objectId":3000004,"class":"Company","vatNumber":665261},"class":"Department","employees":[{"objectId":3000027,"age":35,"name":"Rachel","class":"Employee"},{"objectId":3000019,"age":36,"name":"Ricky","class":"Employee"},{"objectId":3000035,"age":46,"name":"Tim Canterbury","class":"Employee"},{"objectId":3000045,"age":58,"name":"Malcolm","class":"Employee"},{"objectId":3000007,"age":62,"name":"David Brent","class":"Manager"},{"objectId":3000041,"age":64,"name":"Gareth Keenan","class":"Employee"}]}, +{"objectId":3000143,"name":"Sales","company":{"objectId":3000139,"class":"Company","vatNumber":701669},"class":"Department","employees":[{"objectId":3000170,"age":26,"name":"Andy Bernard","class":"Employee"},{"objectId":3000162,"age":30,"name":"Dwight Schrute","class":"Employee"},{"objectId":3000154,"age":34,"name":"Jim Halpert","class":"Employee"},{"objectId":3000178,"age":34,"name":"Stanley Hudson","class":"Employee"},{"objectId":3000142,"age":36,"name":"Michael Scott","class":"Manager"},{"objectId":3000186,"age":58,"name":"Phyllis Lapin-Vance","class":"Employee"}]}, +{"objectId":3000202,"name":"Sales","company":{"objectId":3000198,"class":"Company","vatNumber":831757},"class":"Department","employees":[{"objectId":3000213,"age":31,"name":"Jennifer","class":"Employee"},{"objectId":3000229,"age":32,"name":"Patrick","class":"Employee"},{"objectId":3000201,"age":38,"name":"Gilles Triquet","class":"Manager"},{"objectId":3000221,"age":51,"name":"Delphine","class":"Employee"},{"objectId":3000237,"age":54,"name":"Nicole","class":"Employee"},{"objectId":3000243,"age":64,"name":"Fatou","class":"Employee"}]}, +{"objectId":3000063,"name":"Schadensregulierung","company":{"objectId":3000053,"class":"Company","vatNumber":321941},"class":"Department","employees":[{"objectId":3000102,"age":37,"name":"Josef Müller","class":"Employee"},{"objectId":3000062,"age":45,"name":"Timo Becker","class":"Manager"},{"objectId":3000114,"age":55,"name":"Rita Klüver","class":"Employee"},{"objectId":3000090,"age":58,"name":"Suzanne Landsfried","class":"Employee"},{"objectId":3000078,"age":61,"name":"Maja Decker","class":"Employee"},{"objectId":3000126,"age":63,"name":"Andreas Hermann","class":"Employee"}]}, +{"objectId":3000057,"name":"Schadensregulierung A-L","company":{"objectId":3000053,"class":"Company","vatNumber":321941},"class":"Department","employees":[{"objectId":3000098,"age":31,"name":"Ulf Steinke","class":"Employee"},{"objectId":3000074,"age":33,"name":"Jennifer Schirrmann","class":"Employee"},{"objectId":3000122,"age":35,"name":"Erika Burstedt","class":"Employee"},{"objectId":3000110,"age":45,"name":"Berthold Heisterkamp","class":"Employee"},{"objectId":3000086,"age":48,"name":"Tanja Seifert","class":"Employee"},{"objectId":3000056,"age":59,"name":"Sinan Turçulu","class":"Manager"}]}, +{"objectId":3000060,"name":"Schadensregulierung M-Z","company":{"objectId":3000053,"class":"Company","vatNumber":321941},"class":"Department","employees":[{"objectId":3000124,"age":28,"name":"Frank Montenbruck","class":"Employee"},{"objectId":3000112,"age":28,"name":"Nicole Rückert","class":"Employee"},{"objectId":3000088,"age":29,"name":"Herr Pötsch","class":"Employee"},{"objectId":3000059,"age":42,"name":"Bernd Stromberg","class":"Manager"},{"objectId":3000100,"age":46,"name":"Lars Lehnhoff","class":"Employee"},{"objectId":3000076,"age":48,"name":"Sabine Buhrer","class":"Employee"}]}, +{"objectId":3000258,"name":"Separators","company":{"objectId":3000251,"class":"Company","vatNumber":764575},"class":"Department","employees":[{"objectId":3000268,"age":28,"name":"Comma , here","class":"Employee"},{"objectId":3000284,"age":32,"name":"New line\nhere","class":"Employee"},{"objectId":3000292,"age":41,"name":"Double backwards slash \\","class":"Employee"},{"objectId":3000276,"age":54,"name":"Tab\there","class":"Employee"},{"objectId":3000257,"age":55,"name":"Separator Leader","class":"Manager"}]}, +{"objectId":3000261,"name":"Slashes","company":{"objectId":3000251,"class":"Company","vatNumber":764575},"class":"Department","employees":[{"objectId":3000294,"age":49,"name":"Quot \"","class":"Employee"},{"objectId":3000270,"age":51,"name":"Forward Slash /","class":"Employee"},{"objectId":3000278,"age":62,"name":"Double forward Slash //","class":"Employee"},{"objectId":3000286,"age":63,"name":"Backwards Slash \\'","class":"Employee"},{"objectId":3000260,"age":64,"name":"Slash Leader","class":"Manager"}]} +],"executionTime":"2011.07.19 13:04::35","wasSuccessful":true,"error":null,"statusCode":200} \ No newline at end of file diff --git a/tests/testservice/testresultobjs/service/version/release b/tests/testservice/testresultobjs/service/version/release new file mode 100644 index 00000000..b7d6715e --- /dev/null +++ b/tests/testservice/testresultobjs/service/version/release @@ -0,0 +1 @@ +FOO diff --git a/tests/testservice/testresultobjs/service/version/ws b/tests/testservice/testresultobjs/service/version/ws new file mode 100644 index 00000000..29d6383b --- /dev/null +++ b/tests/testservice/testresultobjs/service/version/ws @@ -0,0 +1 @@ +100 diff --git a/tests/testservice/tsvservice/service/model b/tests/testservice/tsvservice/service/model new file mode 100644 index 00000000..81257f49 --- /dev/null +++ b/tests/testservice/tsvservice/service/model @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/tsvservice/service/query/results b/tests/testservice/tsvservice/service/query/results new file mode 100644 index 00000000..5fc4f815 --- /dev/null +++ b/tests/testservice/tsvservice/service/query/results @@ -0,0 +1,2 @@ +foo bar baz +123 1.23 -1.23 diff --git a/tests/testservice/tsvservice/service/template/results b/tests/testservice/tsvservice/service/template/results new file mode 100644 index 00000000..5fc4f815 --- /dev/null +++ b/tests/testservice/tsvservice/service/template/results @@ -0,0 +1,2 @@ +foo bar baz +123 1.23 -1.23 diff --git a/tests/testservice/tsvservice/service/templates/xml b/tests/testservice/tsvservice/service/templates/xml new file mode 100644 index 00000000..e348cc77 --- /dev/null +++ b/tests/testservice/tsvservice/service/templates/xml @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/testservice/tsvservice/service/version/release b/tests/testservice/tsvservice/service/version/release new file mode 100644 index 00000000..b7d6715e --- /dev/null +++ b/tests/testservice/tsvservice/service/version/release @@ -0,0 +1 @@ +FOO diff --git a/tests/testservice/tsvservice/service/version/ws b/tests/testservice/tsvservice/service/version/ws new file mode 100644 index 00000000..29d6383b --- /dev/null +++ b/tests/testservice/tsvservice/service/version/ws @@ -0,0 +1 @@ +100 diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..54a91162 --- /dev/null +++ b/tox.ini @@ -0,0 +1,24 @@ +[tox] +envlist = + py25, py26, py27, pypy, py32 + +[testenv] +commands = + python setup.py test + python setup.py livetest + +[testenv:py25] +deps = + simplejson + +[testenv:py24] +deps = + simplejson + +[testenv:jython] +basepython=jython +deps = + simplejson +commands = + jython tests/live_lists.py + jython tests/live_summary_test.py