From 9aae9a1905cb9064920b5fa237d34c3b4fb5aa58 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 10 Oct 2023 12:34:21 -0400 Subject: [PATCH] DEVEXP-589 Added eval docs Tossed in a couple more test cases. Curiously could not find a way to return a "binary()" with it being associated with a URI. --- docs/eval.md | 93 +++++++++++++++++++ docs/example-setup.md | 24 +++-- docs/transactions.md | 2 +- marklogic/client.py | 15 +-- .../security/roles/python-docs-role.json | 27 ++++++ .../security/users/python-docs-user.json | 4 +- tests/test_eval.py | 14 +++ 7 files changed, 161 insertions(+), 18 deletions(-) create mode 100644 docs/eval.md create mode 100644 test-app/src/main/ml-config/security/roles/python-docs-role.json diff --git a/docs/eval.md b/docs/eval.md new file mode 100644 index 0000000..7f3a99d --- /dev/null +++ b/docs/eval.md @@ -0,0 +1,93 @@ +--- +layout: default +title: Executing code +nav_order: 5 +--- + +The [MarkLogic REST service extension](https://docs.marklogic.com/REST/client/service-extension) supports the +execution of custom code, whether via an inline script or an existing module in your application's modules database. +The MarkLogic Python client simplifies execution of custom code both by managing some of the complexity of submitting +and custom code and also converting the multipart response into more useful Python data types. + +## Setup + +The examples below all depend on the instructions in the [setup guide](example-setup.md) having already been performed. + +To try out the examples, start a Python shell and first run the following: + +``` +from marklogic import Client +client = Client('http://localhost:8000', digest=('python-user', 'pyth0n')) +``` + +## Executing ad-hoc queries + +The [v1/eval REST endpoint](https://docs.marklogic.com/REST/POST/v1/eval) supports the execution of ad-hoc JavaScript +and XQuery queries. Each type of query can be easily submitted via the client: + +``` +client.eval.javascript("fn.currentDateTime()") +client.eval.xquery("fn:current-dateTime()") +``` + +Variables can optionally be provided via a `dict`: + +``` +results = client.eval.javascript('Sequence.from([{"hello": myValue}])', vars={"myValue": "world"}) +assert "world" == results[0]["hello"] +``` + +Because the REST endpoint returns a sequence of items, the client will always return a list of values. See the section +below on how data types are converted to understand how the client will convert each value into an appropriate Python +data type. + +## Invoking modules + +The [v1/invoke REST endpoint](https://docs.marklogic.com/REST/POST/v1/invoke) supports the execution of JavaScript +and XQuery main modules that have been deployed to your application's modules database. A module can be invoked via +the client in the following manner: + +``` +client.invoke("/path/to/module.sjs") +``` + + +## Conversion of data types + +The REST endpoints for evaluating ad-hoc code and for invoking a module both return a sequence of values, with each +value having MarkLogic-specific type information. The client will use this type information to convert each value into +an appropriate Python data type. For example, each JSON object into the example below is converted into a `dict`: + +``` +results = client.eval.javascript('Sequence.from([{"doc": 1}, {"doc": 2}])') +assert len(results) == 2 +assert results[0]["doc"] == 1 +assert results[1]["doc"] == 2 +``` + +The following table describes how each MarkLogic type is associated with a Python data type. For any +MarkLogic type not listed in the table, the value is not converted and will be of type `bytes`. + +| MarkLogic type | Python type | +| --- | --- | +| string | str | +| integer | int | +| boolean | bool | +| decimal | [Decimal](https://docs.python.org/3/library/decimal.html) | +| map | dict | +| element() | str | +| array | list | +| array-node() | list | +| object-node() | dict or marklogic.documents.Document | +| document-node() | str or marklogic.documents.Document | +| binary() | marklogic.documents.Document | + +For the `object-node()` and `document-node()` entries in the above table, a `marklogic.documents.Document` instance +will be returned if the MarkLogic value is associated with a URI via the multipart `X-URI` header. Otherwise, a `dict` +or `str` is returned respectively. + +## Returning the original HTTP response + +Each `client.eval` method and `client.invoke` accept a `return_response` argument. When that +argument is set to `True`, the original response is returned. This can be useful for custom +processing of the response or debugging requests. diff --git a/docs/example-setup.md b/docs/example-setup.md index d9b6126..23ebc0f 100644 --- a/docs/example-setup.md +++ b/docs/example-setup.md @@ -5,19 +5,29 @@ nav_order: 2 permalink: /setup --- -The examples in this documentation depend on a particular MarkLogic username and password. If you -would like to try these examples out against your own installation of MarkLogic, you will need to create this -MarkLogic user. To do so, please go to the Admin application for your MarkLogic instance - e.g. if you are running MarkLogic locally, this will be at - and authenticate as your "admin" user. -Then perform the following steps to create a new user: +The examples in this documentation depend on a particular MarkLogic user with a role containing specific privileges. +If you would like to try these examples out against your own installation of MarkLogic, you will need to create this +MarkLogic user and role. To do so, please go to the Admin application for your MarkLogic instance - e.g. if you are +running MarkLogic locally, this will be at - and authenticate as your "admin" user. +Then perform the following steps to create a new role: + +1. Click on "Roles" in the "Security" box. +2. Click on "Create". +3. In the form, enter "python-docs-role" for "Role Name". +4. Scroll down and select the "rest-extension-user", "rest-reader", "rest-writer", and "tde-admin" roles. +5. Scroll further down and select the "xdbc:eval", "xdbc:invoke", and "xdmp:eval-in" privileges. +6. Scroll to the top or bottom and click on "OK" to create the role. + +After creating the role, return to the Admin application home page and perform the following steps: 1. Click on "Users" in the "Security" box. 2. Click on "Create". 3. In the form, enter "python-user" for "User Name" and "pyth0n" as the password. -4. Scroll down until you see the "Roles" section. Click on the "rest-reader", "rest-writer", and "security" checkboxes. +4. Scroll down until you see the "Roles" section and select the "python-docs-role" role. 5. Scroll to the top or bottom and click on "OK" to create the user. -(The `security` role is only needed to allow for the user to load documents into the out-of-the-box Schemas database -in MarkLogic; in a production application, an admin or admin-like user would typically be used for this use case.) +(Note that you could use the `admin` role instead to grant full access to all features in MarkLogic, but this is +generally discouraged for security reasons.) You can verify that you correctly created the user by accessing the REST API for the out-of-the-box REST API server in MarkLogic that listens on port 8000. Go to (changing "localhost" to diff --git a/docs/transactions.md b/docs/transactions.md index 1324d9b..ecdd1b6 100644 --- a/docs/transactions.md +++ b/docs/transactions.md @@ -1,7 +1,7 @@ --- layout: default title: Managing transactions -nav_order: 5 +nav_order: 6 --- The [MarkLogic REST transactions service](https://docs.marklogic.com/REST/client/transaction-management) diff --git a/marklogic/client.py b/marklogic/client.py index c446147..df803b7 100644 --- a/marklogic/client.py +++ b/marklogic/client.py @@ -112,16 +112,14 @@ def process_multipart_mixed_response(self, response): transformed_parts = [] for part in parts: encoding = part.encoding - primitive_header = part.headers["X-Primitive".encode(encoding)].decode( - encoding - ) - primitive_function = Client.__primitive_value_converters.get( - primitive_header - ) + header = part.headers["X-Primitive".encode(encoding)].decode(encoding) + primitive_function = Client.__primitive_value_converters.get(header) if primitive_function is not None: transformed_parts.append(primitive_function(part)) else: - transformed_parts.append(part.text) + # Return the binary created by requests_toolbelt so we don't get an + # error trying to convert it to something else. + transformed_parts.append(part.content) return transformed_parts @property @@ -159,6 +157,9 @@ def eval(self): "array-node()": lambda part: json.loads(part.text), "object-node()": lambda part: Client.__process_object_node_part(part), "document-node()": lambda part: Client.__process_document_node_part(part), + # It appears that binary() will only be returned for a binary node retrieved + # from the database, and thus an X-URI will always exist. Have not found a + # scenario that indicates otherwise. "binary()": lambda part: Document( Client.__get_decoded_uri_from_part(part), part.content ), diff --git a/test-app/src/main/ml-config/security/roles/python-docs-role.json b/test-app/src/main/ml-config/security/roles/python-docs-role.json new file mode 100644 index 0000000..e99583f --- /dev/null +++ b/test-app/src/main/ml-config/security/roles/python-docs-role.json @@ -0,0 +1,27 @@ +{ + "role-name": "python-docs-role", + "description": "Used by the documentation, not by tests", + "role": [ + "rest-extension-user", + "rest-reader", + "rest-writer", + "tde-admin" + ], + "privilege": [ + { + "privilege-name": "xdbc:eval", + "action": "http://marklogic.com/xdmp/privileges/xdbc-eval", + "kind": "execute" + }, + { + "privilege-name": "xdbc:invoke", + "action": "http://marklogic.com/xdmp/privileges/xdbc-invoke", + "kind": "execute" + }, + { + "privilege-name": "xdmp:eval-in", + "action": "http://marklogic.com/xdmp/privileges/xdmp-eval-in", + "kind": "execute" + } + ] +} diff --git a/test-app/src/main/ml-config/security/users/python-docs-user.json b/test-app/src/main/ml-config/security/users/python-docs-user.json index 562107e..906073f 100644 --- a/test-app/src/main/ml-config/security/users/python-docs-user.json +++ b/test-app/src/main/ml-config/security/users/python-docs-user.json @@ -3,8 +3,6 @@ "password": "pyth0n", "description": "Used by the documentation, not by tests", "role": [ - "rest-reader", - "rest-writer", - "security" + "python-docs-role" ] } \ No newline at end of file diff --git a/tests/test_eval.py b/tests/test_eval.py index 67ec3a6..7ef936c 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -108,6 +108,20 @@ def test_javascript_script(client): assert [[]] == parts +def test_base64Binary(client): + parts = client.eval.xquery('xs:base64Binary(doc("/musicians/logo.png"))') + assert len(parts) == 1 + assert type(parts[0]) is bytes + + +def test_hexBinary(client): + # No idea what this value is, found it in a DHF test. + b = "3f3c6d78206c657673726f693d6e3122302e20226e656f636964676e223d54552d4622383e3f" + parts = client.eval.xquery(f"xs:hexBinary('{b}')") + assert len(parts) == 1 + assert type(parts[0]) is bytes + + def __verify_common_primitives(parts): assert type(parts[0]) is str assert "A" == parts[0]