diff --git a/.travis.yml b/.travis.yml index 3145629a37..59473b40a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,6 +80,7 @@ before_install: install: - "./scripts/bootstrap --ci $REQUIREMENTS" + - "pipenv run pip list" # To have details about installed libs (isort, pytest, etc.) before_script: # https://docs.travis-ci.com/user/gui-and-headless-browsers/ diff --git a/Pipfile b/Pipfile index 677a6ccc65..d90878ee2f 100644 --- a/Pipfile +++ b/Pipfile @@ -37,28 +37,25 @@ name = "pypi" Babel = ">=2.4.0" Flask-BabelEx = ">=0.9.3" ## Third party invenio modules used by RERO ILS -invenio-oaiharvester = {editable = true, git = "https://github.com/inveniosoftware/invenio-oaiharvester.git", ref = "v1.0.0a4"} -invenio-circulation = {editable = true, git = "https://github.com/inveniosoftware/invenio-circulation.git", ref = "v1.0.0a16"} -## Invenio base modules used by RERO ILS +invenio-oaiharvester = {editable = true, ref = "v1.0.0a4", git = "https://github.com/inveniosoftware/invenio-oaiharvester.git"} +invenio-circulation = {editable = true, ref = "v1.0.0a16", git = "https://github.com/inveniosoftware/invenio-circulation.git"} +## Invenio 3.2.1 base modules used by RERO ILS # same as invenio metadata extras without invenio-search-ui -invenio-indexer = ">=1.0.1,<1.1.0" -invenio-jsonschemas = ">=1.0.0,<1.1.0" -invenio-oaiserver = ">=1.0.3,<1.1.0" -invenio-pidstore = ">=1.0.0,<1.1.0" -invenio-records-rest = ">=1.4.0,<1.5.0" +invenio-indexer = ">=1.1.1,<1.2.0" +invenio-jsonschemas = ">=1.0.1,<1.1.0" +invenio-oaiserver = ">=1.1.1,<1.2.0" +invenio-pidstore = ">=1.1.0,<1.2.0" +invenio-records-rest = ">=1.6.4,<1.7.0" invenio-records-ui = ">=1.0.1,<1.1.0" +invenio-records = ">=1.3.0,<1.4.0" ## Default from Invenio -invenio = {version = "~=3.1.0",extras = ["base", "postgresql", "auth", "elasticsearch6"]} +invenio = {version = "==3.2.1", extras = ["base", "postgresql", "auth", "elasticsearch6" ]} uwsgi = ">=2.0" uwsgitop = ">=0.11" uwsgi-tools = ">=1.1.1" ## Additionnal constraints on invenio base modules # TODO: remove when https://github.com/inveniosoftware/invenio-accounts/issues/306 will be solved invenio-accounts = "<=1.1.2" -# TODO: remove when DynamicPermission will be solved -invenio-access = "<1.2.0" -# separate tables -invenio-records = ">=1.3.0" # TODO: remove when sqlalchemy.exc.NoInspectionAvailable problem will be solved SQLAlchemy = "<=1.3.15" # TODO: remove when email_validator Exception disappears @@ -96,14 +93,14 @@ Flask-Debugtoolbar = ">=0.10.1" # https://github.com/mozilla-services/python-dockerflow/issues/42 Sphinx = "<3.0.2" check-manifest = ">=0.35" -coverage = ">=4.4.1" +coverage = ">=4.5.3" isort = ">=4.3" mock = ">=2.0.0" marshmallow = ">=2.15.1,<3.0.0" -pydocstyle = ">=2.0.0" -pytest = ">=3.3.1" -pytest-cov = ">=2.5.1" -pytest-invenio = ">=1.0.2,<1.1.0" +pydocstyle = ">=3.0.0" +pytest = ">=4.6.4" +pytest-cov = ">=2.7.1" +pytest-invenio = ">=1.2.1,<1.3.0" pytest-mock = ">=1.6.0" pytest-pep8 = ">=1.0.6" pytest-random-order = ">=0.5.4" diff --git a/Pipfile.lock b/Pipfile.lock index 91e9b88ccb..632986736a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -65,18 +65,18 @@ ], "version": "==0.1.0" }, - "billiard": { + "base32-lib": { "hashes": [ - "sha256:42d9a227401ac4fba892918bba0a0c409def5435c4b483267ebfe821afaaba0e" + "sha256:470d1f318a3632397d091bb2773500e484489a473f2001df5c0675e72730ddd3", + "sha256:ebbf80687ef9791580135f372db1c0a09a1a5d02089d57a8a49bf5ef3330221f" ], - "version": "==3.5.0.5" + "version": "==1.0.1" }, - "binaryornot": { + "billiard": { "hashes": [ - "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061", - "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4" + "sha256:42d9a227401ac4fba892918bba0a0c409def5435c4b483267ebfe821afaaba0e" ], - "version": "==0.4.4" + "version": "==3.5.0.5" }, "bleach": { "hashes": [ @@ -160,13 +160,6 @@ ], "version": "==7.1.1" }, - "cookiecutter": { - "hashes": [ - "sha256:430eb882d028afb6102c084bab6cf41f6559a77ce9b18dc6802e3bc0cc5f4a30", - "sha256:efb6b2d4780feda8908a873e38f0e61778c23f6a2ea58215723bcceb5b515dac" - ], - "version": "==1.7.2" - }, "cryptography": { "hashes": [ "sha256:01706e83707767a6c73aec2905f7f01a6b0f28953274262c7257244e1b6e6b41", @@ -319,10 +312,10 @@ }, "flask-limiter": { "hashes": [ - "sha256:d984a57ef37acb6eee29edc864ff22cd4cf090845f06968c015093ffd91e96f1", - "sha256:db2a069402977927282b0fcf650753bfcb50488028def9f5b2398e1d525f2f9f" + "sha256:905c35cd87bf60c92fd87922ae23fe27aa5fb31980bab31fc00807adee9f5a55", + "sha256:9087984ae7eeb862f93bf5b18477a5e5b1e0c907647ae74fba1c7e3f1de63d6f" ], - "version": "==1.2.1" + "version": "==1.1.0" }, "flask-login": { "hashes": [ @@ -423,10 +416,10 @@ }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", + "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" ], - "version": "==2.7" + "version": "==2.9" }, "importlib-metadata": { "hashes": [ @@ -450,19 +443,18 @@ }, "invenio": { "hashes": [ - "sha256:d6be8948deea14bc06b29051d3bf192b4d9b120cc47bbfed7aba1a83fc25e409", - "sha256:f0128f92e36d43a6d5b2f0154a37db3f51536205229ef689dcb43a87e3299b38" + "sha256:ba5badc4635670348df31610af028d98626b8bb62e2dc306de0241def79ac8cd", + "sha256:deff48c8f390182c348edcfaac96452b75e0b9a000875d3bfe4a0311fc7d2f98" ], "index": "pypi", - "version": "==3.1.2" + "version": "==3.2.1" }, "invenio-access": { "hashes": [ - "sha256:1e7439bfdb1c9a697fa41dccc79969af445baa0ddb6248f9cadc1046a2bed36f", - "sha256:430225aa095e4ad11401a84fdf449566498f0e69b8b5c74e77239c1446b9e4da" + "sha256:5bb743c0bf269c6fd5542ccc01a6c0987a496910aa0f4cf3e66523db962c61cc", + "sha256:8a7272447429c5d05ecebc332709a376d29117f1988827295d804bff6784021c" ], - "index": "pypi", - "version": "==1.1.0" + "version": "==1.3.1" }, "invenio-accounts": { "hashes": [ @@ -481,10 +473,10 @@ }, "invenio-app": { "hashes": [ - "sha256:a77aee57118d06909d2187a3e25f3d0a299189e06bb43b4d7404a689119ae75a", - "sha256:df15a9ef65758f82f75f8b392456793c2fc36e9984d15113c852a7d8fd2c52dd" + "sha256:2b850b112d14db1c2eeedfe95128ae9ce139097fbcaf06ff8c26dda6681aed58", + "sha256:e92f565a069a8ca93129ec1b7cab2cc6ca6b506491bbdb20499aab041f572107" ], - "version": "==1.1.1" + "version": "==1.2.5" }, "invenio-assets": { "hashes": [ @@ -495,10 +487,10 @@ }, "invenio-base": { "hashes": [ - "sha256:6e532854a43e4f54fc7d9b1cbda720ce4ce6383b00cee6ba5f7e86ef78f12fe2", - "sha256:f972e10b69a68a80c342f0bcf6b3b02a6a67af4fb6e8e347ce8b482fcb47e973" + "sha256:2b49e941d474fba3990543e0c231d865c75bb4b95c06b78c26231a030f17d8a8", + "sha256:9ae5d1db2865386ec755f39d8e0f4c0b318770af541b0807a02fad59882a038d" ], - "version": "==1.0.2" + "version": "==1.2.2" }, "invenio-cache": { "hashes": [ @@ -509,10 +501,10 @@ }, "invenio-celery": { "hashes": [ - "sha256:22cda704fa8abaa885a327539899887a939832beda7de0d1f1d5d84b4ac153b1", - "sha256:d102dcf3e94cfa11b0973626852f5fd6cac8fa3ba364c42fcec8ecc0b533d838" + "sha256:66160115ff49f625b4273e35c4b31599098dac255baa60c5a95e60b66edb22b6", + "sha256:808f244fa15465703436244a91ca039c6302eb755b873dbb454d65a36e419df4" ], - "version": "==1.0.1" + "version": "==1.1.3" }, "invenio-circulation": { "editable": true, @@ -549,10 +541,10 @@ }, "invenio-indexer": { "hashes": [ - "sha256:ca8c705813eb1256625f3c6d1550a0610be693676d4929f2f63e0fcf17879973" + "sha256:a8b4052604bba21ae1e4ccf5249ad60a8967a8ccdeebec710dcbeea33322f0c5" ], "index": "pypi", - "version": "==1.0.2" + "version": "==1.1.1" }, "invenio-jsonschemas": { "hashes": [ @@ -564,10 +556,10 @@ }, "invenio-logging": { "hashes": [ - "sha256:8c2f6633905d100867814e5894564e79682bca7e45ea2870a6c03a69c4860dec", - "sha256:f3a87ab70993e6c7eb5990e70dda3ab0729b1bfa6c90ecf1f9890018accb926f" + "sha256:2aab96570ea6475f66054883a3430b166d67e3af68979110ddda6cd3e8dff477", + "sha256:34a23a488c99b0d54c9dc374ec6bfa6688348495a5ccf7e460e3793045ba77ee" ], - "version": "==1.1.1" + "version": "==1.2.1" }, "invenio-mail": { "hashes": [ @@ -583,11 +575,11 @@ }, "invenio-oaiserver": { "hashes": [ - "sha256:0ad03228ff0cb1be9bc403420c8ebc2b7b5fc8625fbde14e254b068a12020abe", - "sha256:c3aa3b0e63f79ab68ac88b950f8b67d455bc5458b93bf0b07ffef67d88412100" + "sha256:aa2e5c562e3ce4dcda6cc192c73decf3e5e2814b6a788fa80f18156bf5a8b56b", + "sha256:b685a5288fee11c3798bb3818c01e5e150d051ff11f406e092e2d5e7b379c1dd" ], "index": "pypi", - "version": "==1.0.3" + "version": "==1.1.2" }, "invenio-oauth2server": { "hashes": [ @@ -605,11 +597,11 @@ }, "invenio-pidstore": { "hashes": [ - "sha256:582b28e7e2bdf4df93e5398a0e147b2eb08690ede339249dd3a8c92b713e6f00", - "sha256:77f59075dfa34564b913e5d58f37d55b2c78d7b4fa1eeaa5f8c42576848bb06d" + "sha256:052f9972c4b91176609b339635d4b074fc2a9423a3fe78d9fff8576947099ca7", + "sha256:76a16f24b3a7ac2f898969ce3012bf4b0b9969d5e8c4cd179657cc73c94e1b86" ], "index": "pypi", - "version": "==1.0.0" + "version": "==1.1.0" }, "invenio-records": { "hashes": [ @@ -621,11 +613,11 @@ }, "invenio-records-rest": { "hashes": [ - "sha256:ea5bb3e05d331bd38dfcf154d8fc123558dbeaaad0eb990dab852f946ee8f5b9", - "sha256:f89124c7e1e8fcc16b8652597d3c529a66cfc9f1bbba7e4d40bf880038b29691" + "sha256:3b88511d14182ef385123f04e8f3680600a4035dd80586ff46aeab2edf8f57d7", + "sha256:f549b5c71e8f224c7f09b892484bef3d108ee8cbff740aced2fd4dafc16db32e" ], "index": "pypi", - "version": "==1.4.2" + "version": "==1.6.4" }, "invenio-records-ui": { "hashes": [ @@ -637,17 +629,17 @@ }, "invenio-rest": { "hashes": [ - "sha256:665cdc6d7f47b532e4823dc1a0a62de32e31caf387efc93aff7593a34124ee1d", - "sha256:e5a73fcfc15c052840608950c0524447f1e86d24cc58a8fbf12a48a82f7e582a" + "sha256:3a76fc42cd1265e2a589a6bd8b0fa06d62e651e67a33b547d991b72b251600c9", + "sha256:3f44ef712d8d0c2956ec841aaa3fb6434d3c8cf1dddad05d7a4cc98a50afc671" ], - "version": "==1.0.0" + "version": "==1.1.3" }, "invenio-search": { "hashes": [ - "sha256:3db927a9a447bcfb8f84f4798a3bf656ab8ff6f093d8de83224bdde4518bf830", - "sha256:b51d0af749fb08a6d8e46e8d7d2232e75462d87bb11a78aade33497ed1079773" + "sha256:5956be4f6f024f84d4732307356b618ff826336e437e9d1c7ec99edfc1b7d734", + "sha256:bf104f3ef751d38ea545689c08b25c94d6dc91130096ea890c92315fa519299c" ], - "version": "==1.1.1" + "version": "==1.2.3" }, "invenio-theme": { "hashes": [ @@ -713,13 +705,6 @@ ], "version": "==2.11.2" }, - "jinja2-time": { - "hashes": [ - "sha256:d14eaa4d315e7688daa4969f616f226614350c48730bfa1692d2caebd8c90d40", - "sha256:d3eab6605e3ec8b7a0863df09cc1d23714908fa61aa6986a845c20ba488b4efa" - ], - "version": "==0.2.0" - }, "jsmin": { "hashes": [ "sha256:b6df99b2cd1c75d9d342e4335b535789b8da9107ec748212706ef7bbe5c2553b" @@ -873,10 +858,11 @@ }, "marshmallow": { "hashes": [ - "sha256:7cb1881a0fa84be4b5c1e301168236932c345f6910725f99905d636bfe93e9e9", - "sha256:e9390c0c80437d7a02d84e3d1635dc20f37a8bcb149ca443d93791bdc683f28d" + "sha256:90854221bbb1498d003a0c3cc9d8390259137551917961c8b5258c64026b2f85", + "sha256:ac2e13b30165501b7d41fc0371b8df35944f5849769d136f20e2c5f6cdc6e665" ], - "version": "==2.21.0" + "markers": "python_version >= '3.0.0'", + "version": "==3.5.1" }, "maxminddb": { "hashes": [ @@ -890,11 +876,28 @@ ], "version": "==2018.703" }, - "msgpack-python": { - "hashes": [ - "sha256:378cc8a6d3545b532dfd149da715abae4fda2a3adb6d74e525d0d5e51f46909b" + "msgpack": { + "hashes": [ + "sha256:002a0d813e1f7b60da599bdf969e632074f9eec1b96cbed8fb0973a63160a408", + "sha256:25b3bc3190f3d9d965b818123b7752c5dfb953f0d774b454fd206c18fe384fb8", + "sha256:271b489499a43af001a2e42f42d876bb98ccaa7e20512ff37ca78c8e12e68f84", + "sha256:39c54fdebf5fa4dda733369012c59e7d085ebdfe35b6cf648f09d16708f1be5d", + "sha256:4233b7f86c1208190c78a525cd3828ca1623359ef48f78a6fea4b91bb995775a", + "sha256:5bea44181fc8e18eed1d0cd76e355073f00ce232ff9653a0ae88cb7d9e643322", + "sha256:5dba6d074fac9b24f29aaf1d2d032306c27f04187651511257e7831733293ec2", + "sha256:7a22c965588baeb07242cb561b63f309db27a07382825fc98aecaf0827c1538e", + "sha256:908944e3f038bca67fcfedb7845c4a257c7749bf9818632586b53bcf06ba4b97", + "sha256:9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0", + "sha256:aa5c057eab4f40ec47ea6f5a9825846be2ff6bf34102c560bad5cad5a677c5be", + "sha256:b3758dfd3423e358bbb18a7cccd1c74228dffa7a697e5be6cb9535de625c0dbf", + "sha256:c901e8058dd6653307906c5f157f26ed09eb94a850dddd989621098d347926ab", + "sha256:cec8bf10981ed70998d98431cd814db0ecf3384e6b113366e7f36af71a0fca08", + "sha256:db685187a415f51d6b937257474ca72199f393dad89534ebbdd7d7a3b000080e", + "sha256:e35b051077fc2f3ce12e7c6a34cf309680c63a842db3a0616ea6ed25ad20d272", + "sha256:e7bbdd8e2b277b77782f3ce34734b0dfde6cbe94ddb74de8d733d603c7f9e2b1", + "sha256:ea41c9219c597f1d2bf6b374d951d310d58684b5de9dc4bd2976db9e1e22c140" ], - "version": "==0.5.6" + "version": "==1.0.0" }, "node-semver": { "hashes": [ @@ -953,13 +956,6 @@ "index": "pypi", "version": "==1.1.0" }, - "poyo": { - "hashes": [ - "sha256:3e2ca8e33fdc3c411cd101ca395668395dd5dc7ac775b8e809e3def9f9fe041a", - "sha256:e26956aa780c45f011ca9886f044590e2d8fd8b61db7b1c1cf4e0869f48ed4dd" - ], - "version": "==0.5.0" - }, "prompt-toolkit": { "hashes": [ "sha256:563d1a4140b63ff9dd587bda9557cffb2fe73650205ab6f4383092fb882e7dc8", @@ -1065,12 +1061,6 @@ ], "version": "==1.0.4" }, - "python-slugify": { - "hashes": [ - "sha256:a8fc3433821140e8f409a9831d13ae5deccd0b033d4744d94b31fea141bdd84c" - ], - "version": "==4.0.0" - }, "pytz": { "hashes": [ "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", @@ -1230,13 +1220,6 @@ ], "version": "==0.36.3" }, - "text-unidecode": { - "hashes": [ - "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", - "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93" - ], - "version": "==1.3" - }, "traitlets": { "hashes": [ "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44", @@ -1267,10 +1250,10 @@ }, "urllib3": { "hashes": [ - "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", - "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" ], - "version": "==1.24.3" + "version": "==1.25.9" }, "uwsgi": { "hashes": [ @@ -1316,10 +1299,11 @@ }, "webargs": { "hashes": [ - "sha256:d66a056b3a40c5a3774a650fd349db5970cd91fa8f04ac50d00582c949b45b26", - "sha256:dfe01cd3711cee9a3651bfd5ca67770ace4a6f35f3cae583aa8400bbf7706bb4" + "sha256:4f04918864c7602886335d8099f9b8960ee698b6b914f022736ed50be6b71235", + "sha256:871642a2e0c62f21d5b78f357750ac7a87e6bc734c972f633aa5fb6204fbf29a", + "sha256:fc81c9f9d391acfbce406a319217319fd8b2fd862f7fdb5319ad06944f36ed25" ], - "version": "==6.0.0" + "version": "==5.5.3" }, "webassets": { "hashes": [ @@ -1546,10 +1530,10 @@ }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", + "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" ], - "version": "==2.7" + "version": "==2.9" }, "imagesize": { "hashes": [ @@ -1636,10 +1620,11 @@ }, "marshmallow": { "hashes": [ - "sha256:7cb1881a0fa84be4b5c1e301168236932c345f6910725f99905d636bfe93e9e9", - "sha256:e9390c0c80437d7a02d84e3d1635dc20f37a8bcb149ca443d93791bdc683f28d" + "sha256:90854221bbb1498d003a0c3cc9d8390259137551917961c8b5258c64026b2f85", + "sha256:ac2e13b30165501b7d41fc0371b8df35944f5849769d136f20e2c5f6cdc6e665" ], - "version": "==2.21.0" + "markers": "python_version >= '3.0.0'", + "version": "==3.5.1" }, "mock": { "hashes": [ @@ -1760,11 +1745,11 @@ }, "pytest-invenio": { "hashes": [ - "sha256:096063be36e57586db47e735ea848d0577b7b9b54d868a852af56cb5448dd93b", - "sha256:2a980fe0cb1834e30e887b19576cd52d57b6bc02afca6c3343e59895f3413b82" + "sha256:8f2625312714a61a0ab18b96efa2002253acfa38d1577270d3d120b7b7acb078", + "sha256:9a7e12c21a7cca0bccadc88e03016283fd7aeac5d58c2ade30aeefc9eb686f3b" ], "index": "pypi", - "version": "==1.0.6" + "version": "==1.2.1" }, "pytest-mock": { "hashes": [ @@ -1799,9 +1784,9 @@ }, "python-slugify": { "hashes": [ - "sha256:a8fc3433821140e8f409a9831d13ae5deccd0b033d4744d94b31fea141bdd84c" + "sha256:7723daf30996db26573176bddcdf5fcb98f66dc70df05c9cb29f2c79b8193245" ], - "version": "==4.0.0" + "version": "==1.2.6" }, "pytz": { "hashes": [ @@ -1937,10 +1922,10 @@ }, "urllib3": { "hashes": [ - "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", - "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" ], - "version": "==1.24.3" + "version": "==1.25.9" }, "virtualenv": { "hashes": [ diff --git a/rero_ils/config.py b/rero_ils/config.py index 06bdd98052..b7efc6ff2c 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -40,24 +40,21 @@ PendingToItemInTransitPickup, ToItemOnLoan from invenio_records_rest.facets import terms_filter from invenio_records_rest.utils import allow_all, deny_all -from invenio_search import RecordsSearch - -from rero_ils.modules.api import IlsRecordIndexer from .modules.acq_accounts.api import AcqAccount from .modules.acq_accounts.permissions import can_create_acq_account_factory, \ can_list_acq_account_factory, can_read_update_delete_acq_account_factory from .modules.acq_invoices.api import AcquisitionInvoice -from .modules.acq_order_lines.api import AcqOrderLine, AcqOrderLinesIndexer +from .modules.acq_order_lines.api import AcqOrderLine from .modules.acq_orders.api import AcqOrder from .modules.budgets.api import Budget from .modules.budgets.permissions import can_list_budgets_factory from .modules.circ_policies.api import CircPolicy from .modules.circ_policies.permissions import can_update_circ_policy_factory -from .modules.documents.api import Document, DocumentsIndexer -from .modules.holdings.api import Holding, HoldingsIndexer +from .modules.documents.api import Document +from .modules.holdings.api import Holding from .modules.item_types.api import ItemType -from .modules.items.api import Item, ItemsIndexer +from .modules.items.api import Item from .modules.items.permissions import can_create_item_factory, \ can_update_delete_item_factory from .modules.libraries.api import Library @@ -124,42 +121,54 @@ def _(x): ('de', _('German')), ('it', _('Italian')), ] +# Define the default system currency in used. Each organisation can override +# this parameter using the 'default_currency' field +RERO_ILS_DEFAULT_CURRENCY = 'CHF' # Base templates # ============== #: Global base template. BASE_TEMPLATE = 'rero_ils/page.html' -THEME_BASE_TEMPLATE = BASE_TEMPLATE #: Cover page base template (used for e.g. login/sign-up). COVER_TEMPLATE = 'rero_ils/page_cover.html' -THEME_COVER_TEMPLATE = COVER_TEMPLATE #: Footer base template. -THEME_FOOTER_TEMPLATE = 'rero_ils/footer.html' +FOOTER_TEMPLATE = 'rero_ils/footer.html' #: Header base template. HEADER_TEMPLATE = 'rero_ils/header.html' -#: Header base template. -THEME_HEADER_TEMPLATE = HEADER_TEMPLATE -#: Settings page template used for e.g. display user settings views. -THEME_SETTINGS_TEMPLATE = 'invenio_theme/page_settings.html' +#: Settings base template +SETTINGS_TEMPLATE = 'invenio_theme/page_settings.html' +#: Admin base template +ADMIN_BASE_TEMPLATE = BASE_TEMPLATE + +# Miscellaneous variable around templates +# ======================= #: Template for security pages. SECURITY_LOGIN_USER_TEMPLATE = 'rero_ils/login_user.html' SECURITY_REGISTER_USER_TEMPLATE = 'rero_ils/register_user.html' SECURITY_FORGOT_PASSWORD_TEMPLATE = 'rero_ils/forgot_password.html' SECURITY_RESET_PASSWORD_TEMPLATE = 'rero_ils/reset_password.html' -#: Template for error pages. -THEME_ERROR_TEMPLATE = 'rero_ils/page_error.html' #: Template for tombstone page. RECORDS_UI_TOMBSTONE_TEMPLATE = "rero_ils/tombstone.html" +#: Miscellaneous templates +SEARCH_UI_JSTEMPLATE_RESULTS = ( + 'templates/rero_ils/brief_view_documents.html' +) +SEARCH_UI_SEARCH_TEMPLATE = 'rero_ils/search.html' +SEARCH_UI_JSTEMPLATE_FACETS = 'templates/rero_ils/facets.html' +SEARCH_UI_JSTEMPLATE_RANGE = 'templates/rero_ils/range.html' +SEARCH_UI_JSTEMPLATE_COUNT = 'templates/rero_ils/count.html' +SEARCH_UI_JSTEMPLATE_PAGINATION = 'templates/rero_ils/pagination.html' +SEARCH_UI_SEARCH_MIMETYPE = 'application/rero+json' -RERO_ILS_THEME_SEARCH_ENDPOINT = '/search/documents' - -# Logings -# ======= -LOGGING_SENTRY_LEVEL = "ERROR" -LOGGING_SENTRY_CELERY = True +SEARCH_UI_HEADER_TEMPLATE = 'rero_ils/search_header.html' +REROILS_SEARCHBAR_TEMPLATE = 'templates/rero_ils/searchbar.html' +RERO_ILS_EDITOR_TEMPLATE = 'rero_ils/editor.html' +SECURITY_LOGIN_USER_TEMPLATE = 'rero_ils/login_user.html' # Theme configuration # =================== +#: Brand logo. +THEME_LOGO = 'images/logo_rero_ils.png' #: Site name THEME_SITENAME = _('rero-ils') #: Use default frontpage. @@ -168,29 +177,32 @@ def _(x): THEME_FRONTPAGE_TITLE = _('rero-ils') #: Frontpage template. THEME_FRONTPAGE_TEMPLATE = 'rero_ils/frontpage.html' - +#: Theme base template. +THEME_BASE_TEMPLATE = BASE_TEMPLATE +#: Cover page theme template (used for e.g. login/sign-up). +THEME_COVER_TEMPLATE = COVER_TEMPLATE +#: Footer theme template. +THEME_FOOTER_TEMPLATE = FOOTER_TEMPLATE +#: Header theme template. +THEME_HEADER_TEMPLATE = HEADER_TEMPLATE +#: Settings page template used for e.g. display user settings views. +THEME_SETTINGS_TEMPLATE = SETTINGS_TEMPLATE +#: Template for error pages. +THEME_ERROR_TEMPLATE = 'rero_ils/page_error.html' +#: RERO-ils search endpoint (i.e /search/documents) +RERO_ILS_THEME_SEARCH_ENDPOINT = '/search/documents' +# External CSS for each organisation customization +RERO_ILS_THEME_ORGANISATION_CSS_ENDPOINT = 'https://resources.rero.ch/ils/test/css/' #: Template for including a tracking code for web analytics. THEME_TRACKINGCODE_TEMPLATE = 'rero_ils/trackingcode.html' THEME_JAVASCRIPT_TEMPLATE = 'rero_ils/javascript.html' -#: Brand logo. -THEME_LOGO = 'images/logo_rero_ils.png' -# External CSS for each organisation customization -RERO_ILS_THEME_ORGANISATION_CSS_ENDPOINT = 'https://resources.rero.ch/ils/test/css/' -SEARCH_UI_JSTEMPLATE_RESULTS = ( - 'templates/rero_ils/brief_view_documents.html' -) -SEARCH_UI_SEARCH_TEMPLATE = 'rero_ils/search.html' -SEARCH_UI_JSTEMPLATE_FACETS = 'templates/rero_ils/facets.html' -SEARCH_UI_JSTEMPLATE_RANGE = 'templates/rero_ils/range.html' -SEARCH_UI_JSTEMPLATE_COUNT = 'templates/rero_ils/count.html' -SEARCH_UI_JSTEMPLATE_PAGINATION = 'templates/rero_ils/pagination.html' -SEARCH_UI_SEARCH_MIMETYPE = 'application/rero+json' - -SEARCH_UI_HEADER_TEMPLATE = 'rero_ils/search_header.html' -REROILS_SEARCHBAR_TEMPLATE = 'templates/rero_ils/searchbar.html' -RERO_ILS_EDITOR_TEMPLATE = 'rero_ils/editor.html' -SECURITY_LOGIN_USER_TEMPLATE = 'rero_ils/login_user.html' +# Logings +# ======= +#: Sentry level +LOGGING_SENTRY_LEVEL = "ERROR" +#: Sentry: use celery or not +LOGGING_SENTRY_CELERY = True # Email configuration # =================== @@ -201,8 +213,8 @@ def _(x): # Assets # ====== -#: Static files collection method (defaults to copying files). -# COLLECT_STORAGE = 'flask_collect.storage.file' +#: Static files collection method (defaults to symbolic link to files). +COLLECT_STORAGE = 'flask_collect.storage.link' # Accounts # ======== @@ -212,6 +224,11 @@ def _(x): SECURITY_EMAIL_SUBJECT_REGISTER = _("Welcome to RERO-ILS!") #: Redis session storage URL. ACCOUNTS_SESSION_REDIS_URL = 'redis://localhost:6379/1' +#: Enable session/user id request tracing. This feature will add X-Session-ID +#: and X-User-ID headers to HTTP response. You MUST ensure that NGINX (or other +#: proxies) removes these headers again before sending the response to the +#: client. Set to False, in case of doubt. +ACCOUNTS_USERINFO_HEADERS = False # Disable User Profiles USERPROFILES = False # make security blueprints available to the REST API @@ -228,8 +245,8 @@ def _(x): CELERY_RESULT_BACKEND = 'redis://localhost:6379/2' #: Scheduled tasks configuration (aka cronjobs). CELERY_BEAT_SCHEDULE = { - 'indexer': { - 'task': 'invenio_indexer.tasks.process_bulk_queue', + 'bulk-indexer': { + 'task': 'rero_ils.modules.tasks.process_bulk_queue', 'schedule': timedelta(minutes=5), }, 'accounts': { @@ -242,8 +259,8 @@ def _(x): 'kwargs': dict(name='ebooks'), }, 'notification-creation': { - 'task': - 'rero_ils.modules.notifications.tasks.create_over_and_due_soon_notifications', + 'task': ('rero_ils.modules.notifications.tasks' + '.create_over_and_due_soon_notifications'), 'schedule': crontab(minute="*/5") # TODO: in production set this up once a day }, @@ -286,7 +303,6 @@ def _(x): MAX_CONTENT_LENGTH = 100 * 1024 * 1024 # 100 MiB #: For dev. Set to false when testing on localhost in no debug mode APP_ENABLE_SECURE_HEADERS = True - # TODO: review theses rules for security purposes APP_DEFAULT_SECURE_HEADERS = { # disabled as https is not used by the application: @@ -327,7 +343,7 @@ def _(x): 'session_cookie_secure': True, 'session_cookie_http_only': True, } -#: Sets cookie with the secure flag by default +#: Sets cookie with the secure flag (by default False) SESSION_COOKIE_SECURE = False #: Since HAProxy and Nginx route all requests no matter the host header #: provided, the allowed hosts variable is set to localhost. In production it @@ -339,6 +355,12 @@ def _(x): # ======= OAISERVER_ID_PREFIX = 'oai:ils.rero.ch:' +# TODO: Check if needed one day +# Previewers +# ========== +#: Include IIIF preview for images. +# PREVIEWER_PREFERENCE = ['iiif_image'] + BASE_PREFERENCE + # Debug # ===== # Flask-DebugToolbar is by default enabled when the application is running in @@ -370,10 +392,10 @@ def _(x): pid_type='doc', pid_minter='document_id', pid_fetcher='document_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.documents.api:DocumentsSearch', search_index='documents', search_type=None, - indexer_class=DocumentsIndexer, + indexer_class='rero_ils.modules.documents.api:DocumentsIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -392,7 +414,8 @@ def _(x): }, list_route='/documents/', record_class='rero_ils.modules.documents.api:Document', - item_route='/documents/', + item_route=('/documents/'), # suggesters=dict( # title={ # 'completion': { @@ -414,10 +437,10 @@ def _(x): pid_type='item', pid_minter='item_id', pid_fetcher='item_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.items.api:ItemsSearch', search_index='items', search_type=None, - indexer_class=ItemsIndexer, + indexer_class='rero_ils.modules.items.api:ItemsIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -433,7 +456,8 @@ def _(x): 'application/json': lambda: Item(request.get_json()), }, record_class='rero_ils.modules.items.api:Item', - item_route='/items/', + item_route=('/items/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -447,9 +471,9 @@ def _(x): pid_type='itty', pid_minter='item_type_id', pid_fetcher='item_type_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.item_types.api:ItemTypesSearch', search_index='item_types', - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.item_types.api:ItemTypesIndexer', search_type=None, record_serializers={ 'application/json': ( @@ -466,7 +490,8 @@ def _(x): 'application/json': lambda: ItemType(request.get_json()), }, record_class='rero_ils.modules.item_types.api:ItemType', - item_route='/item_types/', + item_route=('/item_types/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -479,9 +504,9 @@ def _(x): pid_type='hold', pid_minter='holding_id', pid_fetcher='holding_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.holdings.api:HoldingsSearch', search_index='holdings', - indexer_class=HoldingsIndexer, + indexer_class='rero_ils.modules.holdings.api:HoldingsIndexer', search_type=None, record_serializers={ 'application/json': ( @@ -498,7 +523,8 @@ def _(x): 'application/json': lambda: Holding(request.get_json()), }, record_class='rero_ils.modules.holdings.api:Holding', - item_route='/holdings/', + item_route=('/holdings/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -512,9 +538,9 @@ def _(x): pid_type='ptrn', pid_minter='patron_id', pid_fetcher='patron_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.patrons.api:PatronsSearch', search_index='patrons', - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.patrons.api:PatronsIndexer', search_type=None, record_serializers={ 'application/json': ( @@ -531,7 +557,8 @@ def _(x): 'application/json': lambda: Patron(request.get_json()), }, record_class='rero_ils.modules.patrons.api:Patron', - item_route='/patrons/', + item_route=('/patrons/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -545,10 +572,12 @@ def _(x): pid_type='pttr', pid_minter='patron_transaction_id', pid_fetcher='patron_transaction_id', - search_class=RecordsSearch, + search_class=('rero_ils.modules.patron_transactions.api:' + 'PatronTransactionsSearch'), search_index='patron_transactions', search_type=None, - indexer_class=IlsRecordIndexer, + indexer_class=('rero_ils.modules.patron_transactions.api:' + 'PatronTransactionsIndexer'), record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -562,9 +591,12 @@ def _(x): record_loaders={ 'application/json': lambda: PatronTransaction(request.get_json()), }, - record_class='rero_ils.modules.patron_transactions.api:PatronTransaction', + record_class=('rero_ils.modules.patron_transactions.api:' + 'PatronTransaction'), list_route='/patron_transactions/', - item_route='/patron_transactions/', + item_route=('/patron_transactions/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:patron_transactions_search_factory', @@ -578,10 +610,12 @@ def _(x): pid_type='ptre', pid_minter='patron_transaction_event_id', pid_fetcher='patron_transaction_event_id', - search_class=RecordsSearch, + search_class=('rero_ils.modules.patron_transaction_events.api:' + 'PatronTransactionEventsSearch'), search_index='patron_transaction_events', search_type=None, - indexer_class=IlsRecordIndexer, + indexer_class=('rero_ils.modules.patron_transaction_events.api:' + 'PatronTransactionEventsIndexer'), record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -593,11 +627,15 @@ def _(x): ) }, record_loaders={ - 'application/json': lambda: PatronTransactionEvent(request.get_json()), + 'application/json': lambda: PatronTransactionEvent( + request.get_json()), }, - record_class='rero_ils.modules.patron_transaction_events.api:PatronTransactionEvent', + record_class=('rero_ils.modules.patron_transaction_events.api:' + 'PatronTransactionEvent'), list_route='/patron_transaction_events/', - item_route='/patron_transaction_events/', + item_route=('/patron_transaction_events/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:patron_transactions_search_factory', @@ -611,10 +649,10 @@ def _(x): pid_type='ptty', pid_minter='patron_type_id', pid_fetcher='patron_type_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.patron_types.api:PatronTypesSearch', search_index='patron_types', search_type=None, - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.patron_types.api:PatronTypesIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -630,7 +668,9 @@ def _(x): 'application/json': lambda: PatronType(request.get_json()), }, record_class='rero_ils.modules.patron_types.api:PatronType', - item_route='/patron_types/', + item_route=('/patron_types/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -643,9 +683,10 @@ def _(x): pid_type='org', pid_minter='organisation_id', pid_fetcher='organisation_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.organisations.api:OrganisationsSearch', search_index='organisations', - indexer_class=IlsRecordIndexer, + indexer_class=('rero_ils.modules.organisations.api:' + 'OrganisationsIndexer'), search_type=None, record_serializers={ 'application/json': ( @@ -662,10 +703,13 @@ def _(x): 'application/json': lambda: Organisation(request.get_json()), }, record_class='rero_ils.modules.organisations.api:Organisation', - item_route='/organisations/', + item_route=('/organisations/'), default_media_type='application/json', max_result_window=10000, - search_factory_imp='rero_ils.query:organisation_organisation_search_factory', + search_factory_imp=('rero_ils.query:' + 'organisation_organisation_search_factory'), list_permission_factory_imp=can_access_organisation_patrons_factory, create_permission_factory_imp=deny_all, update_permission_factory_imp=deny_all, @@ -676,9 +720,9 @@ def _(x): pid_type='lib', pid_minter='library_id', pid_fetcher='library_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.libraries.api:LibrariesSearch', search_index='libraries', - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.libraries.api:LibrariesIndexer', search_type=None, record_serializers={ 'application/json': ( @@ -695,7 +739,8 @@ def _(x): 'application/json': lambda: Library(request.get_json()), }, record_class='rero_ils.modules.libraries.api:Library', - item_route='/libraries/', + item_route=('/libraries/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -709,9 +754,9 @@ def _(x): pid_type='loc', pid_minter='location_id', pid_fetcher='location_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.locations.api:LocationsSearch', search_index='locations', - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.locations.api:LocationsIndexer', search_type=None, record_serializers={ 'application/json': ( @@ -728,7 +773,8 @@ def _(x): 'application/json': lambda: Location(request.get_json()), }, record_class='rero_ils.modules.locations.api:Location', - item_route='/locations/', + item_route=('/locations/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -741,9 +787,9 @@ def _(x): pid_type='pers', pid_minter='person_id', pid_fetcher='person_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.persons.api:PersonsSearch', search_index='persons', - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.persons.api:PersonsIndexer', search_type=None, record_serializers={ 'application/json': ( @@ -760,7 +806,8 @@ def _(x): 'application/json': lambda: Person(request.get_json()), }, record_class='rero_ils.modules.persons.api:Person', - item_route='/persons/', + item_route=('/persons/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:person_view_search_factory', @@ -774,9 +821,9 @@ def _(x): pid_type='cipo', pid_minter='circ_policy_id', pid_fetcher='circ_policy_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.circ_policies.api:CircPoliciesSearch', search_index='circ_policies', - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.circ_policies.api:CircPoliciesIndexer', search_type=None, record_serializers={ 'application/json': ( @@ -793,7 +840,9 @@ def _(x): }, list_route='/circ_policies/', record_class='rero_ils.modules.circ_policies.api:CircPolicy', - item_route='/circ_policies/', + item_route=('/circ_policies/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -806,9 +855,10 @@ def _(x): pid_type='notif', pid_minter='notification_id', pid_fetcher='notification_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.notifications.api:NotificationsSearch', search_index='notifications', - indexer_class=IlsRecordIndexer, + indexer_class=('rero_ils.modules.notifications.api:' + 'NotificationsIndexer'), search_type=None, record_serializers={ 'application/json': ( @@ -825,7 +875,9 @@ def _(x): }, list_route='/notifications/', record_class='rero_ils.modules.notifications.api:Notification', - item_route='/notifications/', + item_route=('/notifications/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -838,10 +890,10 @@ def _(x): pid_type='vndr', pid_minter='vendor_id', pid_fetcher='vendor_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.vendors.api:VendorsSearch', search_index='vendors', search_type=None, - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.vendors.api:VendorsIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -857,7 +909,8 @@ def _(x): }, record_class='rero_ils.modules.vendors.api:Vendor', list_route='/vendors/', - item_route='/vendors/', + item_route=('/vendors/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -871,10 +924,10 @@ def _(x): pid_type='acac', pid_minter='acq_account_id', pid_fetcher='acq_account_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.acq_accounts.api:AcqAccountsSearch', search_index='acq_accounts', search_type=None, - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.acq_accounts.api:AcqAccountsIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -885,7 +938,8 @@ def _(x): 'rero_ils.modules.serializers:json_v1_search' ), 'application/rero+json': ( - 'rero_ils.modules.acq_accounts.serializers:json_acq_account_search' + 'rero_ils.modules.acq_accounts.serializers:' + 'json_acq_account_search' ), }, record_loaders={ @@ -893,7 +947,9 @@ def _(x): }, record_class='rero_ils.modules.acq_accounts.api:AcqAccount', list_route='/acq_accounts/', - item_route='/acq_accounts/', + item_route=('/acq_accounts/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:acq_accounts_search_factory', @@ -907,10 +963,10 @@ def _(x): pid_type='budg', pid_minter='budget_id', pid_fetcher='budget_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.budgets.api:BudgetsSearch', search_index='budgets', search_type=None, - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.budgets.api:BudgetsIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -926,7 +982,8 @@ def _(x): }, record_class='rero_ils.modules.budgets.api:Budget', list_route='/budgets/', - item_route='/budgets/', + item_route=('/budgets/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -940,10 +997,10 @@ def _(x): pid_type='acor', pid_minter='acq_order_id', pid_fetcher='acq_order_id', - search_class=RecordsSearch, + search_class='rero_ils.modules.acq_orders.api:AcqOrdersSearch', search_index='acq_orders', search_type=None, - indexer_class=IlsRecordIndexer, + indexer_class='rero_ils.modules.acq_orders.api:AcqOrdersIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -962,7 +1019,8 @@ def _(x): }, record_class='rero_ils.modules.acq_orders.api:AcqOrder', list_route='/acq_orders/', - item_route='/acq_orders/', + item_route=('/acq_orders/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -976,10 +1034,12 @@ def _(x): pid_type='acol', pid_minter='acq_order_line_id', pid_fetcher='acq_order_line_id', - search_class=RecordsSearch, + search_class=('rero_ils.modules.acq_order_lines.api:' + 'AcqOrderLinesSearch'), search_index='acq_order_lines', search_type=None, - indexer_class=AcqOrderLinesIndexer, + indexer_class=('rero_ils.modules.acq_order_lines.api:' + 'AcqOrderLinesIndexer'), record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -995,7 +1055,9 @@ def _(x): }, record_class='rero_ils.modules.acq_order_lines.api:AcqOrderLine', list_route='/acq_order_lines/', - item_route='/acq_order_lines/', + item_route=('/acq_order_lines/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -1009,10 +1071,12 @@ def _(x): pid_type='acin', pid_minter='acq_invoice_id', pid_fetcher='acq_invoice_id', - search_class=RecordsSearch, + search_class=('rero_ils.modules.acq_invoices.api:' + 'AcquisitionInvoicesSearch'), search_index='acq_invoices', search_type=None, - indexer_class=IlsRecordIndexer, + indexer_class=('rero_ils.modules.acq_invoices.api:' + 'AcquisitionInvoicesIndexer'), record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -1023,7 +1087,8 @@ def _(x): 'rero_ils.modules.serializers:json_v1_search' ), 'application/rero+json': ( - 'rero_ils.modules.acq_invoices.serializers:json_acquisition_invoice_search' + 'rero_ils.modules.acq_invoices.serializers:' + 'json_acquisition_invoice_search' ) }, record_loaders={ @@ -1031,7 +1096,9 @@ def _(x): }, record_class='rero_ils.modules.acq_invoices.api:AcquisitionInvoice', list_route='/acq_invoices/', - item_route='/acq_invoices/', + item_route=('/acq_invoices/'), default_media_type='application/json', max_result_window=10000, search_factory_imp='rero_ils.query:organisation_search_factory', @@ -1066,28 +1133,36 @@ def _(x): # The organisation or library facet is defined # dynamically during the query (query.py) document_type=dict( - terms=dict(field='type', size=DOCUMENTS_AGGREGATION_SIZE) + terms=dict(field='type', + size=DOCUMENTS_AGGREGATION_SIZE) ), author__en=dict( - terms=dict(field='facet_authors_en', size=DOCUMENTS_AGGREGATION_SIZE) + terms=dict(field='facet_authors_en', + size=DOCUMENTS_AGGREGATION_SIZE) ), author__fr=dict( - terms=dict(field='facet_authors_fr', size=DOCUMENTS_AGGREGATION_SIZE) + terms=dict(field='facet_authors_fr', + size=DOCUMENTS_AGGREGATION_SIZE) ), author__de=dict( - terms=dict(field='facet_authors_de', size=DOCUMENTS_AGGREGATION_SIZE) + terms=dict(field='facet_authors_de', + size=DOCUMENTS_AGGREGATION_SIZE) ), author__it=dict( - terms=dict(field='facet_authors_it', size=DOCUMENTS_AGGREGATION_SIZE) + terms=dict(field='facet_authors_it', + size=DOCUMENTS_AGGREGATION_SIZE) ), language=dict( - terms=dict(field='language.value', size=DOCUMENTS_AGGREGATION_SIZE) + terms=dict(field='language.value', + size=DOCUMENTS_AGGREGATION_SIZE) ), subject=dict( - terms=dict(field='facet_subjects', size=DOCUMENTS_AGGREGATION_SIZE) + terms=dict(field='facet_subjects', + size=DOCUMENTS_AGGREGATION_SIZE) ), status=dict( - terms=dict(field='holdings.items.status', size=DOCUMENTS_AGGREGATION_SIZE) + terms=dict(field='holdings.items.status', + size=DOCUMENTS_AGGREGATION_SIZE) ) ), filters={ @@ -1497,8 +1572,6 @@ def _(x): 'de': ['gnd', 'rero', 'bnf'], } -ADMIN_BASE_TEMPLATE = BASE_TEMPLATE - #: Invenio circulation configuration. CIRCULATION_ITEM_EXISTS = Item.get_record_by_pid CIRCULATION_PATRON_EXISTS = Patron.get_record_by_pid @@ -1621,10 +1694,6 @@ def _(x): ) ) -# Define the default system currency in used. Each organisation can override -# this parameter using the 'default_currency' field -RERO_ILS_DEFAULT_CURRENCY = 'CHF' - WEBPACKEXT_PROJECT = 'rero_ils.webpack:project' # WIKI diff --git a/rero_ils/es_templates/v6/__init__.py b/rero_ils/es_templates/v6/__init__.py new file mode 100644 index 0000000000..d335f866bc --- /dev/null +++ b/rero_ils/es_templates/v6/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2020 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""ES Templates module for RERO-ils.""" diff --git a/rero_ils/modules/acq_accounts/api.py b/rero_ils/modules/acq_accounts/api.py index e68aa3007d..4e5e0afb50 100644 --- a/rero_ils/modules/acq_accounts/api.py +++ b/rero_ils/modules/acq_accounts/api.py @@ -23,7 +23,7 @@ from .models import AcqAccountIdentifier from ..acq_order_lines.api import AcqOrderLinesSearch -from ..api import IlsRecord, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..libraries.api import Library from ..minters import id_minter @@ -48,6 +48,7 @@ class Meta: """Search only on acq_account index.""" index = 'acq_accounts' + doc_types = None class AcqAccount(IlsRecord): @@ -110,3 +111,9 @@ def reasons_not_to_delete(self): if links: cannot_delete['links'] = links return cannot_delete + + +class AcqAccountsIndexer(IlsRecordsIndexer): + """Indexing documents in Elasticsearch.""" + + record_cls = AcqAccount diff --git a/rero_ils/modules/acq_invoices/api.py b/rero_ils/modules/acq_invoices/api.py index 47f23f5e82..27a32a0b9d 100644 --- a/rero_ils/modules/acq_invoices/api.py +++ b/rero_ils/modules/acq_invoices/api.py @@ -22,7 +22,7 @@ from flask import current_app from .models import AcquisitionInvoiceIdentifier -from ..api import IlsRecord, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..libraries.api import Library from ..minters import id_minter @@ -48,6 +48,7 @@ class Meta: """Search only on acq_invoice index.""" index = 'acq_invoices' + doc_types = None class AcquisitionInvoice(IlsRecord): @@ -132,6 +133,12 @@ def vendor_pid(self): return self.replace_refs().get('vendor').get('pid') +class AcquisitionInvoicesIndexer(IlsRecordsIndexer): + """Indexing documents in Elasticsearch.""" + + record_cls = AcquisitionInvoice + + class InvoiceLine(object): """Acquisition Invoice line class.""" diff --git a/rero_ils/modules/acq_order_lines/api.py b/rero_ils/modules/acq_order_lines/api.py index a10a644a45..db11eb1b23 100644 --- a/rero_ils/modules/acq_order_lines/api.py +++ b/rero_ils/modules/acq_order_lines/api.py @@ -22,7 +22,7 @@ from flask import current_app from .models import AcqOrderLineIdentifier -from ..api import IlsRecord, IlsRecordIndexer, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..minters import id_minter from ..providers import Provider @@ -39,33 +39,6 @@ acq_order_line_id_fetcher = partial(id_fetcher, provider=AcqOrderLineProvider) -class AcqOrderLinesIndexer(IlsRecordIndexer): - """Indexing Acquisition Order Line in Elasticsearch.""" - - def index(self, record): - """Index an Acquisition Order Line and update total amount of order.""" - return_value = super(AcqOrderLinesIndexer, self).index(record) - self._update_order_total_amount(record) - - return return_value - - def delete(self, record): - """Delete a Acquisition Order Line and update total amount of order.""" - return_value = super(AcqOrderLinesIndexer, self).delete(record) - self._update_order_total_amount(record) - - return return_value - - def _update_order_total_amount(self, record): - """Update total amount of the order.""" - from ..acq_orders.api import AcqOrder - - order_pid = record.replace_refs()['acq_order']['pid'] - order = AcqOrder.get_record_by_pid(order_pid) - order['total_amount'] = order.get_order_total_amount() - order.update(order, dbcommit=True, reindex=True) - - class AcqOrderLinesSearch(IlsRecordsSearch): """Acquisition Order Line Search.""" @@ -73,6 +46,7 @@ class Meta: """Search only on Acquisition Order Line index.""" index = 'acq_order_lines' + doc_types = None class AcqOrderLine(IlsRecord): @@ -146,3 +120,38 @@ def get_order(self): """Shortcut to the order of the order line.""" from ..acq_orders.api import AcqOrder return AcqOrder.get_record_by_pid(self.order_pid) + + def get_number_of_acq_order_lines(self): + """Get number of aquisition order lines.""" + results = AcqOrderLinesSearch().filter( + 'term', acq_order__pid=self.order_pid).source().count() + return results + + +class AcqOrderLinesIndexer(IlsRecordsIndexer): + """Indexing Acquisition Order Line in Elasticsearch.""" + + record_cls = AcqOrderLine + + def index(self, record): + """Index an Acquisition Order Line and update total amount of order.""" + return_value = super(AcqOrderLinesIndexer, self).index(record) + self._update_order_total_amount(record) + + return return_value + + def delete(self, record): + """Delete a Acquisition Order Line and update total amount of order.""" + return_value = super(AcqOrderLinesIndexer, self).delete(record) + self._update_order_total_amount(record) + + return return_value + + def _update_order_total_amount(self, record): + """Update total amount of the order.""" + from ..acq_orders.api import AcqOrder + + order_pid = record.replace_refs()['acq_order']['pid'] + order = AcqOrder.get_record_by_pid(order_pid) + order['total_amount'] = order.get_order_total_amount() + order.update(order, dbcommit=True, reindex=True) diff --git a/rero_ils/modules/acq_orders/api.py b/rero_ils/modules/acq_orders/api.py index 74529d3bdd..9317e1f114 100644 --- a/rero_ils/modules/acq_orders/api.py +++ b/rero_ils/modules/acq_orders/api.py @@ -23,7 +23,7 @@ from .models import AcqOrderIdentifier from ..acq_order_lines.api import AcqOrderLinesSearch -from ..api import IlsRecord, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..libraries.api import Library from ..minters import id_minter @@ -48,6 +48,7 @@ class Meta: """Search only on acq_order index.""" index = 'acq_orders' + doc_types = None class AcqOrder(IlsRecord): @@ -123,3 +124,9 @@ def reasons_not_to_delete(self): if links: cannot_delete['links'] = links return cannot_delete + + +class AcqOrdersIndexer(IlsRecordsIndexer): + """Indexing documents in Elasticsearch.""" + + record_cls = AcqOrder diff --git a/rero_ils/modules/api.py b/rero_ils/modules/api.py index 866750ab76..a22a674a8e 100644 --- a/rero_ils/modules/api.py +++ b/rero_ils/modules/api.py @@ -61,6 +61,7 @@ class Meta: """Search only on item index.""" index = 'records' + doc_types = None @classmethod def flush(cls): @@ -68,129 +69,6 @@ def flush(cls): current_search.flush_and_refresh(cls.Meta.index) -class IlsRecordIndexer(RecordIndexer): - """Indexing class for ils.""" - - def index(self, record): - """Indexing a record.""" - return_value = super(IlsRecordIndexer, self).index(record) - index_name, doc_type = current_record_to_index(record) - current_search.flush_and_refresh(index_name) - return return_value - - def delete(self, record): - """Delete a record. - - :param record: Record instance. - """ - return_value = super(IlsRecordIndexer, self).delete(record) - index_name, doc_type = current_record_to_index(record) - current_search.flush_and_refresh(index_name) - return return_value - - def bulk_index(self, record_id_iterator, doc_type=None): - """Bulk index records. - - :param record_id_iterator: Iterator yielding record UUIDs. - """ - self._bulk_op(record_id_iterator, op_type='index', doc_type=doc_type) - - def _get_record_class(self, payload): - """Get the record class from payload.""" - # take the first defined doc type for finding the class - pid_type = payload.get('doc_type', 'rec') - record_class = obj_or_import_string( - current_app.config.get('RECORDS_REST_ENDPOINTS').get( - pid_type - ).get('record_class', Record) - ) - return record_class - - def _actionsiter(self, message_iterator): - """Iterate bulk actions. - - :param message_iterator: Iterator yielding messages from a queue. - """ - for message in message_iterator: - payload = message.decode() - try: - record_class = self._get_record_class(payload) - if payload['op'] == 'delete': - yield record_class.indexer()._delete_action( - payload=payload - ) - else: - yield record_class.indexer()._index_action( - payload=payload - ) - message.ack() - except NoResultFound: - message.reject() - except Exception: - message.reject() - current_app.logger.error( - "Failed to index record {0}".format(payload.get('id')), - exc_info=True) - - def _index_action(self, payload): - """Bulk index action. - - :param payload: Decoded message body. - :returns: Dictionary defining an Elasticsearch bulk 'index' action. - """ - record_class = self._get_record_class(payload) - record = record_class.get_record(payload['id']) - index, doc_type = self.record_to_index(record) - - arguments = {} - body = self._prepare_record(record, index, doc_type, arguments) - action = { - '_op_type': 'index', - '_index': index, - '_type': doc_type, - '_id': str(record.id), - '_version': record.revision_id, - '_version_type': self._version_type, - '_source': body - } - action.update(arguments) - - return action - - @staticmethod - def _prepare_record(record, index, doc_type, arguments=None, **kwargs): - """Prepare record data for indexing. - - :param record: The record to prepare. - :param index: The Elasticsearch index. - :param doc_type: The Elasticsearch document type. - :param arguments: The arguments to send to Elasticsearch upon indexing. - :param **kwargs: Extra parameters. - :returns: The record metadata. - """ - if current_app.config['INDEXER_REPLACE_REFS']: - data = record.replace_refs().dumps() - else: - data = record.dumps() - - data['_created'] = pytz.utc.localize(record.created).isoformat() \ - if record.created else None - data['_updated'] = pytz.utc.localize(record.updated).isoformat() \ - if record.updated else None - - # Allow modification of data prior to sending to Elasticsearch. - before_record_index.send( - current_app._get_current_object(), - json=data, - record=record, - index=index, - doc_type=doc_type, - arguments={} if arguments is None else arguments, - **kwargs - ) - return data - - class IlsRecord(Record): """ILS Record class.""" @@ -198,7 +76,20 @@ class IlsRecord(Record): fetcher = None provider = None object_type = 'rec' - indexer = IlsRecordIndexer + + @classmethod + def get_indexer_class(cls): + """Get the indexer from config.""" + try: + indexer = obj_or_import_string( + current_app.config['RECORDS_REST_ENDPOINTS'][ + cls.provider.pid_type + ]['indexer_class'] + ) + except: + # provide default indexer if no indexer is defined in config. + indexer = IlsRecordsIndexer + return indexer def validate(self, **kwargs): """Validate record against schema. @@ -390,15 +281,17 @@ def dbcommit(self, reindex=False, forceindex=False): def reindex(self, forceindex=False): """Reindex record.""" + indexer = self.get_indexer_class() if forceindex: - self.indexer(version_type="external_gte").index(self) + indexer(version_type="external_gte").index(self) else: - self.indexer().index(self) + indexer().index(self) def delete_from_index(self): """Delete record from index.""" + indexer = self.get_indexer_class() try: - self.indexer().delete(self) + indexer().delete(self) except NotFoundError: pass @@ -431,3 +324,125 @@ def organisation_pid(self): if self.get('organisation'): return self.replace_refs()['organisation']['pid'] return None + + +class IlsRecordsIndexer(RecordIndexer): + """Indexing class for ils.""" + + record_cls = IlsRecord + + def index(self, record): + """Indexing a record.""" + return_value = super(IlsRecordsIndexer, self).index(record) + index_name, doc_type = current_record_to_index(record) + # TODO: Do we need to flush everytime the ES index? + # Tests depends on this at the moment. + current_search.flush_and_refresh(index_name) + return return_value + + def delete(self, record): + """Delete a record. + + :param record: Record instance. + """ + return_value = super(IlsRecordsIndexer, self).delete(record) + index_name, doc_type = current_record_to_index(record) + current_search.flush_and_refresh(index_name) + return return_value + + def bulk_index(self, record_id_iterator, doc_type=None): + """Bulk index records. + + :param record_id_iterator: Iterator yielding record UUIDs. + """ + self._bulk_op(record_id_iterator, op_type='index', doc_type=doc_type) + + def _get_record_class(self, payload): + """Get the record class from payload.""" + # take the first defined doc type for finding the class + pid_type = payload.get('doc_type', 'rec') + record_class = obj_or_import_string( + current_app.config.get('RECORDS_REST_ENDPOINTS').get( + pid_type + ).get('record_class', Record) + ) + return record_class + + def _actionsiter(self, message_iterator): + """Iterate bulk actions. + + :param message_iterator: Iterator yielding messages from a queue. + """ + for message in message_iterator: + payload = message.decode() + try: + indexer = self._get_record_class(payload).get_indexer_class() + if payload['op'] == 'delete': + yield indexer()._delete_action(payload=payload) + else: + yield indexer()._index_action(payload=payload) + message.ack() + except NoResultFound: + message.reject() + except Exception: + message.reject() + current_app.logger.error( + "Failed to index record {0}".format(payload.get('id')), + exc_info=True) + + def _index_action(self, payload): + """Bulk index action. + + :param payload: Decoded message body. + :returns: Dictionary defining an Elasticsearch bulk 'index' action. + """ + record = self.record_cls.get_record(payload['id']) + index, doc_type = self.record_to_index(record) + + arguments = {} + body = self._prepare_record(record, index, doc_type, arguments) + action = { + '_op_type': 'index', + '_index': index, + '_type': doc_type, + '_id': str(record.id), + '_version': record.revision_id, + '_version_type': self._version_type, + '_source': body + } + action.update(arguments) + + return action + + @staticmethod + def _prepare_record(record, index, doc_type, arguments=None, **kwargs): + """Prepare record data for indexing. + + :param record: The record to prepare. + :param index: The Elasticsearch index. + :param doc_type: The Elasticsearch document type. + :param arguments: The arguments to send to Elasticsearch upon indexing. + :param **kwargs: Extra parameters. + :returns: The record metadata. + """ + if current_app.config['INDEXER_REPLACE_REFS']: + data = record.replace_refs().dumps() + else: + data = record.dumps() + + data['_created'] = pytz.utc.localize(record.created).isoformat() \ + if record.created else None + data['_updated'] = pytz.utc.localize(record.updated).isoformat() \ + if record.updated else None + + # Allow modification of data prior to sending to Elasticsearch. + before_record_index.send( + current_app._get_current_object(), + json=data, + record=record, + index=index, + doc_type=doc_type, + arguments={} if arguments is None else arguments, + **kwargs + ) + return data diff --git a/rero_ils/modules/budgets/api.py b/rero_ils/modules/budgets/api.py index 5c69c1f041..745fd62791 100644 --- a/rero_ils/modules/budgets/api.py +++ b/rero_ils/modules/budgets/api.py @@ -21,7 +21,7 @@ from .models import BudgetIdentifier from ..acq_accounts.api import AcqAccountsSearch -from ..api import IlsRecord, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..minters import id_minter from ..organisations.api import Organisation @@ -46,6 +46,7 @@ class Meta: """Search only on budget index.""" index = 'budgets' + doc_types = None class Budget(IlsRecord): @@ -87,3 +88,9 @@ def reasons_to_keep(self): if organisation.get('current_budget_pid') == self.pid: others['is_default'] = True return others + + +class BudgetsIndexer(IlsRecordsIndexer): + """Indexing documents in Elasticsearch.""" + + record_cls = Budget diff --git a/rero_ils/modules/circ_policies/api.py b/rero_ils/modules/circ_policies/api.py index 6bcae1c7cb..8a3e7bf49c 100644 --- a/rero_ils/modules/circ_policies/api.py +++ b/rero_ils/modules/circ_policies/api.py @@ -22,10 +22,9 @@ from functools import partial from elasticsearch_dsl import Q -from invenio_search.api import RecordsSearch from .models import CircPolicyIdentifier -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..libraries.api import Library from ..minters import id_minter @@ -43,13 +42,14 @@ circ_policy_id_fetcher = partial(id_fetcher, provider=CircPolicyProvider) -class CircPoliciesSearch(RecordsSearch): +class CircPoliciesSearch(IlsRecordsSearch): """RecordsSearch for Circulation policies.""" class Meta: """Search only on Circulation policies index.""" index = 'circ_policies' + doc_types = None class CircPolicy(IlsRecord): @@ -225,3 +225,9 @@ def reasons_not_to_delete(self): if links: cannot_delete['links'] = links return cannot_delete + + +class CircPoliciesIndexer(IlsRecordsIndexer): + """Indexing documents in Elasticsearch.""" + + record_cls = CircPolicy diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py index a2c023fe8f..e1efa3dccd 100644 --- a/rero_ils/modules/cli.py +++ b/rero_ils/modules/cli.py @@ -25,6 +25,7 @@ import logging import multiprocessing import os +import shutil import sys import traceback from collections import OrderedDict @@ -45,6 +46,7 @@ from flask.cli import with_appcontext from flask_security.confirmable import confirm_user from invenio_accounts.cli import commit, users +from invenio_app.factory import static_folder from invenio_db import db from invenio_indexer.tasks import process_bulk_queue from invenio_pidstore.models import PersistentIdentifier, PIDStatus @@ -58,7 +60,7 @@ from pkg_resources import resource_string from werkzeug.local import LocalProxy -from .api import IlsRecordIndexer +from .api import IlsRecordsIndexer from .documents.dojson.contrib.marc21tojson import marc21 from .items.cli import create_items, reindex_items from .loans.cli import create_loans @@ -206,13 +208,13 @@ def init(force): click.secho('Putting templates...', fg='green', bold=True, file=sys.stderr) with click.progressbar( current_search.put_templates(ignore=[400] if force else None), - length=len(current_search.templates.keys())) as bar: + length=len(current_search.templates)) as bar: for response in bar: bar.label = response click.secho('Creating indexes...', fg='green', bold=True, file=sys.stderr) with click.progressbar( current_search.create(ignore=[400] if force else None), - length=current_search.number_of_indexes) as bar: + length=len(current_search.mappings)) as bar: for name, response in bar: bar.label = name @@ -448,6 +450,7 @@ def test_license(file, extension, license_lines, verbose): lines_with_errors = [] lines = [line.rstrip() for line in file] linenbr = 0 + linemaxnbr = len(lines) prefix = extension.get('prefix') line, linenbr = get_line(lines, linenbr, prefix) while lines[linenbr-1].startswith('#!'): @@ -472,6 +475,9 @@ def test_license(file, extension, license_lines, verbose): if verbose: show_diff(linenbr, license_line, line) lines_with_errors.append(linenbr) + # Fix crash while testing a file with only comments. + if linenbr >= linemaxnbr: + continue line, linenbr = get_line(lines, linenbr, prefix) return lines_with_errors @@ -918,7 +924,7 @@ def run(delayed, concurrency, version_type=None, queue=None, process_bulk_queue.apply_async(**celery_kwargs) else: click.secho('Indexing records...', fg='green') - IlsRecordIndexer(version_type=version_type).process_bulk_queue( + IlsRecordsIndexer(version_type=version_type).process_bulk_queue( es_bulk_kwargs={'raise_on_error': raise_on_error}) @@ -947,7 +953,7 @@ def reindex(pid_type, no_info): ).values( PersistentIdentifier.object_uuid )) - IlsRecordIndexer().bulk_index(query, doc_type=type) + IlsRecordsIndexer().bulk_index(query, doc_type=type) if no_info: click.secho( 'Execute "runindex" command to process the queue!', @@ -1523,3 +1529,26 @@ def export(verbose, pid_type, outfile, pidfile, indent, schema): click.echo('ERROR: Can not export pid:{pid}'.format(pid=pid)) outfile.write(output) outfile.write('\n]\n') + + +@utils.command('set_test_static_folder') +@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) +@with_appcontext +def set_test_static_folder(verbose): + """Creates a static folder link for tests.""" + click.secho('Create symlink for static folder', fg='green') + test_static_folder = os.path.join(sys.prefix, 'var/instance/static') + my_static_folder = static_folder() + if verbose: + msg = '\t{src} --> {dst}'.format( + src=my_static_folder, + dst=test_static_folder + ) + click.secho(msg) + try: + os.unlink(test_static_folder) + except: + pass + os.makedirs(test_static_folder, exist_ok=True) + shutil.rmtree(test_static_folder, ignore_errors=True) + os.symlink(my_static_folder, test_static_folder) diff --git a/rero_ils/modules/documents/api.py b/rero_ils/modules/documents/api.py index 9df8fca55a..e5262b13f2 100644 --- a/rero_ils/modules/documents/api.py +++ b/rero_ils/modules/documents/api.py @@ -28,7 +28,7 @@ from .utils import edition_format_text, publication_statement_text, \ series_format_text, title_format_text_head from ..acq_order_lines.api import AcqOrderLinesSearch -from ..api import IlsRecord, IlsRecordIndexer +from ..api import IlsRecord, IlsRecordsIndexer from ..fetchers import id_fetcher from ..minters import id_minter from ..organisations.api import Organisation @@ -46,16 +46,6 @@ document_id_fetcher = partial(id_fetcher, provider=DocumentProvider) -class DocumentsIndexer(IlsRecordIndexer): - """Indexing documents in Elasticsearch.""" - - def index(self, record): - """Index an document.""" - return_value = super(DocumentsIndexer, self).index(record) - record.index_persons() - return return_value - - class DocumentsSearch(RecordsSearch): """DocumentsSearch.""" @@ -63,6 +53,7 @@ class Meta: """Search only on documents index.""" index = 'documents' + doc_types = None class Document(IlsRecord): @@ -71,7 +62,6 @@ class Document(IlsRecord): minter = document_id_minter fetcher = document_id_fetcher provider = DocumentProvider - indexer = DocumentsIndexer def is_available(self, view_code): """Get availability for document.""" @@ -178,10 +168,22 @@ def index_persons(self, bulk=False): auth_pid = author.get('pid') if auth_pid: from ..persons.api import Person - person = Person.get_record_by_pid(auth_pid) + person = Person.get_record_by_mef_pid(auth_pid) if bulk: persons_ids.append(person.id) else: person.reindex() if persons_ids: - IlsRecordIndexer().bulk_index(persons_ids, doc_type=['pers']) + IlsRecordsIndexer().bulk_index(persons_ids, doc_type=['pers']) + + +class DocumentsIndexer(IlsRecordsIndexer): + """Indexing documents in Elasticsearch.""" + + record_cls = Document + + def index(self, record): + """Index an document.""" + return_value = super(DocumentsIndexer, self).index(record) + record.index_persons() + return return_value diff --git a/rero_ils/modules/documents/mappings/v6/documents/document.json b/rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json similarity index 99% rename from rero_ils/modules/documents/mappings/v6/documents/document.json rename to rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json index e8eb26e018..191ee14d21 100644 --- a/rero_ils/modules/documents/mappings/v6/documents/document.json +++ b/rero_ils/modules/documents/mappings/v6/documents/document-v0.0.1.json @@ -24,7 +24,7 @@ } }, "mappings": { - "document": { + "document-v0.0.1": { "date_detection": false, "numeric_detection": false, "properties": { diff --git a/rero_ils/modules/holdings/api.py b/rero_ils/modules/holdings/api.py index ef2ac78510..41eea1cb54 100644 --- a/rero_ils/modules/holdings/api.py +++ b/rero_ils/modules/holdings/api.py @@ -28,7 +28,7 @@ from invenio_search.api import RecordsSearch from .models import HoldingIdentifier -from ..api import IlsRecord, IlsRecordIndexer +from ..api import IlsRecord, IlsRecordsIndexer from ..errors import MissingRequiredParameterError from ..fetchers import id_fetcher from ..items.api import Item, ItemsSearch @@ -56,6 +56,7 @@ class Meta: """Search only on holdings index.""" index = 'holdings' + doc_types = None @classmethod def flush(cls): @@ -63,16 +64,6 @@ def flush(cls): current_search.flush_and_refresh(cls.Meta.index) -class HoldingsIndexer(IlsRecordIndexer): - """Holdings indexing class.""" - - def index(self, record): - """Indexing a holding record.""" - return_value = super(HoldingsIndexer, self).index(record) - # current_search.flush_and_refresh(HoldingsSearch.Meta.index) - return return_value - - class Holding(IlsRecord): """Holding class.""" @@ -80,7 +71,6 @@ class Holding(IlsRecord): fetcher = holding_id_fetcher provider = HoldingProvider # model_cls = HoldingMetadata - indexer = HoldingsIndexer def delete_from_index(self): """Delete record from index.""" @@ -321,3 +311,15 @@ def create_holding( record = Holding.create( data, dbcommit=True, reindex=True, delete_pid=True) return record.get('pid') + + +class HoldingsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Holding + + def index(self, record): + """Indexing a holding record.""" + return_value = super(HoldingsIndexer, self).index(record) + # current_search.flush_and_refresh(HoldingsSearch.Meta.index) + return return_value diff --git a/rero_ils/modules/indexer_utils.py b/rero_ils/modules/indexer_utils.py index 108c3607d7..e9ae3597ea 100644 --- a/rero_ils/modules/indexer_utils.py +++ b/rero_ils/modules/indexer_utils.py @@ -20,8 +20,8 @@ import re from flask import current_app +from invenio_indexer.utils import schema_to_index from invenio_search import current_search -from invenio_search.utils import schema_to_index def record_to_index(record): @@ -39,8 +39,11 @@ def record_to_index(record): schema = schema.get('$ref', '') # put all document in the same index + # 'document-minimal-v0.0.1.json' becomes 'document-v0.0.1.json' if re.search(r'/documents/', schema): - schema = re.sub(r'-.*\.json', '.json', schema) + schema = re.sub( + r'/document(?P-\D+)?(?P-v[\d,\.]+).json', + r'/document\g.json', schema) # authorities specific transformation if re.search(r'/authorities/', schema): schema = re.sub(r'/authorities/', '/persons/', schema) diff --git a/rero_ils/modules/item_types/api.py b/rero_ils/modules/item_types/api.py index cf187c98bd..1cef40e34a 100644 --- a/rero_ils/modules/item_types/api.py +++ b/rero_ils/modules/item_types/api.py @@ -23,10 +23,9 @@ from elasticsearch_dsl import Q from flask_babelex import gettext as _ -from invenio_search.api import RecordsSearch from .models import ItemTypeIdentifier -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..circ_policies.api import CircPoliciesSearch from ..fetchers import id_fetcher from ..minters import id_minter @@ -44,13 +43,14 @@ item_type_id_fetcher = partial(id_fetcher, provider=ItemTypeProvider) -class ItemTypesSearch(RecordsSearch): +class ItemTypesSearch(IlsRecordsSearch): """ItemTypeSearch.""" class Meta: """Search only on item_types index.""" index = 'item_types' + doc_types = None class ItemType(IlsRecord): @@ -155,3 +155,9 @@ def reasons_not_to_delete(self): if links: cannot_delete['links'] = links return cannot_delete + + +class ItemTypesIndexer(IlsRecordsIndexer): + """Indexing documents in Elasticsearch.""" + + record_cls = ItemType diff --git a/rero_ils/modules/items/api.py b/rero_ils/modules/items/api.py index 37c77470d2..aae1326474 100644 --- a/rero_ils/modules/items/api.py +++ b/rero_ils/modules/items/api.py @@ -32,7 +32,8 @@ from invenio_search import current_search from .models import ItemIdentifier, ItemStatus -from ..api import IlsRecord, IlsRecordError, IlsRecordIndexer, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordError, IlsRecordsIndexer, \ + IlsRecordsSearch from ..circ_policies.api import CircPolicy from ..documents.api import Document, DocumentsSearch from ..errors import InvalidRecordID @@ -60,65 +61,6 @@ item_id_fetcher = partial(id_fetcher, provider=ItemProvider) -class ItemsIndexer(IlsRecordIndexer): - """Indexing items in Elasticsearch.""" - - def index(self, record): - """Index an item.""" - from ..holdings.api import Holding - # get the old holding record if exists - items = ItemsSearch().filter( - 'term', pid=record.get('pid') - ).source().execute().hits - - holding_pid = None - if items.total: - item = items.hits[0]['_source'] - holding_pid = item.get('holding', {}).get('pid') - - return_value = super(ItemsIndexer, self).index(record) - document_pid = record.replace_refs()['document']['pid'] - document = Document.get_record_by_pid(document_pid) - document.reindex() - current_search.flush_and_refresh(DocumentsSearch.Meta.index) - - # check if old holding can be deleted - if holding_pid: - holding_rec = Holding.get_record_by_pid(holding_pid) - try: - # TODO: Need to split DB and elasticsearch deletion. - holding_rec.delete(force=False, dbcommit=True, delindex=True) - except IlsRecordError.NotDeleted: - pass - - return return_value - - def delete(self, record): - """Delete a record. - - :param record: Record instance. - """ - from ..holdings.api import Holding - - return_value = super(ItemsIndexer, self).delete(record) - rec_with_refs = record.replace_refs() - document_pid = rec_with_refs['document']['pid'] - document = Document.get_record_by_pid(document_pid) - document.reindex() - current_search.flush_and_refresh(DocumentsSearch.Meta.index) - - holding = rec_with_refs.get('holding', '') - if holding: - holding_rec = Holding.get_record_by_pid(holding.get('pid')) - try: - # TODO: Need to split DB and elasticsearch deletion. - holding_rec.delete(force=False, dbcommit=True, delindex=True) - except IlsRecordError.NotDeleted: - pass - - return return_value - - class ItemsSearch(IlsRecordsSearch): """ItemsSearch.""" @@ -126,6 +68,7 @@ class Meta: """Search only on item index.""" index = 'items' + doc_types = None @classmethod def flush(cls): @@ -213,7 +156,6 @@ class Item(IlsRecord): minter = item_id_minter fetcher = item_id_fetcher provider = ItemProvider - indexer = ItemsIndexer statuses = { 'ITEM_ON_LOAN': 'on_loan', @@ -1035,3 +977,64 @@ def dumps(self, **kwargs): dump = super(Item, self).dumps(**kwargs) dump['available'] = self.available return dump + + +class ItemsIndexer(IlsRecordsIndexer): + """Indexing items in Elasticsearch.""" + + record_cls = Item + + def index(self, record): + """Index an item.""" + from ..holdings.api import Holding + # get the old holding record if exists + items = ItemsSearch().filter( + 'term', pid=record.get('pid') + ).source().execute().hits + + holding_pid = None + if items.total: + item = items.hits[0]['_source'] + holding_pid = item.get('holding', {}).get('pid') + + return_value = super(ItemsIndexer, self).index(record) + document_pid = record.replace_refs()['document']['pid'] + document = Document.get_record_by_pid(document_pid) + document.reindex() + current_search.flush_and_refresh(DocumentsSearch.Meta.index) + + # check if old holding can be deleted + if holding_pid: + holding_rec = Holding.get_record_by_pid(holding_pid) + try: + # TODO: Need to split DB and elasticsearch deletion. + holding_rec.delete(force=False, dbcommit=True, delindex=True) + except IlsRecordError.NotDeleted: + pass + + return return_value + + def delete(self, record): + """Delete a record. + + :param record: Record instance. + """ + from ..holdings.api import Holding + + return_value = super(ItemsIndexer, self).delete(record) + rec_with_refs = record.replace_refs() + document_pid = rec_with_refs['document']['pid'] + document = Document.get_record_by_pid(document_pid) + document.reindex() + current_search.flush_and_refresh(DocumentsSearch.Meta.index) + + holding = rec_with_refs.get('holding', '') + if holding: + holding_rec = Holding.get_record_by_pid(holding.get('pid')) + try: + # TODO: Need to split DB and elasticsearch deletion. + holding_rec.delete(force=False, dbcommit=True, delindex=True) + except IlsRecordError.NotDeleted: + pass + + return return_value diff --git a/rero_ils/modules/libraries/api.py b/rero_ils/modules/libraries/api.py index 4f3a2ad75b..f9acfd0f3d 100644 --- a/rero_ils/modules/libraries/api.py +++ b/rero_ils/modules/libraries/api.py @@ -23,10 +23,9 @@ import pytz from dateutil import parser from dateutil.rrule import FREQNAMES, rrule -from invenio_search.api import RecordsSearch from .models import LibraryIdentifier -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..locations.api import LocationsSearch from ..minters import id_minter @@ -50,13 +49,14 @@ class LibraryNeverOpen(Exception): """Raised when the library has no open days.""" -class LibrariesSearch(RecordsSearch): +class LibrariesSearch(IlsRecordsSearch): """Libraries search.""" class Meta(): """Meta class.""" index = 'libraries' + doc_types = None class Library(IlsRecord): @@ -313,3 +313,9 @@ def get_timezone(self): # TODO: Use BABEL_DEFAULT_TIMEZONE by default default = pytz.timezone('Europe/Zurich') return default + + +class LibrariesIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Library diff --git a/rero_ils/modules/loans/api.py b/rero_ils/modules/loans/api.py index 500e08530e..63ac03948d 100644 --- a/rero_ils/modules/loans/api.py +++ b/rero_ils/modules/loans/api.py @@ -30,7 +30,7 @@ from invenio_circulation.search.api import search_by_patron_item_or_document from invenio_jsonschemas import current_jsonschemas -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..documents.api import Document from ..libraries.api import Library from ..locations.api import Location @@ -54,6 +54,16 @@ class LoanAction(object): NO = 'no' +class LoansSearch(IlsRecordsSearch): + """Libraries search.""" + + class Meta(): + """Meta class.""" + + index = 'loans' + doc_types = None + + class Loan(IlsRecord): """Loan class.""" @@ -364,3 +374,9 @@ def get_overdue_loans(): if now > due_date + timedelta(days=days_after): overdue_loans.append(loan) return overdue_loans + + +class LoansIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Loan diff --git a/rero_ils/modules/locations/api.py b/rero_ils/modules/locations/api.py index a31858c936..0f61c0788d 100644 --- a/rero_ils/modules/locations/api.py +++ b/rero_ils/modules/locations/api.py @@ -20,10 +20,9 @@ from functools import partial from flask_babelex import gettext as _ -from invenio_search.api import RecordsSearch from .models import LocationIdentifier -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..minters import id_minter from ..providers import Provider @@ -40,13 +39,14 @@ location_id_fetcher = partial(id_fetcher, provider=LocationProvider) -class LocationsSearch(RecordsSearch): +class LocationsSearch(IlsRecordsSearch): """RecordsSearch for locations.""" class Meta: """Search only on locations index.""" index = 'locations' + doc_types = None class Location(IlsRecord): @@ -132,3 +132,9 @@ def organisation_pid(self): library = Library.get_record_by_pid(self.library_pid) return library.organisation_pid + + +class LocationsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Location diff --git a/rero_ils/modules/notifications/api.py b/rero_ils/modules/notifications/api.py index 453b2e9c60..0f23198da4 100644 --- a/rero_ils/modules/notifications/api.py +++ b/rero_ils/modules/notifications/api.py @@ -25,11 +25,10 @@ import ciso8601 from flask import current_app -from invenio_search.api import RecordsSearch from .dispatcher import Dispatcher from .models import NotificationIdentifier, NotificationMetadata -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..circ_policies.api import CircPolicy from ..documents.api import Document from ..fetchers import id_fetcher @@ -53,13 +52,14 @@ notification_id_fetcher = partial(id_fetcher, provider=NotificationProvider) -class NotificationsSearch(RecordsSearch): +class NotificationsSearch(IlsRecordsSearch): """RecordsSearch for Notifications.""" class Meta: """Search only on Notifications index.""" index = 'notifications' + doc_types = None class Notification(IlsRecord): @@ -271,6 +271,12 @@ def patron_transactions(self): yield PatronTransaction.get_record_by_pid(result.pid) +class NotificationsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Notification + + def get_availability_notification(loan): """Returns availability notification from loan.""" results = NotificationsSearch().filter( diff --git a/rero_ils/modules/organisations/api.py b/rero_ils/modules/organisations/api.py index 37ed7fa270..e836bdddd4 100644 --- a/rero_ils/modules/organisations/api.py +++ b/rero_ils/modules/organisations/api.py @@ -20,10 +20,8 @@ from functools import partial -from invenio_search.api import RecordsSearch - from .models import OrganisationIdentifier -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..item_types.api import ItemTypesSearch from ..libraries.api import LibrariesSearch, Library @@ -42,13 +40,14 @@ organisation_id_fetcher = partial(id_fetcher, provider=OrganisationProvider) -class OrganisationSearch(RecordsSearch): +class OrganisationsSearch(IlsRecordsSearch): """Organisation search.""" class Meta(): """Meta class.""" index = 'organisations' + doc_types = None class Organisation(IlsRecord): @@ -104,7 +103,7 @@ def all_code(cls): @classmethod def get_record_by_viewcode(cls, viewcode): """Get record by view code.""" - result = OrganisationSearch().filter( + result = OrganisationsSearch().filter( 'term', code=viewcode ).execute() @@ -132,7 +131,7 @@ def online_circulation_category(self): @classmethod def get_record_by_online_harvested_source(cls, source): """Get record by online harvested source.""" - results = OrganisationSearch().filter( + results = OrganisationsSearch().filter( 'term', online_harvested_source=source).scan() try: return Organisation.get_record_by_pid(next(results).pid) @@ -143,3 +142,9 @@ def get_online_locations(self): """Get list of online locations.""" return [library.online_location for library in self.get_libraries() if library.online_location] + + +class OrganisationsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Organisation diff --git a/rero_ils/modules/patron_transaction_events/api.py b/rero_ils/modules/patron_transaction_events/api.py index 49eba75314..e1619cd9c2 100644 --- a/rero_ils/modules/patron_transaction_events/api.py +++ b/rero_ils/modules/patron_transaction_events/api.py @@ -23,7 +23,7 @@ from flask import current_app from .models import PatronTransactionEventIdentifier -from ..api import IlsRecord, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..minters import id_minter from ..providers import Provider @@ -49,6 +49,7 @@ class Meta: """Search only on patron_transaction_event index.""" index = 'patron_transaction_events' + doc_types = None class PatronTransactionEvent(IlsRecord): @@ -173,3 +174,9 @@ def build_patron_transaction_event_ref(patron_transaction, data): data['amount'] = patron_transaction.get('total_amount') return data + + +class PatronTransactionEventsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = PatronTransactionEvent diff --git a/rero_ils/modules/patron_transactions/api.py b/rero_ils/modules/patron_transactions/api.py index c849d93312..359f4b7e84 100644 --- a/rero_ils/modules/patron_transactions/api.py +++ b/rero_ils/modules/patron_transactions/api.py @@ -23,7 +23,7 @@ from flask import current_app from .models import PatronTransactionIdentifier -from ..api import IlsRecord, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..minters import id_minter from ..organisations.api import Organisation @@ -52,6 +52,7 @@ class Meta: """Search only on patron transaction index.""" index = 'patron_transactions' + doc_types = None class PatronTransaction(IlsRecord): @@ -222,3 +223,9 @@ def build_patron_transaction_ref(notification, data): } data['total_amount'] = calculate_overdue_amount(notification) return data + + +class PatronTransactionsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = PatronTransaction diff --git a/rero_ils/modules/patron_types/api.py b/rero_ils/modules/patron_types/api.py index bc5ac42f67..3f4a9dcdc1 100644 --- a/rero_ils/modules/patron_types/api.py +++ b/rero_ils/modules/patron_types/api.py @@ -22,10 +22,9 @@ from functools import partial from elasticsearch_dsl import Q -from invenio_search.api import RecordsSearch from .models import PatronTypeIdentifier -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..circ_policies.api import CircPoliciesSearch from ..fetchers import id_fetcher from ..minters import id_minter @@ -44,13 +43,14 @@ patron_type_id_fetcher = partial(id_fetcher, provider=PatronTypeProvider) -class PatronTypesSearch(RecordsSearch): +class PatronTypesSearch(IlsRecordsSearch): """PatronTypeSearch.""" class Meta: """Search only on patrons index.""" index = 'patron_types' + doc_types = None class PatronType(IlsRecord): @@ -117,3 +117,9 @@ def reasons_not_to_delete(self): if links: cannot_delete['links'] = links return cannot_delete + + +class PatronTypesIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = PatronType diff --git a/rero_ils/modules/patrons/api.py b/rero_ils/modules/patrons/api.py index b6aefaff85..bd1d145b17 100644 --- a/rero_ils/modules/patrons/api.py +++ b/rero_ils/modules/patrons/api.py @@ -25,12 +25,11 @@ from flask_security.recoverable import send_reset_password_instructions from invenio_accounts.ext import hash_password from invenio_circulation.proxies import current_circulation -from invenio_search.api import RecordsSearch from werkzeug.local import LocalProxy from werkzeug.utils import cached_property from .models import PatronIdentifier -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..libraries.api import Library from ..minters import id_minter @@ -54,13 +53,14 @@ patron_id_fetcher = partial(id_fetcher, provider=PatronProvider) -class PatronsSearch(RecordsSearch): +class PatronsSearch(IlsRecordsSearch): """PatronsSearch.""" class Meta: """Search only on patrons index.""" index = 'patrons' + doc_types = None class Patron(IlsRecord): @@ -310,3 +310,9 @@ def organisation_pid(self): patron_type = PatronType.get_record_by_pid(self.patron_type_pid) return patron_type.organisation_pid return None + + +class PatronsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Patron diff --git a/rero_ils/modules/persons/api.py b/rero_ils/modules/persons/api.py index 69c5f62bab..475eb4ed74 100644 --- a/rero_ils/modules/persons/api.py +++ b/rero_ils/modules/persons/api.py @@ -22,12 +22,11 @@ from elasticsearch_dsl import A from flask import current_app from invenio_db import db -from invenio_search.api import RecordsSearch from requests import codes as requests_codes from requests import get as requests_get from .models import PersonIdentifier -from ..api import IlsRecord +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..documents.api import DocumentsSearch from ..fetchers import id_fetcher from ..minters import id_minter @@ -46,13 +45,14 @@ person_id_fetcher = partial(id_fetcher, provider=PersonProvider) -class PersonsSearch(RecordsSearch): +class PersonsSearch(IlsRecordsSearch): """Mef person search.""" class Meta(): """Meta class.""" index = 'persons' + doc_types = None class Person(IlsRecord): @@ -189,3 +189,9 @@ def organisation_pids(self): if result.doc_count: organisations.add(result.key) return list(organisations) + + +class PersonsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Person diff --git a/rero_ils/modules/tasks.py b/rero_ils/modules/tasks.py new file mode 100644 index 0000000000..c1117e6350 --- /dev/null +++ b/rero_ils/modules/tasks.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2019 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Celery tasks to index records.""" + +from celery import shared_task + +from .api import IlsRecordsIndexer + + +@shared_task(ignore_result=True) +def process_bulk_queue(version_type=None, es_bulk_kwargs=None): + """Process bulk indexing queue. + + :param str version_type: Elasticsearch version type. + :param dict es_bulk_kwargs: Passed to + :func:`elasticsearch:elasticsearch.helpers.bulk`. + Note: You can start multiple versions of this task. + """ + IlsRecordsIndexer(version_type=version_type).process_bulk_queue( + es_bulk_kwargs=es_bulk_kwargs) + + +@shared_task(ignore_result=True) +def index_record(record_uuid): + """Index a single record. + + :param record_uuid: The record UUID. + """ + IlsRecordsIndexer().index_by_id(record_uuid) + + +@shared_task(ignore_result=True) +def delete_record(record_uuid): + """Delete a single record. + + :param record_uuid: The record UUID. + """ + IlsRecordsIndexer().delete_by_id(record_uuid) diff --git a/rero_ils/modules/utils.py b/rero_ils/modules/utils.py index bf848bd7c3..ffa3092008 100644 --- a/rero_ils/modules/utils.py +++ b/rero_ils/modules/utils.py @@ -27,7 +27,7 @@ from flask import current_app from invenio_records_rest.utils import obj_or_import_string -from .api import IlsRecordIndexer +from .api import IlsRecordsIndexer def strtotime(strtime): @@ -43,7 +43,7 @@ def do_bulk_index(uuids, doc_type='rec', process=False, verbose=False): """Bulk index records.""" if verbose: click.echo(' add to index: {count}'.format(count=len(uuids))) - indexer = IlsRecordIndexer() + indexer = IlsRecordsIndexer() retry = True minutes = 1 while retry: diff --git a/rero_ils/modules/vendors/api.py b/rero_ils/modules/vendors/api.py index 737c8f2016..06f2eda7b3 100644 --- a/rero_ils/modules/vendors/api.py +++ b/rero_ils/modules/vendors/api.py @@ -22,7 +22,7 @@ from .models import VendorIdentifier from ..acq_invoices.api import AcquisitionInvoicesSearch from ..acq_orders.api import AcqOrdersSearch -from ..api import IlsRecord, IlsRecordsSearch +from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch from ..fetchers import id_fetcher from ..minters import id_minter from ..providers import Provider @@ -46,6 +46,7 @@ class Meta: """Search only on vendor index.""" index = 'vendors' + doc_types = None class Vendor(IlsRecord): @@ -84,3 +85,9 @@ def reasons_not_to_delete(self): if links: cannot_delete['links'] = links return cannot_delete + + +class VendorsIndexer(IlsRecordsIndexer): + """Holdings indexing class.""" + + record_cls = Vendor diff --git a/rero_ils/templates/__init__.py b/rero_ils/templates/__init__.py new file mode 100644 index 0000000000..502723629c --- /dev/null +++ b/rero_ils/templates/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2020 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Template module for RERO-ils.""" diff --git a/run-tests.sh b/run-tests.sh index 81cf177869..e7df14008e 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -94,6 +94,7 @@ if [ $# -eq 0 ] info_msg "Sphinx-build:" pipenv run sphinx-build -qnNW docs docs/_build/html info_msg "Tests:" + # pipenv run invenio utils set_test_static_folder -v pipenv run test fi if [ "$1" = "external" ] diff --git a/scripts/setup b/scripts/setup index a3df430c20..5eb68a0b0d 100755 --- a/scripts/setup +++ b/scripts/setup @@ -170,6 +170,7 @@ eval ${PREFIX} invenio index queue purge delete set -e eval ${PREFIX} invenio index destroy --force --yes-i-know # Override index init to load templates before mapping +# TODO: check if invenio index init --force works (to delete utils init --force) info_msg "Override index init to load templates before mapping" eval ${PREFIX} invenio utils init --force # eval ${PREFIX} invenio index init --force @@ -285,14 +286,14 @@ fi if ${LOADPERSONS} then info_msg "- PERSONS: ${DOCUMENTS} ${DONT_STOP}" - eval ${PREFIX} pipenv run invenio fixtures create --pid_type pers --schema 'http://ils.rero.ch/schema/persons/person-v0.0.1.json' ${PERSONS} --append ${DONT_STOP} + eval ${PREFIX} pipenv run invenio fixtures create --pid_type pers --schema 'http://ils.rero.ch/schema/persons/person-v0.0.1.json' ${PERSONS} --append ${CREATE_LAZY} ${DONT_STOP} info_msg "Indexing Persons:" eval ${PREFIX} pipenv run invenio utils reindex -t pers --yes-i-know --no-info eval ${PREFIX} pipenv run invenio utils runindex --raise-on-error fi info_msg "- Documents: ${DOCUMENTS} ${DONT_STOP}" -eval ${PREFIX} invenio fixtures create --pid_type doc --schema 'http://ils.rero.ch/schema/documents/document-v0.0.1.json' ${DOCUMENTS} --append ${DONT_STOP} +eval ${PREFIX} invenio fixtures create --pid_type doc --schema 'http://ils.rero.ch/schema/documents/document-v0.0.1.json' ${DOCUMENTS} --append ${CREATE_LAZY} ${DONT_STOP} if ${CREATE_ITEMS_HOLDINGS_SMALL} then @@ -334,10 +335,6 @@ eval ${PREFIX} invenio utils runindex --raise-on-error eval ${PREFIX} invenio utils reindex -t doc --yes-i-know --no-info eval ${PREFIX} invenio utils runindex --raise-on-error -# index persons -eval ${PREFIX} invenio utils reindex -t pers --yes-i-know --no-info -eval ${PREFIX} invenio utils runindex --raise-on-error - # create circulation transactions info_msg "Circulation transactions:" eval ${PREFIX} invenio fixtures create_loans ${DATA_PATH}/loans.json @@ -345,18 +342,19 @@ eval ${PREFIX} invenio fixtures create_loans ${DATA_PATH}/loans.json # # ACQUISITION # create vendors info_msg "Acquisition vendors:" -eval ${PREFIX} invenio fixtures create --pid_type vndr ${DATA_PATH}/vendors.json --append +eval ${PREFIX} invenio fixtures create --pid_type vndr ${DATA_PATH}/vendors.json --append ${CREATE_LAZY} ${DONT_STOP} eval ${PREFIX} invenio utils reindex -t vndr --yes-i-know --no-info # create library budgets info_msg "Library budgets:" -eval ${PREFIX} invenio fixtures create --pid_type budg ${DATA_PATH}/budgets.json --append +eval ${PREFIX} invenio fixtures create --pid_type budg ${DATA_PATH}/budgets.json --append ${CREATE_LAZY} ${DONT_STOP} eval ${PREFIX} invenio utils reindex -t budg --yes-i-know --no-info # create acquisition accounts info_msg "Acquisition accounts:" -eval ${PREFIX} invenio fixtures create --pid_type acac ${DATA_PATH}/acq_accounts.json --append +eval ${PREFIX} invenio fixtures create --pid_type acac ${DATA_PATH}/acq_accounts.json --append ${CREATE_LAZY} ${DONT_STOP} eval ${PREFIX} invenio utils reindex -t acac --yes-i-know --no-info + eval ${PREFIX} invenio utils runindex --raise-on-error # # OAI configuration diff --git a/setup.py b/setup.py index 3a07fdf282..dec3255a4d 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,7 @@ def run(self): 'items = rero_ils.modules.items.api_views:api_blueprint', 'persons = rero_ils.modules.persons.views:api_blueprint', 'holdings = rero_ils.modules.holdings.api_views:api_blueprint', - 'monitoring = rero_ils.modules.monitoring:api_blueprint' + 'monitoring = rero_ils.modules.monitoring:api_blueprint', ], 'invenio_config.module': [ 'rero_ils = rero_ils.config', @@ -144,7 +144,7 @@ def run(self): 'utils = rero_ils.modules.cli:utils', 'oaiharvester = rero_ils.modules.ebooks.cli:oaiharvester', 'apiharvester = rero_ils.modules.apiharvester.cli:apiharvester', - 'monitoring = rero_ils.modules.monitoring:monitoring' + 'monitoring = rero_ils.modules.monitoring:monitoring', ], 'invenio_db.models': [ 'organisations = rero_ils.modules.organisations.models', @@ -167,7 +167,7 @@ def run(self): 'acq_order_lines = rero_ils.modules.acq_order_lines.models', 'acq_invoices = rero_ils.modules.acq_invoices.models', 'patron_transactions = rero_ils.modules.patron_transactions.models', - 'patron_transaction_events = rero_ils.modules.patron_transaction_events.models' + 'patron_transaction_events = rero_ils.modules.patron_transaction_events.models', ], 'invenio_pidstore.minters': [ 'organisation_id = rero_ils.modules.organisations.api:organisation_id_minter', @@ -192,8 +192,7 @@ def run(self): 'patron_transaction_event_id = rero_ils.modules.patron_transaction_events.api:patron_transaction_event_id_minter' ], 'invenio_pidstore.fetchers': [ - 'organisation_id = rero_ils.modules.organisations' - '.api:organisation_id_fetcher', + 'organisation_id = rero_ils.modules.organisations.api:organisation_id_fetcher', 'library_id = rero_ils.modules.libraries.api:library_id_fetcher', 'location_id = rero_ils.modules.locations.api:location_id_fetcher', 'document_id = rero_ils.modules.documents.api:document_id_fetcher', @@ -212,7 +211,7 @@ def run(self): 'acq_order_line_id = rero_ils.modules.acq_order_lines.api:acq_order_line_id_fetcher', 'acq_invoice_id = rero_ils.modules.acq_invoices.api:acq_invoice_id_fetcher', 'patron_transaction_id = rero_ils.modules.patron_transactions.api:patron_transaction_id_fetcher', - 'patron_transaction_event_id = rero_ils.modules.patron_transaction_events.api:patron_transaction_event_id_fetcher' + 'patron_transaction_event_id = rero_ils.modules.patron_transaction_events.api:patron_transaction_event_id_fetcher', ], 'invenio_jsonschemas.schemas': [ 'organisations = rero_ils.modules.organisations.jsonschemas', @@ -235,7 +234,7 @@ def run(self): 'acq_order_lines = rero_ils.modules.acq_order_lines.jsonschemas', 'acq_invoices = rero_ils.modules.acq_invoices.jsonschemas', 'patron_transactions = rero_ils.modules.patron_transactions.jsonschemas', - 'patron_transaction_events = rero_ils.modules.patron_transaction_events.jsonschemas' + 'patron_transaction_events = rero_ils.modules.patron_transaction_events.jsonschemas', ], 'invenio_search.mappings': [ 'organisations = rero_ils.modules.organisations.mappings', @@ -258,15 +257,16 @@ def run(self): 'acq_order_lines = rero_ils.modules.acq_order_lines.mappings', 'acq_invoices = rero_ils.modules.acq_invoices.mappings', 'patron_transactions = rero_ils.modules.patron_transactions.mappings', - 'patron_transaction_events = rero_ils.modules.patron_transaction_events.mappings' + 'patron_transaction_events = rero_ils.modules.patron_transaction_events.mappings', ], 'invenio_search.templates': [ - 'base-record = rero_ils.es_templates:list_es_templates' + 'base-record = rero_ils.es_templates:list_es_templates', ], 'invenio_celery.tasks': [ - 'rero_ils_oaiharvest = rero_ils.modules.ebooks.tasks', - 'rero_ils_mefharvest = rero_ils.modules.apiharvester.tasks', - 'rero_ils_notifications = rero_ils.modules.notifications.tasks', + 'modules = rero_ils.modules.tasks', + 'ebooks = rero_ils.modules.ebooks.tasks', + 'apiharvester = rero_ils.modules.apiharvester.tasks', + 'notifications = rero_ils.modules.notifications.tasks', ], 'invenio_records.jsonresolver': [ 'organisations = rero_ils.modules.organisations.jsonresolver', diff --git a/tests/conftest.py b/tests/conftest.py index 854d2805e9..b17cbcd457 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,10 @@ """Common pytest fixtures and plugins.""" import json +import os +import shutil +import sys +import tempfile from os.path import dirname, join import pytest @@ -40,18 +44,18 @@ def es(appctx): should used the function-scoped :py:data:`es_clear` fixture to leave the indexes clean for the following tests. """ - from elasticsearch.exceptions import RequestError + from invenio_search.errors import IndexAlreadyExistsError from invenio_search import current_search, current_search_client try: list(current_search.put_templates()) - except RequestError: + except IndexAlreadyExistsError: current_search_client.indices.delete_template('*') list(current_search.put_templates()) try: list(current_search.create()) - except RequestError: + except IndexAlreadyExistsError: list(current_search.delete(ignore=[404])) list(current_search.create()) current_search_client.indices.refresh() @@ -121,3 +125,38 @@ def app_config(app_config): app_config['WIKI_CONTENT_DIR'] = help_test_dir app_config['WIKI_UPLOAD_FOLDER'] = join(help_test_dir, 'files') return app_config + + +@pytest.fixture(scope='module') +def instance_path(): + """Temporary instance path. + + Scope: module + + This fixture creates a temporary directory if the + environment variable ``INVENIO_INSTANCE_PATH`` is not be set. + This directory is then automatically removed. + """ + invenio_instance_path = os.environ.get('INVENIO_INSTANCE_PATH') + invenio_static_folder = os.environ.get('INVENIO_STATIC_FOLDER') + path = invenio_instance_path + # static folder + if not invenio_static_folder: + if invenio_instance_path: + os.environ['INVENIO_STATIC_FOLDER'] = os.path.join( + invenio_instance_path, 'static') + else: + os.environ['INVENIO_STATIC_FOLDER'] = os.path.join( + sys.prefix, 'var/instance/static') + # instance path + if not path: + path = tempfile.mkdtemp() + os.environ['INVENIO_INSTANCE_PATH'] = path + yield path + # clean static folder variable + if not invenio_static_folder: + os.environ.pop('INVENIO_STATIC_FOLDER', None) + # clean instance path variable and remove temp dir + if not invenio_instance_path: + os.environ.pop('INVENIO_INSTANCE_PATH', None) + shutil.rmtree(path) diff --git a/tests/fixtures/organisations.py b/tests/fixtures/organisations.py index 9de7b8d3df..9f417aa25a 100644 --- a/tests/fixtures/organisations.py +++ b/tests/fixtures/organisations.py @@ -26,7 +26,8 @@ from rero_ils.modules.item_types.api import ItemType, ItemTypesSearch from rero_ils.modules.libraries.api import LibrariesSearch, Library from rero_ils.modules.locations.api import Location, LocationsSearch -from rero_ils.modules.organisations.api import Organisation, OrganisationSearch +from rero_ils.modules.organisations.api import Organisation, \ + OrganisationsSearch from rero_ils.modules.patron_types.api import PatronType, PatronTypesSearch @@ -44,7 +45,7 @@ def org_martigny(app, org_martigny_data): delete_pid=False, dbcommit=True, reindex=True) - flush_index(OrganisationSearch.Meta.index) + flush_index(OrganisationsSearch.Meta.index) return org @@ -56,7 +57,7 @@ def organisation_temp(app, org_martigny): dbcommit=True, delete_pid=True, reindex=True) - flush_index(OrganisationSearch.Meta.index) + flush_index(OrganisationsSearch.Meta.index) return org @@ -74,7 +75,7 @@ def org_sion(app, org_sion_data): delete_pid=False, dbcommit=True, reindex=True) - flush_index(OrganisationSearch.Meta.index) + flush_index(OrganisationsSearch.Meta.index) return org diff --git a/tests/ui/test_api.py b/tests/ui/test_api.py index 90c2e48646..fbecc42665 100644 --- a/tests/ui/test_api.py +++ b/tests/ui/test_api.py @@ -55,6 +55,7 @@ class Meta: """Search only on test index.""" index = 'records-record-v1.0.0' + doc_types = None class ProviderTest(BaseProvider): diff --git a/tests/ui/test_indexer_utils.py b/tests/ui/test_indexer_utils.py index 07e94be36f..ae57940aa7 100644 --- a/tests/ui/test_indexer_utils.py +++ b/tests/ui/test_indexer_utils.py @@ -27,7 +27,11 @@ def test_record_to_index(app): assert record_to_index({ '$schema': 'https://ils.rero.ch/schema/' 'documents/document-minimal-v0.0.1.json' - }) == ('documents-document', 'document') + }) == ('documents-document-v0.0.1', 'document-v0.0.1') + assert record_to_index({ + '$schema': 'https://ils.rero.ch/schema/' + 'documents/document-v0.0.1.json' + }) == ('documents-document-v0.0.1', 'document-v0.0.1') # for mef-persons assert record_to_index({