diff --git a/.gitignore b/.gitignore index 2cb10742..c74cf805 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ htmlcov/ nosetests.xml coverage.xml *,cover +.pytest_cache # end-to-end testing .e2e_screenshots diff --git a/Pipfile b/Pipfile index a7d5dc98..32ae7891 100644 --- a/Pipfile +++ b/Pipfile @@ -6,13 +6,14 @@ name = "pypi" [packages] Babel = ">=2.4.0" Flask-BabelEx = ">=0.9.3" -invenio = { version = "==3.1.0", extras = ["base", "metadata", "postgresql", "auth", "elasticsearch6" ]} +invenio = {version = "==3.1.0",extras = ["base", "metadata", "postgresql", "auth", "elasticsearch6" ]} # TODO: remove this contraint once it is solved in invenio -raven = { version=">=6.0", extras=["flask"] } +raven = {version = ">=6.0",extras = ["flask"]} uwsgi = ">=2.0" uwsgitop = ">=0.11" uwsgi-tools = ">=1.1.1" lxml = ">=3.5.0,<4.2.6" +pycountry = "*" [dev-packages] Flask-Debugtoolbar = ">=0.10.1" diff --git a/Pipfile.lock b/Pipfile.lock index 3802d93f..e55e77f6 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6b511811dbd7063df6a6ee5ee368bf6ce4fe3ece83474773b82572a6468b5df5" + "sha256": "bfcb953244e29ca5cb45ef55285b95c68ccad396df53c96c6846bf6f2601d527" }, "pipfile-spec": 6, "requires": { @@ -24,10 +24,10 @@ }, "amqp": { "hashes": [ - "sha256:043beb485774ca69718a35602089e524f87168268f0d1ae115f28b88d27f92d7", - "sha256:35a3b5006ca00b21aaeec8ceea07130f07b902dd61bfe42815039835f962f5f1" + "sha256:aa4409446139676943a2eaa27d5f58caf750f4ca5a89f888c452afd86be6a67d", + "sha256:cbb6f87d53cac612a594f982b717cc1c54c6a1e17943a0a0d32dc6cc9e2120c8" ], - "version": "==2.4.2" + "version": "==2.5.0" }, "angular-gettext-babel": { "hashes": [ @@ -45,10 +45,10 @@ }, "arrow": { "hashes": [ - "sha256:002f2315cf4c8404de737c42860441732d339bbc57fee584e2027520e055ecc1", - "sha256:82dd5e13b733787d4eb0fef42d1ee1a99136dc1d65178f70373b3678b3181bfc" + "sha256:03404b624e89ac5e4fc19c52045fa0f3203419fd4dd64f6e8958c522580a574a", + "sha256:41be7ea4c53c2cf57bf30f2d614f60c411160133f7a0a8c49111c30fb7e725b5" ], - "version": "==0.13.2" + "version": "==0.14.2" }, "asn1crypto": { "hashes": [ @@ -114,10 +114,10 @@ }, "certifi": { "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" ], - "version": "==2019.3.9" + "version": "==2019.6.16" }, "cffi": { "hashes": [ @@ -175,27 +175,24 @@ }, "cryptography": { "hashes": [ - "sha256:066f815f1fe46020877c5983a7e747ae140f517f1b09030ec098503575265ce1", - "sha256:210210d9df0afba9e000636e97810117dc55b7157c903a55716bb73e3ae07705", - "sha256:26c821cbeb683facb966045e2064303029d572a87ee69ca5a1bf54bf55f93ca6", - "sha256:2afb83308dc5c5255149ff7d3fb9964f7c9ee3d59b603ec18ccf5b0a8852e2b1", - "sha256:2db34e5c45988f36f7a08a7ab2b69638994a8923853dec2d4af121f689c66dc8", - "sha256:409c4653e0f719fa78febcb71ac417076ae5e20160aec7270c91d009837b9151", - "sha256:45a4f4cf4f4e6a55c8128f8b76b4c057027b27d4c67e3fe157fa02f27e37830d", - "sha256:48eab46ef38faf1031e58dfcc9c3e71756a1108f4c9c966150b605d4a1a7f659", - "sha256:6b9e0ae298ab20d371fc26e2129fd683cfc0cfde4d157c6341722de645146537", - "sha256:6c4778afe50f413707f604828c1ad1ff81fadf6c110cb669579dea7e2e98a75e", - "sha256:8c33fb99025d353c9520141f8bc989c2134a1f76bac6369cea060812f5b5c2bb", - "sha256:9873a1760a274b620a135054b756f9f218fa61ca030e42df31b409f0fb738b6c", - "sha256:9b069768c627f3f5623b1cbd3248c5e7e92aec62f4c98827059eed7053138cc9", - "sha256:9e4ce27a507e4886efbd3c32d120db5089b906979a4debf1d5939ec01b9dd6c5", - "sha256:acb424eaca214cb08735f1a744eceb97d014de6530c1ea23beb86d9c6f13c2ad", - "sha256:c8181c7d77388fe26ab8418bb088b1a1ef5fde058c6926790c8a0a3d94075a4a", - "sha256:d4afbb0840f489b60f5a580a41a1b9c3622e08ecb5eec8614d4fb4cd914c4460", - "sha256:d9ed28030797c00f4bc43c86bf819266c76a5ea61d006cd4078a93ebf7da6bfd", - "sha256:e603aa7bb52e4e8ed4119a58a03b60323918467ef209e6ff9db3ac382e5cf2c6" - ], - "version": "==2.6.1" + "sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c", + "sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643", + "sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216", + "sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799", + "sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a", + "sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9", + "sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc", + "sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8", + "sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53", + "sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1", + "sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609", + "sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292", + "sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e", + "sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6", + "sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed", + "sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d" + ], + "version": "==2.7" }, "decorator": { "hashes": [ @@ -288,10 +285,10 @@ }, "flask-cors": { "hashes": [ - "sha256:7ad56ee3b90d4955148fc25a2ecaa1124fc84298471e266a7fea59aeac4405a5", - "sha256:7e90bf225fdf163d11b84b59fb17594d0580a16b97ab4e1146b1fb2737c1cfec" + "sha256:72170423eb4612f0847318afff8c247b38bd516b7737adfc10d1c2cdbb382d16", + "sha256:f4d97201660e6bbcff2d89d082b5b6d31abee04b1b3003ee073a6fd25ad1d69a" ], - "version": "==3.0.7" + "version": "==3.0.8" }, "flask-kvsession": { "hashes": [ @@ -407,10 +404,10 @@ }, "importlib-metadata": { "hashes": [ - "sha256:027cfc6524613de726789072f95d2e4cc64dd1dee8096d42d13f2ead5bd302f5", - "sha256:0d05199e1f0b1a8707a1b9c46476d4a49807fb56cb1b0737db1d37feb42fe31d" + "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", + "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" ], - "version": "==0.15" + "version": "==0.18" }, "infinity": { "hashes": [ @@ -529,9 +526,9 @@ }, "invenio-indexer": { "hashes": [ - "sha256:f51a7eda14048da000b4a1ee74a137fd33d9b96119daa968af1b2fe6a4421e8c" + "sha256:ca8c705813eb1256625f3c6d1550a0610be693676d4929f2f63e0fcf17879973" ], - "version": "==1.0.1" + "version": "==1.0.2" }, "invenio-jsonschemas": { "hashes": [ @@ -542,10 +539,10 @@ }, "invenio-logging": { "hashes": [ - "sha256:1987d2ce7bb17c910b1e67cc553e191bf5b39c18c4cc118f408bf35ad23f2fd1", - "sha256:bf750a67a71c5d9b30680127baf4c25765fccc6df4dc9a2114ca94eef5672cb0" + "sha256:8c2f6633905d100867814e5894564e79682bca7e45ea2870a6c03a69c4860dec", + "sha256:f3a87ab70993e6c7eb5990e70dda3ab0729b1bfa6c90ecf1f9890018accb926f" ], - "version": "==1.1.0" + "version": "==1.1.1" }, "invenio-mail": { "hashes": [ @@ -671,10 +668,10 @@ }, "jedi": { "hashes": [ - "sha256:2bb0603e3506f708e792c7f4ad8fc2a7a9d9c2d292a358fbbd58da531695595b", - "sha256:2c6bcd9545c7d6440951b12b44d373479bf18123a401a52025cf98563fbd826c" + "sha256:49ccb782651bb6f7009810d17a3316f8867dde31654c750506970742e18b553d", + "sha256:79d0f6595f3846dffcbe667cc6dc821b96e5baa8add125176c31a3917eb19d58" ], - "version": "==0.13.3" + "version": "==0.14.0" }, "jinja2": { "hashes": [ @@ -733,10 +730,10 @@ }, "kombu": { "hashes": [ - "sha256:389ba09e03b15b55b1a7371a441c894fd8121d174f5583bbbca032b9ea8c9edd", - "sha256:7b92303af381ef02fad6899fd5f5a9a96031d781356cd8e505fa54ae5ddee181" + "sha256:55b71d3785def3470a16217fe0780f9e6f95e61bf9ad39ef8dce0177224eab77", + "sha256:eb365ea795cd7e629ba2f1f398e0c3ba354b91ef4de225ffdf6ab45fdfc7d581" ], - "version": "==4.5.0" + "version": "==4.6.3" }, "limits": { "hashes": [ @@ -783,9 +780,9 @@ }, "mako": { "hashes": [ - "sha256:7165919e78e1feb68b4dbe829871ea9941398178fa58e6beedb9ba14acf63965" + "sha256:0cfa65de3a835e87eeca6ac856b3013aade55f49e32515f65d999f91a2324162" ], - "version": "==1.0.10" + "version": "==1.0.12" }, "markupsafe": { "hashes": [ @@ -822,10 +819,10 @@ }, "marshmallow": { "hashes": [ - "sha256:0e497a6447ffaad55578138ca512752de7a48d12f444996ededc3d6bf8a09ca2", - "sha256:e21a4dea20deb167c723e0ffb13f4cf33bcbbeb8a334e92406a3308cedea2826" + "sha256:9cedfc5b6f568d57e8a2cf3d293fbd81b05e5ef557854008d03e25660a39ccfd", + "sha256:a4d99922116a76e5abd8f997ec0519086e24814b7e1e1344bebe2a312ba50235" ], - "version": "==2.19.2" + "version": "==2.19.5" }, "maxminddb": { "hashes": [ @@ -860,10 +857,10 @@ }, "parso": { "hashes": [ - "sha256:17cc2d7a945eb42c3569d4564cdf49bde221bc2b552af3eca9c1aad517dcdd33", - "sha256:2e9574cb12e7112a87253e14e2c380ce312060269d04bd018478a3c92ea9a376" + "sha256:5052bb33be034cba784193e74b1cde6ebf29ae8b8c1e4ad94df0c4209bfc4826", + "sha256:db5881df1643bf3e66c097bfd8935cf03eae73f4cb61ae4433c9ea4fb6613446" ], - "version": "==0.4.0" + "version": "==0.5.0" }, "passlib": { "hashes": [ @@ -911,36 +908,36 @@ }, "psycopg2-binary": { "hashes": [ - "sha256:007ca0df127b1862fc010125bc4100b7a630efc6841047bd11afceadb4754611", - "sha256:03c49e02adf0b4d68f422fdbd98f7a7c547beb27e99a75ed02298f85cb48406a", - "sha256:0a1232cdd314e08848825edda06600455ad2a7adaa463ebfb12ece2d09f3370e", - "sha256:131c80d0958c89273d9720b9adf9df1d7600bb3120e16019a7389ab15b079af5", - "sha256:2de34cc3b775724623f86617d2601308083176a495f5b2efc2bbb0da154f483a", - "sha256:2eddc31500f73544a2a54123d4c4b249c3c711d31e64deddb0890982ea37397a", - "sha256:484f6c62bdc166ee0e5be3aa831120423bf399786d1f3b0304526c86180fbc0b", - "sha256:4c2d9369ed40b4a44a8ccd6bc3a7db6272b8314812d2d1091f95c4c836d92e06", - "sha256:70f570b5fa44413b9f30dbc053d17ef3ce6a4100147a10822f8662e58d473656", - "sha256:7a2b5b095f3bd733aab101c89c0e1a3f0dfb4ebdc26f6374805c086ffe29d5b2", - "sha256:804914a669186e2843c1f7fbe12b55aad1b36d40a28274abe6027deffad9433d", - "sha256:8520c03172da18345d012949a53617a963e0191ccb3c666f23276d5326af27b5", - "sha256:90da901fc33ea393fc644607e4a3916b509387e9339ec6ebc7bfded45b7a0ae9", - "sha256:a582416ad123291a82c300d1d872bdc4136d69ad0b41d57dc5ca3df7ef8e3088", - "sha256:ac8c5e20309f4989c296d62cac20ee456b69c41fd1bc03829e27de23b6fa9dd0", - "sha256:b2cf82f55a619879f8557fdaae5cec7a294fac815e0087c4f67026fdf5259844", - "sha256:b59d6f8cfca2983d8fdbe457bf95d2192f7b7efdb2b483bf5fa4e8981b04e8b2", - "sha256:be08168197021d669b9964bd87628fa88f910b1be31e7010901070f2540c05fd", - "sha256:be0f952f1c365061041bad16e27e224e29615d4eb1fb5b7e7760a1d3d12b90b6", - "sha256:c1c9a33e46d7c12b9c96cf2d4349d783e3127163fd96254dcd44663cf0a1d438", - "sha256:d18c89957ac57dd2a2724ecfe9a759912d776f96ecabba23acb9ecbf5c731035", - "sha256:d7e7b0ff21f39433c50397e60bf0995d078802c591ca3b8d99857ea18a7496ee", - "sha256:da0929b2bf0d1f365345e5eb940d8713c1d516312e010135b14402e2a3d2404d", - "sha256:de24a4962e361c512d3e528ded6c7480eab24c655b8ca1f0b761d3b3650d2f07", - "sha256:e45f93ff3f7dae2202248cf413a87aeb330821bf76998b3cf374eda2fc893dd7", - "sha256:f046aeae1f7a845041b8661bb7a52449202b6c5d3fb59eb4724e7ca088811904", - "sha256:f1dc2b7b2748084b890f5d05b65a47cd03188824890e9a60818721fd492249fb", - "sha256:fcbe7cf3a786572b73d2cd5f34ed452a5f5fac47c9c9d1e0642c457a148f9f88" - ], - "version": "==2.8.2" + "sha256:080c72714784989474f97be9ab0ddf7b2ad2984527e77f2909fcd04d4df53809", + "sha256:110457be80b63ff4915febb06faa7be002b93a76e5ba19bf3f27636a2ef58598", + "sha256:171352a03b22fc099f15103959b52ee77d9a27e028895d7e5fde127aa8e3bac5", + "sha256:19d013e7b0817087517a4b3cab39c084d78898369e5c46258aab7be4f233d6a1", + "sha256:249b6b21ae4eb0f7b8423b330aa80fab5f821b9ffc3f7561a5e2fd6bb142cf5d", + "sha256:2ac0731d2d84b05c7bb39e85b7e123c3a0acd4cda631d8d542802c88deb9e87e", + "sha256:2b6d561193f0dc3f50acfb22dd52ea8c8dfbc64bcafe3938b5f209cc17cb6f00", + "sha256:2bd23e242e954214944481124755cbefe7c2cf563b1a54cd8d196d502f2578bf", + "sha256:3e1239242ca60b3725e65ab2f13765fc199b03af9eaf1b5572f0e97bdcee5b43", + "sha256:3eb70bb697abbe86b1d2b1316370c02ba320bfd1e9e35cf3b9566a855ea8e4e5", + "sha256:51a2fc7e94b98bd1bb5d4570936f24fc2b0541b63eccadf8fdea266db8ad2f70", + "sha256:52f1bdafdc764b7447e393ed39bb263eccb12bfda25a4ac06d82e3a9056251f6", + "sha256:5b3581319a3951f1e866f4f6c5e42023db0fae0284273b82e97dfd32c51985cd", + "sha256:63c1b66e3b2a3a336288e4bcec499e0dc310cd1dceaed1c46fa7419764c68877", + "sha256:8123a99f24ecee469e5c1339427bcdb2a33920a18bb5c0d58b7c13f3b0298ba3", + "sha256:85e699fcabe7f817c0f0a412d4e7c6627e00c412b418da7666ff353f38e30f67", + "sha256:8dbff4557bbef963697583366400822387cccf794ccb001f1f2307ed21854c68", + "sha256:908d21d08d6b81f1b7e056bbf40b2f77f8c499ab29e64ec5113052819ef1c89b", + "sha256:af39d0237b17d0a5a5f638e9dffb34013ce2b1d41441fd30283e42b22d16858a", + "sha256:af51bb9f055a3f4af0187149a8f60c9d516cf7d5565b3dac53358796a8fb2a5b", + "sha256:b2ecac57eb49e461e86c092761e6b8e1fd9654dbaaddf71a076dcc869f7014e2", + "sha256:cd37cc170678a4609becb26b53a2bc1edea65177be70c48dd7b39a1149cabd6e", + "sha256:d17e3054b17e1a6cb8c1140f76310f6ede811e75b7a9d461922d2c72973f583e", + "sha256:d305313c5a9695f40c46294d4315ed3a07c7d2b55e48a9010dad7db7a66c8b7f", + "sha256:dd0ef0eb1f7dd18a3f4187226e226a7284bda6af5671937a221766e6ef1ee88f", + "sha256:e1adff53b56db9905db48a972fb89370ad5736e0450b96f91bcf99cadd96cfd7", + "sha256:f0d43828003c82dbc9269de87aa449e9896077a71954fbbb10a614c017e65737", + "sha256:f78e8b487de4d92640105c1389e5b90be3496b1d75c90a666edd8737cc2dbab7" + ], + "version": "==2.8.3" }, "ptyprocess": { "hashes": [ @@ -949,6 +946,14 @@ ], "version": "==0.6.0" }, + "pycountry": { + "hashes": [ + "sha256:104a8ca94c700898c42a0172da2eab5a5675c49637b729a11db9e1dac2d983cd", + "sha256:8ec4020b2b15cd410893d573820d42ee12fe50365332e58c0975c953b60a16de" + ], + "index": "pypi", + "version": "==18.12.8" + }, "pycparser": { "hashes": [ "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" @@ -1084,9 +1089,9 @@ }, "sqlalchemy": { "hashes": [ - "sha256:c7fef198b43ef31dfd783d094fd5ee435ce8717592e6784c45ba337254998017" + "sha256:c30925d60af95443458ebd7525daf791f55762b106049ae71e18f8dd58084c2f" ], - "version": "==1.3.4" + "version": "==1.3.5" }, "sqlalchemy-continuum": { "hashes": [ @@ -1099,9 +1104,9 @@ "encrypted" ], "hashes": [ - "sha256:3f1cb542cf0549a0de508d4919f3ad693a36230bf4cd13fdd6253549fec71182" + "sha256:0ebd4d176a5786233db9f2e92040476fcff8b1b426fdbbb7ee4f478280ee9166" ], - "version": "==0.33.11" + "version": "==0.34.0" }, "traitlets": { "hashes": [ @@ -1175,11 +1180,11 @@ }, "webargs": { "hashes": [ - "sha256:34ccbd72bf76fd5779e1092b75497bf0b03ca19e6c1784a4f49e2b2bf2e6133a", - "sha256:3d707434a73abe1fca231935c41e6743eae16fc7888a4bfd666145af1554464d", - "sha256:888f3e7e0b74b760732388da4e73640fcab452045d78268c31fb610b3cb397b9" + "sha256:6b81ce44572d4f345104aa41c734fdc01165f054a061a8ebb1b46e89851e1170", + "sha256:713bd63440ee078ce48ca953d254d51e5f1a6fa0c76fb521fc596306c78d95a5", + "sha256:e2394ea7e422c1e795681cee5e8b1c6083bab7db6d7a380841130cbbae173d29" ], - "version": "==5.3.1" + "version": "==5.3.2" }, "webassets": { "hashes": [ @@ -1280,10 +1285,10 @@ }, "certifi": { "hashes": [ - "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", - "sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae" + "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", + "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" ], - "version": "==2019.3.9" + "version": "==2019.6.16" }, "chardet": { "hashes": [ @@ -1294,11 +1299,11 @@ }, "check-manifest": { "hashes": [ - "sha256:d3ef229a2629024f082922faa2173ceaf19390b7aa3c63bf0eea000dbee8a6dc", - "sha256:dba4749e3874d8f83eb4f0814c9c331fcade6c6325f885deb7b14aa1487fb323" + "sha256:8754cc8efd7c062a3705b442d1c23ff702d4477b41a269c2e354b25e1f5535a4", + "sha256:a4c555f658a7c135b8a22bd26c2e55cfaf5876e4d5962d8c25652f2addd556bc" ], "index": "pypi", - "version": "==0.38" + "version": "==0.39" }, "click": { "hashes": [ @@ -1390,10 +1395,10 @@ }, "importlib-metadata": { "hashes": [ - "sha256:027cfc6524613de726789072f95d2e4cc64dd1dee8096d42d13f2ead5bd302f5", - "sha256:0d05199e1f0b1a8707a1b9c46476d4a49807fb56cb1b0737db1d37feb42fe31d" + "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", + "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" ], - "version": "==0.15" + "version": "==0.18" }, "isort": { "hashes": [ @@ -1452,10 +1457,10 @@ }, "marshmallow": { "hashes": [ - "sha256:0e497a6447ffaad55578138ca512752de7a48d12f444996ededc3d6bf8a09ca2", - "sha256:e21a4dea20deb167c723e0ffb13f4cf33bcbbeb8a334e92406a3308cedea2826" + "sha256:9cedfc5b6f568d57e8a2cf3d293fbd81b05e5ef557854008d03e25660a39ccfd", + "sha256:a4d99922116a76e5abd8f997ec0519086e24814b7e1e1344bebe2a312ba50235" ], - "version": "==2.19.2" + "version": "==2.19.5" }, "mock": { "hashes": [ @@ -1526,11 +1531,11 @@ }, "pytest": { "hashes": [ - "sha256:1a8aa4fa958f8f451ac5441f3ac130d9fc86ea38780dd2715e6d5c5882700b24", - "sha256:b8bf138592384bd4e87338cb0f256bf5f615398a649d4bd83915f0e4047a5ca6" + "sha256:4a784f1d4f2ef198fe9b7aef793e9fa1a3b2f84e822d9b3a64a181293a572d45", + "sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da" ], "index": "pypi", - "version": "==4.5.0" + "version": "==4.6.3" }, "pytest-cache": { "hashes": [ @@ -1629,11 +1634,11 @@ }, "sphinx": { "hashes": [ - "sha256:423280646fb37944dd3c85c58fb92a20d745793a9f6c511f59da82fa97cd404b", - "sha256:de930f42600a4fef993587633984cc5027dedba2464bcf00ddace26b40f8d9ce" + "sha256:22538e1bbe62b407cf5a8aabe1bb15848aa66bb79559f42f5202bbce6b757a69", + "sha256:f9a79e746b87921cabc3baa375199c6076d1270cee53915dbd24fdbeaaacc427" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.1.2" }, "sphinxcontrib-applehelp": { "hashes": [ diff --git a/pytest.ini b/pytest.ini index 229d56d9..e65fcf81 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,6 +7,8 @@ # more details. [pytest] -pep8ignore = docs/conf.py ALL +pep8ignore = + *.py W503 + docs/conf.py ALL addopts = --pep8 --doctest-glob="*.rst" --doctest-modules --cov=sonar --cov-report=term-missing --ignore=setup.py testpaths = docs tests sonar diff --git a/run-tests.sh b/run-tests.sh index 9e000238..8056ad73 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -12,7 +12,7 @@ # released. pipenv check --ignore 36759 --ignore 36810 && \ pipenv run pydocstyle sonar tests docs && \ -pipenv run isort -rc -c -df && \ +pipenv run isort -rc -c -df --multi-line=3 --trailing-comma --force-grid-wrap=0 --use-parentheses --line-width=79 && \ pipenv run check-manifest --ignore ".travis-*,docs/_build*" && \ pipenv run sphinx-build -qnNW docs docs/_build/html && \ pipenv run test diff --git a/sonar/config.py b/sonar/config.py index e1ad7952..6eae9fe9 100644 --- a/sonar/config.py +++ b/sonar/config.py @@ -269,15 +269,15 @@ def _(x): 'documents': dict( aggs=dict( institution=dict(terms=dict(field='institution.pid')), - language=dict(terms=dict(field='languages.language')), + language=dict(terms=dict(field='languages')), author=dict(terms=dict(field='facet_authors')), subject=dict(terms=dict(field='facet_subjects')) ), filters={ - _('institution'): terms_filter('institution.pid'), - _('language'): terms_filter('languages.language'), - _('author'): terms_filter('facet_authors'), - _('subject'): terms_filter('facet_subjects'), + 'institution': terms_filter('institution.pid'), + 'language': terms_filter('languages'), + 'author': terms_filter('facet_authors'), + 'subject': terms_filter('facet_subjects'), } ) } diff --git a/sonar/modules/config.py b/sonar/modules/config.py index 057d7ad6..8c776ed0 100644 --- a/sonar/modules/config.py +++ b/sonar/modules/config.py @@ -8,4 +8,4 @@ """SONAR configuration.""" -SONAR_APP_API_URL = 'https://localhost:5000/api/' +SONAR_APP_API_URL = "https://localhost:5000/api/" diff --git a/sonar/modules/documents/cli.py b/sonar/modules/documents/cli.py index a013d0e1..979dd5ad 100644 --- a/sonar/modules/documents/cli.py +++ b/sonar/modules/documents/cli.py @@ -8,9 +8,6 @@ """Documents CLI commands.""" -import json -from functools import partial - import click import requests from click.exceptions import ClickException @@ -19,7 +16,6 @@ from flask.cli import with_appcontext from invenio_db import db from invenio_indexer.api import RecordIndexer -from invenio_records import Record from sonar.modules.documents.dojson.contrib.marc21tojson import marc21tojson from sonar.modules.institutions.api import InstitutionRecord @@ -32,9 +28,9 @@ def documents(): """Documents CLI commands.""" -@documents.command('import') -@click.argument('institution') -@click.option('--pages', '-p', required=True, type=int, default=10) +@documents.command("import") +@click.argument("institution") +@click.option("--pages", "-p", required=True, type=int, default=10) @with_appcontext def import_documents(institution, pages): """Import documents from RERO doc. @@ -42,53 +38,64 @@ def import_documents(institution, pages): institution: String institution filter for retreiving documents pages: Number of pages to import """ - url = current_app.config.get('SONAR_DOCUMENTS_RERO_DOC_URL') + url = current_app.config.get("SONAR_DOCUMENTS_RERO_DOC_URL") click.secho( 'Importing {pages} pages of records for "{institution}" ' - 'from {url}'.format(pages=pages, institution=institution, url=url)) + "from {url}".format(pages=pages, institution=institution, url=url) + ) # Get institution record from database institution_record = InstitutionRecord.get_record_by_pid(institution) if not institution_record: - raise ClickException('Institution record not found in database') + raise ClickException("Institution record not found in database") institution_ref_link = InstitutionRecord.get_ref_link( - 'institutions', institution_record['pid']) + "institutions", institution_record["pid"] + ) # mapping between institution key and RERO doc filter institution_map = current_app.config.get( - 'SONAR_DOCUMENTS_INSTITUTIONS_MAP') + "SONAR_DOCUMENTS_INSTITUTIONS_MAP" + ) if not institution_map: - raise ClickException('Institution map not found in configuration') + raise ClickException("Institution map not found in configuration") if institution not in institution_map: raise ClickException( 'Institution map for "{institution}" not found in configuration, ' - 'keys available {keys}'.format( - institution=institution, - keys=institution_map.keys())) + "keys available {keys}".format( + institution=institution, keys=institution_map.keys() + ) + ) key = institution_map[institution] current_page = 1 indexer = RecordIndexer() - while(current_page <= pages): - click.echo('Importing records {start} to {end}... '.format( - start=(current_page*10-9), end=(current_page*10)), nl=False) + while current_page <= pages: + click.echo( + "Importing records {start} to {end}... ".format( + start=(current_page * 10 - 9), end=(current_page * 10) + ), + nl=False, + ) # Read Marc21 data for current page response = requests.get( - '{url}?of=xm&jrec={first_record}&c=NAVSITE.{institution}' - .format(url=url, - first_record=(current_page*10-9), - institution=key.upper()), stream=True) + "{url}?of=xm&jrec={first_record}&c=NAVSITE.{institution}".format( + url=url, + first_record=(current_page * 10 - 9), + institution=key.upper(), + ), + stream=True, + ) response.raw.decode_content = True - if(response.status_code != 200): + if response.status_code != 200: raise ClickException('Request to "{url}" failed'.format(url=url)) ids = [] @@ -101,7 +108,7 @@ def import_documents(institution, pages): record = marc21tojson.do(record) # Add institution - record['institution'] = {'$ref': institution_ref_link} + record["institution"] = {"$ref": institution_ref_link} # Register record to DB db_record = DocumentRecord.create(record) @@ -116,6 +123,6 @@ def import_documents(institution, pages): current_page += 1 - click.secho('Done', fg='green', nl=True) + click.secho("Done", fg="green", nl=True) - click.secho('Finished', fg='green') + click.secho("Finished", fg="green") diff --git a/sonar/modules/documents/dojson/contrib/marc21tojson/__init__.py b/sonar/modules/documents/dojson/contrib/marc21tojson/__init__.py index 6cc8ff2a..afe3320c 100644 --- a/sonar/modules/documents/dojson/contrib/marc21tojson/__init__.py +++ b/sonar/modules/documents/dojson/contrib/marc21tojson/__init__.py @@ -10,4 +10,4 @@ from .model import marc21tojson -__all__ = ('marc21tojson') +__all__ = "marc21tojson" diff --git a/sonar/modules/documents/dojson/contrib/marc21tojson/model.py b/sonar/modules/documents/dojson/contrib/marc21tojson/model.py index 19ee78c1..3cfa7b3d 100644 --- a/sonar/modules/documents/dojson/contrib/marc21tojson/model.py +++ b/sonar/modules/documents/dojson/contrib/marc21tojson/model.py @@ -8,6 +8,7 @@ """MARC21 model definition.""" +# pylint: disable=unused-argument import re import sys @@ -19,61 +20,59 @@ def remove_punctuation(data): """Remove punctuation from data.""" - try: - if data[-1:] == ',': - data = data[:-1] - if data[-2:] == ' :': - data = data[:-2] - if data[-2:] == ' ;': - data = data[:-2] - if data[-2:] == ' /': - data = data[:-2] - if data[-2:] == ' -': - data = data[:-2] - except Exception: - pass + if not isinstance(data, str): + return data + + if data[-1:] == ",": + data = data[:-1] + if data[-2:] == " :": + data = data[:-2] + if data[-2:] == " ;": + data = data[:-2] + if data[-2:] == " /": + data = data[:-2] + if data[-2:] == " -": + data = data[:-2] + return data -def get_mef_person_link(id, key, value): +def get_mef_person_link(person_id, key, value): """Get mef person link.""" # https://mef.test.rero.ch/api/mef/?q=rero.rero_pid:A012327677 - PROD_HOST = 'mef.rero.ch' - DEV_HOST = 'mef.test.rero.ch' + prod_host = "mef.rero.ch" + dev_host = "mef.test.rero.ch" mef_url = None - if id: - identifier = id[1:].split(')') + if person_id: + identifier = person_id[1:].split(")") url = "{mef}/?q={org}.{org}_pid:{pid}".format( - mef="https://{host}/api/mef".format(host=DEV_HOST), + mef="https://{host}/api/mef".format(host=dev_host), org=identifier[0].lower(), - pid=identifier[1] + pid=identifier[1], ) request = requests.get(url=url) if request.status_code == requests.codes.ok: data = request.json() - hits = data.get('hits', {}).get('hits') + hits = data.get("hits", {}).get("hits") if hits: - mef_url = hits[0].get('links').get('self') - mef_url = mef_url.replace(DEV_HOST, PROD_HOST) + mef_url = hits[0].get("links").get("self") + mef_url = mef_url.replace(dev_host, prod_host) else: print( - 'ERROR: MEF person not found', + "ERROR: MEF person not found", url, key, value, - file=sys.stderr + file=sys.stderr, ) else: print( - 'ERROR: MEF request', - url, - request.status_code, - file=sys.stderr + "ERROR: MEF request", url, request.status_code, file=sys.stderr ) return mef_url -@marc21tojson.over('type', 'leader') +@marc21tojson.over("type", "leader") def marc21_to_type(self, key, value): """ Get document type. @@ -86,34 +85,33 @@ def marc21_to_type(self, key, value): Sounds: LDR/6: i|j E-books (imported from Cantook) """ - type = None + document_type = None type_of_record = value[6] bibliographic_level = value[7] - if type_of_record == 'a': - if bibliographic_level == 'm': - type = 'book' - elif bibliographic_level == 's': - type = 'journal' - elif bibliographic_level == 'a': - type = 'article' - elif type_of_record in ['c', 'd']: - type = 'score' - elif type_of_record in ['i', 'j']: - type = 'sound' - elif type_of_record == 'g': - type = 'video' - # Todo 007 - return type - - -@marc21tojson.over('pid', '^001') + if type_of_record == "a": + if bibliographic_level == "m": + document_type = "book" + elif bibliographic_level == "s": + document_type = "journal" + elif bibliographic_level == "a": + document_type = "article" + elif type_of_record in ["c", "d"]: + document_type = "score" + elif type_of_record in ["i", "j"]: + document_type = "sound" + elif type_of_record == "g": + document_type = "video" + return document_type + + +@marc21tojson.over("pid", "^001") @utils.ignore_value def marc21_to_pid(self, key, value): """Get ID.""" return value -@marc21tojson.over('title', '^245..') +@marc21tojson.over("title", "^245..") @utils.ignore_value def marc21_to_title(self, key, value): """Get title. @@ -121,28 +119,28 @@ def marc21_to_title(self, key, value): title: 245$a without the punctuaction. If there's a $b, then 245$a : $b without the " /" """ - main_title = remove_punctuation(value.get('a')) - sub_title = value.get('b') + main_title = remove_punctuation(value.get("a")) + sub_title = value.get("b") # responsability = value.get('c') if sub_title: - main_title += ' : ' + ' : '.join( + main_title += " : " + " : ".join( utils.force_list(remove_punctuation(sub_title)) ) return main_title -@marc21tojson.over('titlesProper', '^730..') +@marc21tojson.over("titlesProper", "^730..") @utils.for_each_value @utils.ignore_value -def marc21_to_titlesProper(self, key, value): +def marc21_to_titles_proper(self, key, value): """Test dojson marc21titlesProper. titleProper: 730$a """ - return value.get('a') + return value.get("a") -@marc21tojson.over('languages', '^008') +@marc21tojson.over("languages", "^008") @utils.ignore_value def marc21_to_languages(self, key, value): """Get languages. @@ -150,24 +148,24 @@ def marc21_to_languages(self, key, value): languages: 008 and 041 [$a, repetitive] """ language = value.strip()[35:38] - to_return = [{'language': language}] + to_return = [language] return to_return -@marc21tojson.over('translatedFrom', '^041..') +@marc21tojson.over("translatedFrom", "^041..") @utils.ignore_value -def marc21_to_translatedFrom(self, key, value): +def marc21_to_translated_from(self, key, value): """Get translatedFrom. translatedFrom: 041 [$h repetitive] languages: 008 and 041 [$a, repetitive] """ - languages = self.get('languages', []) + languages = self.get("languages", []) unique_lang = [] if languages != []: - unique_lang.append(languages[0]['language']) + unique_lang.append(languages[0]) - language = value.get('a') + language = value.get("a") if language: for lang in utils.force_list(language): if lang not in unique_lang: @@ -175,17 +173,17 @@ def marc21_to_translatedFrom(self, key, value): languages = [] for lang in unique_lang: - languages.append({'language': lang}) + languages.append(lang) - self['languages'] = languages - translated = value.get('h') + self["languages"] = languages + translated = value.get("h") if translated: return list(utils.force_list(translated)) else: return None -@marc21tojson.over('authors', '[17][01]0..') +@marc21tojson.over("authors", "[17][01]0..") @utils.for_each_value @utils.ignore_value def marc21_to_author(self, key, value): @@ -199,79 +197,79 @@ def marc21_to_author(self, key, value): authors.qualifier: 100 $c or 700 $c (facultatif) authors.type: if 100 or 700 then person, if 710 then organisation """ - if not (key[4] == '2'): + if not key[4] == "2": author = {} - author['type'] = 'person' - if value.get('0'): - ref = get_mef_person_link(value.get('0'), key, value) + author["type"] = "person" + if value.get("0"): + ref = get_mef_person_link(value.get("0"), key, value) if ref: - author['$ref'] = ref + author["$ref"] = ref # we do not have a $ref - if not author.get('$ref'): - author['name'] = remove_punctuation(value.get('a')) - author_subs = utils.force_list(value.get('b')) + if not author.get("$ref"): + author["name"] = remove_punctuation(value.get("a")) + author_subs = utils.force_list(value.get("b")) if author_subs: for author_sub in author_subs: - author['name'] += ' ' + remove_punctuation(author_sub) - if key[:3] == '710': - author['type'] = 'organisation' + author["name"] += " " + remove_punctuation(author_sub) + if key[:3] == "710": + author["type"] = "organisation" else: - if value.get('c'): - author['qualifier'] = remove_punctuation(value.get('c')) - if value.get('d'): - author['date'] = remove_punctuation(value.get('d')) + if value.get("c"): + author["qualifier"] = remove_punctuation(value.get("c")) + if value.get("d"): + author["date"] = remove_punctuation(value.get("d")) return author else: return None -@marc21tojson.over('publishers', '^260..') +@marc21tojson.over("publishers", "^260..") @utils.ignore_value -def marc21_to_publishers_publicationDate(self, key, value): +def marc21_to_publishers_publication_date(self, key, value): """Get publisher. publisher.name: 260 [$b repetitive] (without the , but keep the ;) publisher.place: 260 [$a repetitive] (without the : but keep the ;) publicationDate: 260 [$c repetitive] (but take only the first one) """ - lasttag = '?' - publishers = self.get('publishers', []) + lasttag = "?" + publishers = self.get("publishers", []) publisher = {} indexes = {} - lasttag = '?' - for tag in value['__order__']: + lasttag = "?" + for tag in value["__order__"]: index = indexes.get(tag, 0) data = value[tag] - if type(data) == tuple: + if isinstance(data, tuple): data = data[index] - if tag == 'a' and index > 0 and lasttag != 'a': + if tag == "a" and index > 0 and lasttag != "a": publishers.append(remove_punctuation(publisher)) publisher = {} - if tag == 'a': - place = publisher.get('place', []) + if tag == "a": + place = publisher.get("place", []) place.append(remove_punctuation(data)) - publisher['place'] = place - elif tag == 'b': - name = publisher.get('name', []) + publisher["place"] = place + elif tag == "b": + name = publisher.get("name", []) name.append(remove_punctuation(data)) - publisher['name'] = name - elif tag == 'c' and index == 0: + publisher["name"] = name + elif tag == "c" and index == 0: # 4 digits - date = re.match(r'.*?(\d{4})', data).group(1) - self['publicationYear'] = int(date) + date = re.match(r".*?(\d{4})", data).group(1) + self["publicationYear"] = int(date) # create free form if different - if data != str(self['publicationYear']): - self['freeFormedPublicationDate'] = data + if data != str(self["publicationYear"]): + self["freeFormedPublicationDate"] = data indexes[tag] = index + 1 lasttag = tag publishers.append(publisher) return publishers -@marc21tojson.over('formats', '^300..') +@marc21tojson.over("formats", "^300..") @utils.ignore_value def marc21_to_description(self, key, value): """Get extent, otherMaterialCharacteristics, formats. @@ -280,27 +278,27 @@ def marc21_to_description(self, key, value): otherMaterialCharacteristics: 300$b (the first one if many) formats: 300 [$c repetitive] """ - if value.get('a'): - if not self.get('extent', None): - self['extent'] = remove_punctuation( - utils.force_list(value.get('a'))[0] + if value.get("a"): + if not self.get("extent", None): + self["extent"] = remove_punctuation( + utils.force_list(value.get("a"))[0] ) - if value.get('b'): - if self.get('otherMaterialCharacteristics', []) == []: - self['otherMaterialCharacteristics'] = remove_punctuation( - utils.force_list(value.get('b'))[0] + if value.get("b"): + if self.get("otherMaterialCharacteristics", []) == []: + self["otherMaterialCharacteristics"] = remove_punctuation( + utils.force_list(value.get("b"))[0] ) - if value.get('c'): - formats = self.get('formats', None) + if value.get("c"): + formats = self.get("formats", None) if not formats: - data = value.get('c') + data = value.get("c") formats = list(utils.force_list(data)) return formats else: return None -@marc21tojson.over('series', '^490..') +@marc21tojson.over("series", "^490..") @utils.for_each_value @utils.ignore_value def marc21_to_series(self, key, value): @@ -310,54 +308,57 @@ def marc21_to_series(self, key, value): series.number: [490$v repetitive] """ series = {} - name = value.get('a') + name = value.get("a") if name: - series['name'] = ', '.join(utils.force_list(name)) - number = value.get('v') + series["name"] = ", ".join(utils.force_list(name)) + number = value.get("v") if number: - series['number'] = ', '.join(utils.force_list(number)) + series["number"] = ", ".join(utils.force_list(number)) return series -@marc21tojson.over('abstracts', '^520..') -@utils.for_each_value +@marc21tojson.over("abstracts", "^520..") @utils.ignore_value def marc21_to_abstracts(self, key, value): """Get abstracts. abstract: [520$a repetitive] """ - return ', '.join(utils.force_list(value.get('a'))) + if not self.get("abstracts", None): + self["abstracts"] = {} + self["abstracts"][value.get("9")] = value.get("a") + return None -@marc21tojson.over('identifiers', '^020..') + +@marc21tojson.over("identifiers", "^020..") @utils.ignore_value def marc21_to_identifier_isbn(self, key, value): """Get identifier isbn. identifiers:isbn: 020$a """ - if value.get('a'): - identifiers = self.get('identifiers', {}) - identifiers['isbn'] = value.get('a') + if value.get("a"): + identifiers = self.get("identifiers", {}) + identifiers["isbn"] = value.get("a") return identifiers else: return None -@marc21tojson.over('identifiers', '^035..') +@marc21tojson.over("identifiers", "^035..") @utils.ignore_value -def marc21_to_identifier_reroID(self, key, value): +def marc21_to_identifier_rero_id(self, key, value): """Get identifier reroId. identifiers:reroID: 035$a """ - identifiers = self.get('identifiers', {}) - identifiers['reroID'] = value.get('a') + identifiers = self.get("identifiers", {}) + identifiers["reroID"] = value.get("a") return identifiers -@marc21tojson.over('notes', '^500..') +@marc21tojson.over("notes", "^500..") @utils.for_each_value @utils.ignore_value def marc21_to_notes(self, key, value): @@ -365,21 +366,21 @@ def marc21_to_notes(self, key, value): note: [500$a repetitive] """ - return value.get('a') + return value.get("a") -@marc21tojson.over('is_part_of', '^773..') +@marc21tojson.over("is_part_of", "^773..") @utils.ignore_value def marc21_to_is_part_of(self, key, value): """Get is_part_of. is_part_of: [773$t repetitive] """ - if not self.get('is_part_of', None): - return value.get('t') + if not self.get("is_part_of", None): + return value.get("t") -@marc21tojson.over('subjects', '^6....') +@marc21tojson.over("subjects", "^6....") @utils.ignore_value def marc21_to_subjects(self, key, value): """Get subjects. @@ -387,4 +388,8 @@ def marc21_to_subjects(self, key, value): subjects: 6xx [duplicates could exist between several vocabularies, if possible deduplicate] """ - return value.get('a').split(' ; ') + if not self.get("subjects", None): + self["subjects"] = {} + + self["subjects"][value.get("9")] = value.get("a").split(" ; ") + return None diff --git a/sonar/modules/documents/jsonschemas/documents/document-v1.0.0.json b/sonar/modules/documents/jsonschemas/documents/document-v1.0.0.json index 99589be0..b167071f 100644 --- a/sonar/modules/documents/jsonschemas/documents/document-v1.0.0.json +++ b/sonar/modules/documents/jsonschemas/documents/document-v1.0.0.json @@ -4,7 +4,10 @@ "additionalProperties": false, "title": "Schema for document", "type": "object", - "required": ["pid", "title"], + "required": [ + "pid", + "title" + ], "properties": { "$schema": { "type": "string", @@ -23,11 +26,14 @@ "abstracts": { "title": "Abstract", "description": "Abstract of the resource.", - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 3 + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^fre|eng|ger|ita|spa$": { + "title": "Abstract translation", + "description": "Translation for the abstract and language key.", + "type": "string" + } } }, "identifiers": { @@ -65,33 +71,14 @@ "uniqueItems": true, "minItems": 1, "items": { - "type": "object", - "required": [ - "language" - ], - "properties": { - "language": { - "title": "Language", - "description": "Required. Language of the resource, primary or not.", - "type": "string", - "default": "fre", - "validationMessage": "Required. Language of the resource, primary or not.", - "enum": [ - "fre", - "ger", - "eng", - "ita", - "spa", - "ara", - "chi", - "lat", - "heb", - "jpn", - "por", - "rus" - ] - } - } + "type": "string", + "enum": [ + "fre", + "ger", + "eng", + "ita", + "esp" + ] } }, "authors": { @@ -160,11 +147,18 @@ "subjects": { "title": "Subject", "description": "Subject of the resource.", - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^fre|eng|ger|ita|spa$": { + "title": "Subject translation", + "description": "Translation for the subject and language key.", + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } } }, "notes": { @@ -186,7 +180,9 @@ "type": "string" } }, - "required": ["$ref"] + "required": [ + "$ref" + ] } } } \ No newline at end of file diff --git a/sonar/modules/documents/loaders/__init__.py b/sonar/modules/documents/loaders/__init__.py index b686667f..ccd75c5b 100644 --- a/sonar/modules/documents/loaders/__init__.py +++ b/sonar/modules/documents/loaders/__init__.py @@ -10,13 +10,13 @@ from __future__ import absolute_import, print_function -from invenio_records_rest.loaders.marshmallow import json_patch_loader, \ - marshmallow_loader +from invenio_records_rest.loaders.marshmallow import ( + json_patch_loader, + marshmallow_loader, +) from ..marshmallow import DocumentMetadataSchemaV1 json_v1 = marshmallow_loader(DocumentMetadataSchemaV1) -__all__ = ( - 'json_v1', -) +__all__ = ("json_v1",) diff --git a/sonar/modules/documents/mappings/v6/documents/document-v1.0.0.json b/sonar/modules/documents/mappings/v6/documents/document-v1.0.0.json index 77ff0583..5c9916f8 100644 --- a/sonar/modules/documents/mappings/v6/documents/document-v1.0.0.json +++ b/sonar/modules/documents/mappings/v6/documents/document-v1.0.0.json @@ -15,11 +15,41 @@ "type": "text" }, "abstracts": { - "type": "text" + "type": "object", + "properties": { + "eng": { + "type": "text", + "analyzer": "english" + }, + "fre": { + "type": "text", + "analyzer": "french" + }, + "ger": { + "type": "text", + "analyzer": "german" + } + } }, "subjects": { - "type": "text", - "copy_to": "facet_subjects" + "type": "object", + "properties": { + "eng": { + "type": "text", + "analyzer": "english", + "copy_to": "facet_subjects" + }, + "fre": { + "type": "text", + "analyzer": "french", + "copy_to": "facet_subjects" + }, + "ger": { + "type": "text", + "analyzer": "german", + "copy_to": "facet_subjects" + } + } }, "facet_subjects": { "type": "keyword" @@ -39,12 +69,7 @@ } }, "languages": { - "type": "object", - "properties": { - "language": { - "type": "keyword" - } - } + "type": "keyword" }, "is_part_of": { "type": "text" @@ -87,4 +112,4 @@ } } } -} +} \ No newline at end of file diff --git a/sonar/modules/documents/marshmallow/json.py b/sonar/modules/documents/marshmallow/json.py index bcc61039..703f629b 100644 --- a/sonar/modules/documents/marshmallow/json.py +++ b/sonar/modules/documents/marshmallow/json.py @@ -10,10 +10,12 @@ from __future__ import absolute_import, print_function -from invenio_records_rest.schemas import Nested, StrictKeysMixin -from invenio_records_rest.schemas.fields import DateString, \ - PersistentIdentifier, SanitizedUnicode -from marshmallow import fields, missing, validate +from invenio_records_rest.schemas import StrictKeysMixin +from invenio_records_rest.schemas.fields import ( + PersistentIdentifier, + SanitizedUnicode, +) +from marshmallow import fields class DocumentMetadataSchemaV1(StrictKeysMixin): @@ -21,7 +23,7 @@ class DocumentMetadataSchemaV1(StrictKeysMixin): pid = PersistentIdentifier() title = SanitizedUnicode(required=True) - abstracts = fields.List(fields.Str()) + abstracts = fields.Dict() authors = fields.Dict(dump_only=True) institution = fields.Dict(dump_only=True) diff --git a/sonar/modules/documents/serializers/__init__.py b/sonar/modules/documents/serializers/__init__.py index 2b1bb451..d59b5b4d 100644 --- a/sonar/modules/documents/serializers/__init__.py +++ b/sonar/modules/documents/serializers/__init__.py @@ -11,8 +11,10 @@ from __future__ import absolute_import, print_function from invenio_records_rest.serializers.json import JSONSerializer -from invenio_records_rest.serializers.response import record_responsify, \ - search_responsify +from invenio_records_rest.serializers.response import ( + record_responsify, + search_responsify, +) from ..marshmallow import DocumentSchemaV1 @@ -24,12 +26,8 @@ # Records-REST serializers # ======================== #: JSON record serializer for individual records. -json_v1_response = record_responsify(json_v1, 'application/json') +json_v1_response = record_responsify(json_v1, "application/json") #: JSON record serializer for search results. -json_v1_search = search_responsify(json_v1, 'application/json') +json_v1_search = search_responsify(json_v1, "application/json") -__all__ = ( - 'json_v1', - 'json_v1_response', - 'json_v1_search', -) +__all__ = ("json_v1", "json_v1_response", "json_v1_search") diff --git a/sonar/modules/documents/static/templates/documents/search_ui/results.html b/sonar/modules/documents/static/templates/documents/search_ui/results.html index a696109e..6a32e5cb 100644 --- a/sonar/modules/documents/static/templates/documents/search_ui/results.html +++ b/sonar/modules/documents/static/templates/documents/search_ui/results.html @@ -9,10 +9,12 @@
{{record.metadata.institution.name}}
-{{author.name}}
-- {{record.metadata.abstracts[0]}} +
{{author.name}}
++ {{abstract}}