diff --git a/.travis.yml b/.travis.yml index 39ec3a48..9b04b238 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: - - 3.8.3 + - 3.8.5 services: - docker diff --git a/Dockerfile b/Dockerfile index b3e898f5..0004f0c8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.3-alpine3.12 as base +FROM python:3.8.5-alpine3.12 as base FROM base as builder RUN apk --no-cache --update-cache add gcc python3 python3-dev py-pip build-base wget RUN ln -s /usr/include/locale.h /usr/include/xlocale.h diff --git a/Pipfile b/Pipfile index 824b0198..917cf43f 100644 --- a/Pipfile +++ b/Pipfile @@ -28,4 +28,4 @@ rethinkdb = "==2.3" [dev-packages] [requires] -python_version = "3.8.3" +python_version = "3.8.5" diff --git a/Pipfile.lock b/Pipfile.lock index 467126e5..6e230c1a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "4e58db5aca3cf9edf61c5649f8ef5e6e696e41845ebabf7b6f01288f7d48fbd6" + "sha256": "0ca00bc8605102d1ac3eb8b3607fd53ee0626abe0c5bd3e5413eaba5e52d2db6" }, "pipfile-spec": 6, "requires": { - "python_version": "3.8.3" + "python_version": "3.8.5" }, "sources": [ { @@ -21,6 +21,7 @@ "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197", "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.4.0" }, "attrs": { @@ -28,6 +29,7 @@ "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.2.0" }, "certifi": { @@ -50,6 +52,7 @@ "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==7.1.2" }, "clickclick": { @@ -68,46 +71,48 @@ "sha256:5439e9659a89c4380d93a07acfbf3380d70be4130574de8881e5f0dfec7ad0e2" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==2.7.0" }, "coverage": { "hashes": [ - "sha256:098a703d913be6fbd146a8c50cc76513d726b022d170e5e98dc56d958fd592fb", - "sha256:16042dc7f8e632e0dcd5206a5095ebd18cb1d005f4c89694f7f8aafd96dd43a3", - "sha256:1adb6be0dcef0cf9434619d3b892772fdb48e793300f9d762e480e043bd8e716", - "sha256:27ca5a2bc04d68f0776f2cdcb8bbd508bbe430a7bf9c02315cd05fb1d86d0034", - "sha256:28f42dc5172ebdc32622a2c3f7ead1b836cdbf253569ae5673f499e35db0bac3", - "sha256:2fcc8b58953d74d199a1a4d633df8146f0ac36c4e720b4a1997e9b6327af43a8", - "sha256:304fbe451698373dc6653772c72c5d5e883a4aadaf20343592a7abb2e643dae0", - "sha256:30bc103587e0d3df9e52cd9da1dd915265a22fad0b72afe54daf840c984b564f", - "sha256:40f70f81be4d34f8d491e55936904db5c527b0711b2a46513641a5729783c2e4", - "sha256:4186fc95c9febeab5681bc3248553d5ec8c2999b8424d4fc3a39c9cba5796962", - "sha256:46794c815e56f1431c66d81943fa90721bb858375fb36e5903697d5eef88627d", - "sha256:4869ab1c1ed33953bb2433ce7b894a28d724b7aa76c19b11e2878034a4e4680b", - "sha256:4f6428b55d2916a69f8d6453e48a505c07b2245653b0aa9f0dee38785939f5e4", - "sha256:52f185ffd3291196dc1aae506b42e178a592b0b60a8610b108e6ad892cfc1bb3", - "sha256:538f2fd5eb64366f37c97fdb3077d665fa946d2b6d95447622292f38407f9258", - "sha256:64c4f340338c68c463f1b56e3f2f0423f7b17ba6c3febae80b81f0e093077f59", - "sha256:675192fca634f0df69af3493a48224f211f8db4e84452b08d5fcebb9167adb01", - "sha256:700997b77cfab016533b3e7dbc03b71d33ee4df1d79f2463a318ca0263fc29dd", - "sha256:8505e614c983834239f865da2dd336dcf9d72776b951d5dfa5ac36b987726e1b", - "sha256:962c44070c281d86398aeb8f64e1bf37816a4dfc6f4c0f114756b14fc575621d", - "sha256:9e536783a5acee79a9b308be97d3952b662748c4037b6a24cbb339dc7ed8eb89", - "sha256:9ea749fd447ce7fb1ac71f7616371f04054d969d412d37611716721931e36efd", - "sha256:a34cb28e0747ea15e82d13e14de606747e9e484fb28d63c999483f5d5188e89b", - "sha256:a3ee9c793ffefe2944d3a2bd928a0e436cd0ac2d9e3723152d6fd5398838ce7d", - "sha256:aab75d99f3f2874733946a7648ce87a50019eb90baef931698f96b76b6769a46", - "sha256:b1ed2bdb27b4c9fc87058a1cb751c4df8752002143ed393899edb82b131e0546", - "sha256:b360d8fd88d2bad01cb953d81fd2edd4be539df7bfec41e8753fe9f4456a5082", - "sha256:b8f58c7db64d8f27078cbf2a4391af6aa4e4767cc08b37555c4ae064b8558d9b", - "sha256:c1bbb628ed5192124889b51204de27c575b3ffc05a5a91307e7640eff1d48da4", - "sha256:c2ff24df02a125b7b346c4c9078c8936da06964cc2d276292c357d64378158f8", - "sha256:c890728a93fffd0407d7d37c1e6083ff3f9f211c83b4316fae3778417eab9811", - "sha256:c96472b8ca5dc135fb0aa62f79b033f02aa434fb03a8b190600a5ae4102df1fd", - "sha256:ce7866f29d3025b5b34c2e944e66ebef0d92e4a4f2463f7266daa03a1332a651", - "sha256:e26c993bd4b220429d4ec8c1468eca445a4064a61c74ca08da7429af9bc53bb0" - ], - "version": "==5.2.1" + "sha256:0203acd33d2298e19b57451ebb0bed0ab0c602e5cf5a818591b4918b1f97d516", + "sha256:0f313707cdecd5cd3e217fc68c78a960b616604b559e9ea60cc16795c4304259", + "sha256:1c6703094c81fa55b816f5ae542c6ffc625fec769f22b053adb42ad712d086c9", + "sha256:1d44bb3a652fed01f1f2c10d5477956116e9b391320c94d36c6bf13b088a1097", + "sha256:280baa8ec489c4f542f8940f9c4c2181f0306a8ee1a54eceba071a449fb870a0", + "sha256:29a6272fec10623fcbe158fdf9abc7a5fa032048ac1d8631f14b50fbfc10d17f", + "sha256:2b31f46bf7b31e6aa690d4c7a3d51bb262438c6dcb0d528adde446531d0d3bb7", + "sha256:2d43af2be93ffbad25dd959899b5b809618a496926146ce98ee0b23683f8c51c", + "sha256:381ead10b9b9af5f64646cd27107fb27b614ee7040bb1226f9c07ba96625cbb5", + "sha256:47a11bdbd8ada9b7ee628596f9d97fbd3851bd9999d398e9436bd67376dbece7", + "sha256:4d6a42744139a7fa5b46a264874a781e8694bb32f1d76d8137b68138686f1729", + "sha256:50691e744714856f03a86df3e2bff847c2acede4c191f9a1da38f088df342978", + "sha256:530cc8aaf11cc2ac7430f3614b04645662ef20c348dce4167c22d99bec3480e9", + "sha256:582ddfbe712025448206a5bc45855d16c2e491c2dd102ee9a2841418ac1c629f", + "sha256:63808c30b41f3bbf65e29f7280bf793c79f54fb807057de7e5238ffc7cc4d7b9", + "sha256:71b69bd716698fa62cd97137d6f2fdf49f534decb23a2c6fc80813e8b7be6822", + "sha256:7858847f2d84bf6e64c7f66498e851c54de8ea06a6f96a32a1d192d846734418", + "sha256:78e93cc3571fd928a39c0b26767c986188a4118edc67bc0695bc7a284da22e82", + "sha256:7f43286f13d91a34fadf61ae252a51a130223c52bfefb50310d5b2deb062cf0f", + "sha256:86e9f8cd4b0cdd57b4ae71a9c186717daa4c5a99f3238a8723f416256e0b064d", + "sha256:8f264ba2701b8c9f815b272ad568d555ef98dfe1576802ab3149c3629a9f2221", + "sha256:9342dd70a1e151684727c9c91ea003b2fb33523bf19385d4554f7897ca0141d4", + "sha256:9361de40701666b034c59ad9e317bae95c973b9ff92513dd0eced11c6adf2e21", + "sha256:9669179786254a2e7e57f0ecf224e978471491d660aaca833f845b72a2df3709", + "sha256:aac1ba0a253e17889550ddb1b60a2063f7474155465577caa2a3b131224cfd54", + "sha256:aef72eae10b5e3116bac6957de1df4d75909fc76d1499a53fb6387434b6bcd8d", + "sha256:bd3166bb3b111e76a4f8e2980fa1addf2920a4ca9b2b8ca36a3bc3dedc618270", + "sha256:c1b78fb9700fc961f53386ad2fd86d87091e06ede5d118b8a50dea285a071c24", + "sha256:c3888a051226e676e383de03bf49eb633cd39fc829516e5334e69b8d81aae751", + "sha256:c5f17ad25d2c1286436761b462e22b5020d83316f8e8fcb5deb2b3151f8f1d3a", + "sha256:c851b35fc078389bc16b915a0a7c1d5923e12e2c5aeec58c52f4aa8085ac8237", + "sha256:cb7df71de0af56000115eafd000b867d1261f786b5eebd88a0ca6360cccfaca7", + "sha256:cedb2f9e1f990918ea061f28a0f0077a07702e3819602d3507e2ff98c8d20636", + "sha256:e8caf961e1b1a945db76f1b5fa9c91498d15f545ac0ababbe575cfab185d3bd8" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==5.3" }, "coveralls": { "hashes": [ @@ -150,6 +155,7 @@ "hashes": [ "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.18.2" }, "geocoder": { @@ -171,7 +177,8 @@ "geomet": { "hashes": [ "sha256:91d754f7c298cbfcabd3befdb69c641c27fe75e808b27aa55028605761d17e95", - "sha256:a41a1e336b381416d6cbed7f1745c848e91defaa4d4c1bdc1312732e46ffad2b" + "sha256:a41a1e336b381416d6cbed7f1745c848e91defaa4d4c1bdc1312732e46ffad2b", + "sha256:87ae0fc42e532b9e98969c0bbf895a5e0b2bb4f6f775cf51a74e6482f1f35c2b" ], "index": "pypi", "version": "==0.2.1.post1" @@ -189,6 +196,7 @@ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.10" }, "inflection": { @@ -196,6 +204,7 @@ "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2" ], + "markers": "python_version >= '3.5'", "version": "==0.5.1" }, "influxdb": { @@ -211,6 +220,7 @@ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "jinja2": { @@ -218,6 +228,7 @@ "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==2.11.2" }, "jsonschema": { @@ -263,6 +274,7 @@ "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.1" }, "more-itertools": { @@ -270,6 +282,7 @@ "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" ], + "markers": "python_version >= '3.5'", "version": "==8.5.0" }, "openapi-spec-validator": { @@ -293,6 +306,7 @@ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.13.1" }, "py": { @@ -300,72 +314,75 @@ "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "pymongo": { "hashes": [ - "sha256:03dc64a9aa7a5d405aea5c56db95835f6a2fa31b3502c5af1760e0e99210be30", - "sha256:05fcc6f9c60e6efe5219fbb5a30258adb3d3e5cbd317068f3d73c09727f2abb6", - "sha256:076a7f2f7c251635cf6116ac8e45eefac77758ee5a77ab7bd2f63999e957613b", + "sha256:ce208f80f398522e49d9db789065c8ad2cd37b21bd6b23d30053474b7416af11", + "sha256:d9de8427a5601799784eb0e7fa1b031aa64086ce04de29df775a8ca37eedac41", + "sha256:475a34a0745c456ceffaec4ce86b7e0983478f1b6140890dff7b161e7bcd895b", "sha256:137e6fa718c7eff270dbd2fc4b90d94b1a69c9e9eb3f3de9e850a7fd33c822dc", + "sha256:7307024b18266b302f4265da84bb1effb5d18999ef35b30d17592959568d5c0a", + "sha256:4797c0080f41eba90404335e5ded3aa66731d303293a675ff097ce4ea3025bb9", "sha256:1f865b1d1c191d785106f54df9abdc7d2f45a946b45fd1ea0a641b4f982a2a77", - "sha256:213c445fe7e654621c6309e874627c35354b46ef3ee807f5a1927dc4b30e1a67", - "sha256:25e617daf47d8dfd4e152c880cd0741cbdb48e51f54b8de9ddbfe74ecd87dd16", - "sha256:3d9bb1ba935a90ec4809a8031efd988bdb13cdba05d9e9a3e9bf151bf759ecde", - "sha256:40696a9a53faa7d85aaa6fd7bef1cae08f7882640bad08c350fb59dee7ad069b", - "sha256:421aa1b92c291c429668bd8d8d8ec2bd00f183483a756928e3afbf2b6f941f00", "sha256:4437300eb3a5e9cc1a73b07d22c77302f872f339caca97e9bf8cf45eca8fa0d2", - "sha256:455f4deb00158d5ec8b1d3092df6abb681b225774ab8a59b3510293b4c8530e3", - "sha256:475a34a0745c456ceffaec4ce86b7e0983478f1b6140890dff7b161e7bcd895b", - "sha256:4797c0080f41eba90404335e5ded3aa66731d303293a675ff097ce4ea3025bb9", - "sha256:4ae23fbbe9eadf61279a26eba866bbf161a6f7e2ffad14a42cf20e9cb8e94166", - "sha256:4b32744901ee9990aa8cd488ec85634f443526def1e5190a407dc107148249d7", - "sha256:50127b13b38e8e586d5e97d342689405edbd74ad0bd891d97ee126a8c7b6e45f", - "sha256:50531caa7b4be1c4ed5e2d5793a4e51cc9bd62a919a6fd3299ef7c902e206eab", + "sha256:d64c98277ea80e4484f1332ab107e8dfd173a7dcf1bdbf10a9cccc97aaab145f", + "sha256:c4869141e20769b65d2d72686e7a7eb141ce9f3168106bed3e7dcced54eb2422", + "sha256:3d9bb1ba935a90ec4809a8031efd988bdb13cdba05d9e9a3e9bf151bf759ecde", + "sha256:d38b35f6eef4237b1d0d8e845fc1546dad85c55eba447e28c211da8c7ef9697c", + "sha256:e6a15cf8f887d9f578dd49c6fb3a99d53e1d922fdd67a245a67488d77bf56eb2", + "sha256:7a4a6f5b818988a3917ec4baa91d1143242bdfece8d38305020463955961266a", "sha256:68220b81850de8e966d4667d5c325a96c6ac0d6adb3d18935d6e3d325d441f48", + "sha256:e8d188ee39bd0ffe76603da887706e4e7b471f613625899ddf1e27867dc6a0d3", + "sha256:05fcc6f9c60e6efe5219fbb5a30258adb3d3e5cbd317068f3d73c09727f2abb6", + "sha256:076a7f2f7c251635cf6116ac8e45eefac77758ee5a77ab7bd2f63999e957613b", "sha256:689142dc0c150e9cb7c012d84cac2c346d40beb891323afb6caf18ec4caafae0", - "sha256:6a15e2bee5c4188369a87ed6f02de804651152634a46cca91966a11c8abd2550", + "sha256:e8c446882cbb3774cd78c738c9f58220606b702b7c1655f1423357dc51674054", + "sha256:8ea13d0348b4c96b437d944d7068d59ed4a6c98aaa6c40d8537a2981313f1c66", "sha256:7122ffe597b531fb065d3314e704a6fe152b81820ca5f38543e70ffcc95ecfd4", - "sha256:7307024b18266b302f4265da84bb1effb5d18999ef35b30d17592959568d5c0a", - "sha256:7a4a6f5b818988a3917ec4baa91d1143242bdfece8d38305020463955961266a", + "sha256:f6efca006a81e1197b925a7d7b16b8f61980697bb6746587aad8842865233218", + "sha256:63a5387e496a98170ffe638b435c0832c0f2011a6f4ff7a2880f17669fff8c03", + "sha256:50127b13b38e8e586d5e97d342689405edbd74ad0bd891d97ee126a8c7b6e45f", + "sha256:d0565481dc196986c484a7fb13214fc6402201f7fb55c65fd215b3324962fe6c", "sha256:83c5a3ecd96a9f3f11cfe6dfcbcec7323265340eb24cc996acaecea129865a3a", + "sha256:b7c522292407fa04d8195032493aac937e253ad9ae524aab43b9d9d242571f03", "sha256:890b0f1e18dbd898aeb0ab9eae1ab159c6bcbe87f0abb065b0044581d8614062", - "sha256:8deda1f7b4c03242f2a8037706d9584e703f3d8c74d6d9cac5833db36fe16c42", - "sha256:8ea13d0348b4c96b437d944d7068d59ed4a6c98aaa6c40d8537a2981313f1c66", - "sha256:91e96bf85b7c07c827d339a386e8a3cf2e90ef098c42595227f729922d0851df", + "sha256:4b32744901ee9990aa8cd488ec85634f443526def1e5190a407dc107148249d7", "sha256:96782ebb3c9e91e174c333208b272ea144ed2a684413afb1038e3b3342230d72", + "sha256:c0d660a186e36c526366edf8a64391874fe53cf8b7039224137aee0163c046df", "sha256:9755c726aa6788f076114dfdc03b92b03ff8860316cca00902cce88bcdb5fedd", - "sha256:9dbab90c348c512e03f146e93a5e2610acec76df391043ecd46b6b775d5397e6", - "sha256:9ee0eef254e340cc11c379f797af3977992a7f2c176f1a658740c94bf677e13c", - "sha256:9fc17fdac8f1973850d42e51e8ba6149d93b1993ed6768a24f352f926dd3d587", + "sha256:ef76535776c0708a85258f6dc51d36a2df12633c735f6d197ed7dfcaa7449b99", "sha256:a2787319dc69854acdfd6452e6a8ba8f929aeb20843c7f090e04159fc18e6245", - "sha256:b7c522292407fa04d8195032493aac937e253ad9ae524aab43b9d9d242571f03", + "sha256:9fc17fdac8f1973850d42e51e8ba6149d93b1993ed6768a24f352f926dd3d587", + "sha256:9ee0eef254e340cc11c379f797af3977992a7f2c176f1a658740c94bf677e13c", + "sha256:03dc64a9aa7a5d405aea5c56db95835f6a2fa31b3502c5af1760e0e99210be30", + "sha256:91e96bf85b7c07c827d339a386e8a3cf2e90ef098c42595227f729922d0851df", + "sha256:25e617daf47d8dfd4e152c880cd0741cbdb48e51f54b8de9ddbfe74ecd87dd16", + "sha256:455f4deb00158d5ec8b1d3092df6abb681b225774ab8a59b3510293b4c8530e3", + "sha256:4ae23fbbe9eadf61279a26eba866bbf161a6f7e2ffad14a42cf20e9cb8e94166", + "sha256:cccf1e7806f12300e3a3b48f219e111000c2538483e85c869c35c1ae591e6ce9", + "sha256:9dbab90c348c512e03f146e93a5e2610acec76df391043ecd46b6b775d5397e6", "sha256:bd312794f51e37dcf77f013d40650fe4fbb211dd55ef2863839c37480bd44369", - "sha256:c0d660a186e36c526366edf8a64391874fe53cf8b7039224137aee0163c046df", - "sha256:c4869141e20769b65d2d72686e7a7eb141ce9f3168106bed3e7dcced54eb2422", "sha256:cc4057f692ac35bbe82a0a908d42ce3a281c9e913290fac37d7fa3bd01307dfb", - "sha256:cccf1e7806f12300e3a3b48f219e111000c2538483e85c869c35c1ae591e6ce9", - "sha256:ce208f80f398522e49d9db789065c8ad2cd37b21bd6b23d30053474b7416af11", - "sha256:d0565481dc196986c484a7fb13214fc6402201f7fb55c65fd215b3324962fe6c", - "sha256:d1b3366329c45a474b3bbc9b9c95d4c686e03f35da7fd12bc144626d1f2a7c04", + "sha256:50531caa7b4be1c4ed5e2d5793a4e51cc9bd62a919a6fd3299ef7c902e206eab", + "sha256:421aa1b92c291c429668bd8d8d8ec2bd00f183483a756928e3afbf2b6f941f00", + "sha256:8deda1f7b4c03242f2a8037706d9584e703f3d8c74d6d9cac5833db36fe16c42", "sha256:d226e0d4b9192d95079a9a29c04dd81816b1ce8903b8c174a39224fe978547cb", - "sha256:d38b35f6eef4237b1d0d8e845fc1546dad85c55eba447e28c211da8c7ef9697c", - "sha256:d64c98277ea80e4484f1332ab107e8dfd173a7dcf1bdbf10a9cccc97aaab145f", - "sha256:d9de8427a5601799784eb0e7fa1b031aa64086ce04de29df775a8ca37eedac41", - "sha256:e6a15cf8f887d9f578dd49c6fb3a99d53e1d922fdd67a245a67488d77bf56eb2", - "sha256:e8c446882cbb3774cd78c738c9f58220606b702b7c1655f1423357dc51674054", - "sha256:e8d188ee39bd0ffe76603da887706e4e7b471f613625899ddf1e27867dc6a0d3", - "sha256:ef76535776c0708a85258f6dc51d36a2df12633c735f6d197ed7dfcaa7449b99", - "sha256:f6efca006a81e1197b925a7d7b16b8f61980697bb6746587aad8842865233218" + "sha256:40696a9a53faa7d85aaa6fd7bef1cae08f7882640bad08c350fb59dee7ad069b", + "sha256:6a15e2bee5c4188369a87ed6f02de804651152634a46cca91966a11c8abd2550", + "sha256:d1b3366329c45a474b3bbc9b9c95d4c686e03f35da7fd12bc144626d1f2a7c04", + "sha256:213c445fe7e654621c6309e874627c35354b46ef3ee807f5a1927dc4b30e1a67" ], "index": "pypi", "version": "==3.11.0" }, "pyrsistent": { "hashes": [ - "sha256:28669905fe725965daa16184933676547c5bb40a5153055a8dee2a4bd7933ad3" + "sha256:2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e" ], - "version": "==0.16.0" + "markers": "python_version >= '3.5'", + "version": "==0.17.3" }, "pytest": { "hashes": [ @@ -458,6 +475,7 @@ "sha256:74815c25aad1fe0b5fb994e96c3de63e8695164358a80138352aaadfa4760350", "sha256:d6865ed1d135ddb124a619d7cd3a5b505f69a7c92e248024dd7e48bc77752af5" ], + "markers": "python_version >= '3.5'", "version": "==1.2.0" }, "six": { @@ -465,6 +483,7 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "swagger-ui-bundle": { @@ -479,6 +498,7 @@ "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.25.10" }, "werkzeug": { @@ -486,6 +506,7 @@ "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.0.1" } }, diff --git a/src/conftest.py b/src/conftest.py index 69c84f27..2a46792e 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -5,7 +5,6 @@ import pytest import requests - QL_HOST = os.environ.get('QL_HOST', 'quantumleap') QL_PORT = 8668 QL_URL = "http://{}:{}/v2".format(QL_HOST, QL_PORT) @@ -41,20 +40,34 @@ def clean_mongo(): do_clean_mongo() +def headers(service, service_path, content_type=True): + h = {} + if content_type: + h['Content-Type'] = 'application/json' + if service: + h['Fiware-Service'] = service + if service_path: + h['Fiware-ServicePath'] = service_path + + return h + + +# TODO we have fully fledged client library, why not using that? class OrionClient(object): + def __init__(self, host, port): self.url = 'http://{}:{}'.format(host, port) - def subscribe(self, subscription): + def subscribe(self, subscription, service=None, service_path=None): r = requests.post('{}/v2/subscriptions'.format(self.url), data=json.dumps(subscription), - headers={'Content-Type': 'application/json'}) + headers=headers(service, service_path)) return r - def insert(self, entity): + def insert(self, entity, service=None, service_path=None): r = requests.post('{}/v2/entities'.format(self.url), data=json.dumps(entity), - headers={'Content-Type': 'application/json'}) + headers=headers(service, service_path)) return r @@ -133,6 +146,7 @@ def delete_entities(self, entity_type=None, fiware_service=None, with Translator(host=CRATE_HOST, port=CRATE_PORT) as trans: yield trans + @pytest.fixture def entity(): entity = { @@ -151,6 +165,7 @@ def entity(): } return entity + @pytest.fixture def sameEntityWithDifferentAttrs(): """ @@ -199,6 +214,70 @@ def sameEntityWithDifferentAttrs(): ] return entities + +@pytest.fixture +def diffEntityWithDifferentAttrs(): + """ + Two updates for the same entity with different attributes. + The first update has temperature and pressure but the second update has only temperature. + """ + entities = [ + { + 'id': 'Room1', + 'type': 'Room', + 'temperature': { + 'value': 24.2, + 'type': 'Number', + 'metadata': { + 'dateModified': { + 'type': 'DateTime', + 'value': '2019-05-09T15:28:30.000Z' + } + } + }, + 'pressure': { + 'value': 720, + 'type': 'Number', + 'metadata': { + 'dateModified': { + 'type': 'DateTime', + 'value': '2019-05-09T15:28:30.000Z' + } + } + } + }, + { + 'id': 'Room2', + 'type': 'Room', + 'temperature': { + 'value': 25.2, + 'type': 'Number', + 'metadata': { + 'dateModified': { + 'type': 'DateTime', + 'value': '2019-05-09T15:28:30.000Z' + } + } + } + }, + { + 'id': 'Room3', + 'type': 'Room', + 'temperature': { + 'value': 25.2, + 'type': 'Number', + 'metadata': { + 'dateModified': { + 'type': 'DateTime', + 'value': '2019-05-09T15:28:30.000Z' + } + } + } + } + ] + return entities + + @pytest.fixture def air_quality_observed(): """ diff --git a/src/exceptions/exceptions.py b/src/exceptions/exceptions.py index 6af570d8..d1f8e879 100644 --- a/src/exceptions/exceptions.py +++ b/src/exceptions/exceptions.py @@ -4,6 +4,9 @@ class QLError(Exception): """ Error raised in QuantumLeap usage. """ + def __init__(self, message="Quantum Leap Error"): + self.message = message + super().__init__(self.message) class UnsupportedOption(QLError): @@ -36,3 +39,12 @@ class InvalidParameterValue(QLError): def __init__(self, par_value='', par_name=''): msg = "The parameter value '{}' for parameter {} is not valid." QLError.__init__(self, msg.format(par_value, par_name)) + + +class InvalidHeaderValue(QLError): + """ + Passed parameter value is not valid. + """ + def __init__(self, header_value='', header_name='', message=''): + msg = "The header value '{}' for header {} is not valid. {}" + QLError.__init__(self, msg.format(header_value, header_name, message)) diff --git a/src/reporter/reporter.py b/src/reporter/reporter.py index 696f8395..229d1a46 100644 --- a/src/reporter/reporter.py +++ b/src/reporter/reporter.py @@ -26,6 +26,7 @@ interest and make QL actually perform the corresponding subscription to orion. I.e, QL must be told where orion is. """ + from flask import request from geocoding import geocoding from geocoding.factory import get_geo_cache, is_geo_coding_available @@ -41,13 +42,18 @@ TIME_INDEX_HEADER_NAME from geocoding.location import normalize_location from utils.cfgreader import EnvReader, StrVar +from exceptions.exceptions import AmbiguousNGSIIdError, UnsupportedOption, \ + NGSIUsageError, InvalidParameterValue, InvalidHeaderValue def log(): r = EnvReader(log=logging.getLogger(__name__).info) level = r.read(StrVar('LOGLEVEL', 'INFO')).upper() - logging.basicConfig(level=level, format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s', datefmt='%Y-%m-%d %I:%M:%S') + logging.basicConfig(level=level, + format='%(asctime)s.%(msecs)03d ' + '%(levelname)s:%(name)s:%(message)s', + datefmt='%Y-%m-%d %I:%M:%S') return logging.getLogger(__name__) @@ -105,24 +111,24 @@ def _validate_payload(payload): payload[attr].update({'value': None}) log().warning( 'An entity update is missing value for attribute {}'.format(attr)) - + def _filter_empty_entities(payload): log().debug('Received payload: {}'.format(payload)) attrs = list(iter_entity_attrs(payload)) - Flag = False + empty = False attrs.remove('time_index') for j in attrs: value = payload[j]['value'] if isinstance(value, int) and value is not None: - Flag = True + empty = True elif value: - Flag = True - if Flag: + empty = True + if empty: return payload else: return None - + def _filter_no_type_no_value_entities(payload): attrs = list(iter_entity_attrs(payload)) @@ -138,7 +144,6 @@ def _filter_no_type_no_value_entities(payload): def notify(): - if request.json is None: return 'Discarding notification due to lack of request body. ' \ 'Lost in a redirect maybe?', 400 @@ -148,12 +153,13 @@ def notify(): 'content.', 400 payload = request.json['data'] - + # preprocess and validate each entity update for entity in payload: # Validate entity update error = _validate_payload(entity) if error: + # TODO in this way we return error for even if only one entity is wrong return error, 400 # Add TIME_INDEX attribute custom_index = request.headers.get(TIME_INDEX_HEADER_NAME, None) @@ -188,11 +194,19 @@ def notify(): try: with translator_for(fiware_s) as trans: trans.insert(payload, fiware_s, fiware_sp) - except: - msg = "Notification not processed or not updated for payload: %s" % (payload) + except Exception as e: + msg = "Notification not processed or not updated for payload: %s. " \ + "%s" % (payload, str(e)) log().error(msg) - return msg, 500 - msg = "Notification successfully processed for : 'tenant' %s, 'fiwareServicePath' %s, 'entity_id' %s" % (fiware_s, fiware_sp, entity_id) + error_code = 500 + if e.__class__ == InvalidHeaderValue or \ + e.__class__ == InvalidParameterValue or \ + e.__class__ == NGSIUsageError: + error_code = 400 + return msg, error_code + msg = "Notification successfully processed for : 'tenant' %s, " \ + "'fiwareServicePath' %s, " \ + "'entity_id' %s" % (fiware_s, fiware_sp, entity_id) log().info(msg) return msg @@ -277,8 +291,7 @@ def subscribe(orion_url, if r is None or not r.ok: msg = { "error": "Bad Request", - "description": "Orion is not reachable by QuantumLeap at {}" - .format(orion_url) + "description": "Orion is not reachable at {}".format(orion_url) } return msg, 400 diff --git a/src/reporter/tests/test_notify.py b/src/reporter/tests/test_notify.py index b536851a..92e0ed11 100644 --- a/src/reporter/tests/test_notify.py +++ b/src/reporter/tests/test_notify.py @@ -1,20 +1,55 @@ from datetime import datetime, timezone from conftest import QL_URL from utils.common import assert_equal_time_index_arrays +from reporter.tests.utils import delete_entity_type import copy import json import pytest import requests import time + notify_url = "{}/notify".format(QL_URL) -HEADERS_PUT = {'Content-Type': 'application/json'} +services = ['t1', 't2'] + +SLEEP_TIME = 1 + + +def query_url(entity_type='Room', eid='Room1', attr_name='temperature', values=False): + url = "{qlUrl}/entities/{entityId}/attrs/{attrName}" + if values: + url += '/value' + return url.format( + qlUrl=QL_URL, + entityId=eid, + attrName=attr_name, + ) + + +def notify_header(service=None, service_path=None): + return headers(service, service_path, True) + +def query_header(service=None, service_path=None): + return headers(service, service_path, False) -def test_invalid_no_body(clean_mongo, clean_crate): + +def headers(service=None, service_path=None, content_type=True): + h = {} + if content_type: + h['Content-Type'] = 'application/json' + if service: + h['Fiware-Service'] = service + if service_path: + h['Fiware-ServicePath'] = service_path + + return h + +@pytest.mark.parametrize("service", services) +def test_invalid_no_body(service): r = requests.post('{}'.format(notify_url), data=json.dumps(None), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 400 assert r.json() == { 'detail': 'Request body is not valid JSON', @@ -24,19 +59,21 @@ def test_invalid_no_body(clean_mongo, clean_crate): } -def test_invalid_empty_body(clean_mongo, clean_crate): +@pytest.mark.parametrize("service", services) +def test_invalid_empty_body(service): r = requests.post('{}'.format(notify_url), data=json.dumps({}), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 400 assert r.json()['detail'] == "'data' is a required property" -def test_invalid_no_type(notification, clean_mongo, clean_crate): +@pytest.mark.parametrize("service", services) +def test_invalid_no_type(notification, service): notification['data'][0].pop('type') r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 400 assert r.json() == {'detail': "'type' is a required property - 'data.0'", 'status': 400, @@ -44,11 +81,12 @@ def test_invalid_no_type(notification, clean_mongo, clean_crate): 'type': 'about:blank'} -def test_invalid_no_id(notification, clean_mongo, clean_crate): +@pytest.mark.parametrize("service", services) +def test_invalid_no_id(notification, service): notification['data'][0].pop('id') r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 400 assert r.json() == {'detail': "'id' is a required property - 'data.0'", 'status': 400, @@ -56,58 +94,96 @@ def test_invalid_no_id(notification, clean_mongo, clean_crate): 'type': 'about:blank'} -def test_invalid_no_attr(notification, clean_mongo, clean_crate): +@pytest.mark.parametrize("service", services) +def test_invalid_no_attr(notification, service): notification['data'][0].pop('temperature') r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 200 + delete_entity_type(service, 'Room') -def test_invalid_no_value(notification, clean_mongo, clean_crate): +@pytest.mark.parametrize("service", services) +def test_invalid_no_value(notification, service): notification['data'][0]['temperature'].pop('value') r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 200 + delete_entity_type(service, 'Room') -def test_valid_notification(notification, clean_mongo, clean_crate): +@pytest.mark.parametrize("service", services) +def test_valid_notification(notification, service): r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 200 assert r.json().startswith('Notification successfully processed') + time.sleep(SLEEP_TIME) + r = requests.get(query_url(), params=None, headers=query_header(service)) + assert r.status_code == 200, r.text + delete_entity_type(service, 'Room') -def test_valid_no_modified(notification, clean_mongo, clean_crate): +@pytest.mark.parametrize("service", services) +def test_valid_no_modified(notification, service): notification['data'][0]['temperature']['metadata'].pop('dateModified') r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 200 assert r.json().startswith('Notification successfully processed') + time.sleep(SLEEP_TIME) + r = requests.get(query_url(), params=None, headers=query_header(service)) + assert r.status_code == 200, r.text + delete_entity_type(service, 'Room') + + +def do_integration(entity, subscription, orion_client, service=None, service_path=None): + orion_client.subscribe(subscription, service, service_path) + time.sleep(SLEEP_TIME) + + orion_client.insert(entity, service, service_path) + time.sleep(4 * SLEEP_TIME) # Give time for notification to be processed. -def do_integration(entity, notify_url, orion_client, crate_translator): - entity_id = entity['id'] + entities_url = "{}/entities".format(QL_URL) + + r = requests.get(entities_url, params=None, headers=None) + assert r.status_code == 200 + entities = r.json() + assert len(entities) == 1 + + assert entities[0]['id'] == entity['id'] + assert entities[0]['type'] == entity['type'] + + delete_entity_type(None, entity['type']) + + +def test_integration(entity, orion_client): + """ + Test Reporter using input directly from an Orion notification and output + directly to Cratedb. + """ subscription = { "description": "Integration Test subscription", "subject": { "entities": [ - { - "id": entity_id, - "type": "Room" - } + { + "id": entity['id'], + "type": "Room" + } ], "condition": { - "attrs": [ - "temperature", - ] + "attrs": [ + "temperature", + ] } - }, + }, "notification": { "http": { - "url": notify_url + "url": notify_url }, "attrs": [ "temperature", @@ -116,67 +192,86 @@ def do_integration(entity, notify_url, orion_client, crate_translator): }, "throttling": 1, } - orion_client.subscribe(subscription) - time.sleep(2) + do_integration(entity, subscription, orion_client) - orion_client.insert(entity) - time.sleep(4) # Give time for notification to be processed. - crate_translator._refresh([entity['type']]) - entities = crate_translator.query() - assert len(entities) == 1 - - assert entities[0]['id'] == entity['id'] - assert entities[0]['type'] == entity['type'] - obtained_values = entities[0]['temperature']['values'] +def test_air_quality_observed(air_quality_observed, orion_client): + entity = air_quality_observed + subscription = { + "description": "Test subscription", + "subject": { + "entities": [ + { + "id": entity['id'], + "type": entity['type'] + } + ], + "condition": { + "attrs": [] # all attributes + } + }, + "notification": { + "http": { + "url": notify_url + }, + "attrs": [], # all attributes + "metadata": ["dateCreated", "dateModified"] + } + } + do_integration(entity, subscription, orion_client) - # Not exactly one because first insert generates 2 notifications, see... - # https://fiware-orion.readthedocs.io/en/master/user/walkthrough_apiv2/index.html#subscriptions - expected_value = entity['temperature']['value'] - assert obtained_values[0] == pytest.approx(expected_value) -def test_integration(entity, orion_client, clean_mongo, crate_translator): +def test_integration_multiple_entities(diffEntityWithDifferentAttrs, orion_client): """ Test Reporter using input directly from an Orion notification and output directly to Cratedb. """ - do_integration(entity, notify_url, orion_client, crate_translator) -def test_air_quality_observed(air_quality_observed, orion_client, clean_mongo, - crate_translator): - entity = air_quality_observed + subscription = { - "description": "Test subscription", + "description": "Integration Test subscription", "subject": { "entities": [ - { - "id": entity['id'], - "type": entity['type'] - } + { + "idPattern": ".*", + "type": "Room" + } ], "condition": { - "attrs": [] # all attributes + "attrs": [ + "temperature", + ] } - }, + }, "notification": { "http": { - "url": notify_url + "url": notify_url }, - "attrs": [], # all attributes + "attrs": [ + "temperature", + ], "metadata": ["dateCreated", "dateModified"] - } + }, + "throttling": 1, } - orion_client.subscribe(subscription) - orion_client.insert(entity) + orion_client.subscribe(subscription, "service", "/Root/#") - time.sleep(3) # Give time for notification to be processed. + for idx, e in enumerate(diffEntityWithDifferentAttrs): + orion_client.insert(e, "service", "/Root/{}".format(idx)) + time.sleep(10 * SLEEP_TIME) # Give time for notification to be processed. - entities = crate_translator.query() - assert len(entities) == 1 + entities_url = "{}/entities".format(QL_URL) + + r = requests.get(entities_url, params=None, headers=query_header("service", "/Root")) + assert r.status_code == 200 + entities = r.json() + assert len(entities) == 3 + delete_entity_type("service", diffEntityWithDifferentAttrs[0]['type']) @pytest.mark.skip(reason="See issue #105") -def test_geocoding(notification, clean_mongo, crate_translator): +@pytest.mark.parametrize("service", services) +def test_geocoding(service, notification): # Add an address attribute to the entity notification['data'][0]['address'] = { 'type': 'StructuredValue', @@ -195,13 +290,17 @@ def test_geocoding(notification, clean_mongo, crate_translator): } r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 200 assert r.json() == 'Notification successfully processed' - time.sleep(3) # Give time for notification to be processed. + time.sleep(2 * SLEEP_TIME) # Give time for notification to be processed. + + entities_url = "{}/entities".format(QL_URL) - entities = crate_translator.query() + r = requests.get(entities_url, params=None, headers=query_header(service)) + assert r.status_code == 200 + entities = r.json() assert len(entities) == 1 assert 'location' in entities[0] @@ -209,20 +308,79 @@ def test_geocoding(notification, clean_mongo, crate_translator): lon, lat = entities[0]['location']['values'][0].split(',') assert float(lon) == pytest.approx(60.1707129, abs=1e-2) assert float(lat) == pytest.approx(24.9412167, abs=1e-2) + delete_entity_type(service, notification['data'][0]['type']) -def test_multiple_data_elements(notification, sameEntityWithDifferentAttrs, clean_mongo, clean_crate): +@pytest.mark.parametrize("service", services) +def test_multiple_data_elements(service, notification, diffEntityWithDifferentAttrs): """ Test that the notify API can process notifications containing multiple elements in the data array. """ - notification['data'] = sameEntityWithDifferentAttrs + notification['data'] = diffEntityWithDifferentAttrs + r = requests.post('{}'.format(notify_url), data=json.dumps(notification), + headers=notify_header(service)) + assert r.status_code == 200 + assert r.json().startswith('Notification successfully processed') + + entities_url = "{}/entities".format(QL_URL) + time.sleep(SLEEP_TIME) + r = requests.get(entities_url, params=None, headers=query_header(service)) + entities = r.json() + assert len(entities) == 3 + delete_entity_type(None, diffEntityWithDifferentAttrs[0]['type']) + + +@pytest.mark.parametrize("service", services) +def test_multiple_data_elements_invalid_different_servicepath(service, notification, diffEntityWithDifferentAttrs): + """ + Test that the notify API can process notifications containing multiple elements in the data array + and different fiwareServicePath. + """ + + notify_headers = notify_header(service) + + notify_headers['Fiware-ServicePath'] = '/Test/Path1, /Test/Path1, /Test/Path2, /Test/Path3' + + notification['data'] = diffEntityWithDifferentAttrs + + r = requests.post('{}'.format(notify_url), data=json.dumps(notification), + headers=notify_headers) + assert r.status_code == 400 + assert r.json().startswith('Notification not processed') + + +@pytest.mark.parametrize("service", services) +def test_multiple_data_elements_different_servicepath(service, notification, diffEntityWithDifferentAttrs): + """ + Test that the notify API can process notifications containing multiple elements in the data array + and different fiwareServicePath. + """ + + notify_headers = notify_header(service) + + notify_headers['Fiware-ServicePath'] = '/Test/Path1, /Test/Path1, /Test/Path2' + + query_headers = query_header(service) + + query_headers['Fiware-ServicePath'] = '/Test' + + notification['data'] = diffEntityWithDifferentAttrs + r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_headers) assert r.status_code == 200 assert r.json().startswith('Notification successfully processed') + entities_url = "{}/entities".format(QL_URL) + time.sleep(2*SLEEP_TIME) + r = requests.get(entities_url, params=None, headers=query_headers) + entities = r.json() + assert len(entities) == 3 + delete_entity_type(service, diffEntityWithDifferentAttrs[0]['type']) + -def test_time_index(notification, clean_mongo, crate_translator): +@pytest.mark.parametrize("service", services) +def test_time_index(service, notification): # If present, use entity-level dateModified as time_index global_modified = datetime(2000, 1, 2, 0, 0, 0, 0, timezone.utc).isoformat() modified = { @@ -233,15 +391,17 @@ def test_time_index(notification, clean_mongo, crate_translator): r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 200 assert r.json().startswith('Notification successfully processed') - time.sleep(1) + time.sleep(SLEEP_TIME) entity_type = notification['data'][0]['type'] - crate_translator._refresh([entity_type]) - entities = crate_translator.query() + entities_url = "{}/entities".format(QL_URL) + time.sleep(SLEEP_TIME) + r = requests.get(entities_url, params=None, headers=query_header(service)) + entities = r.json() assert len(entities) == 1 assert_equal_time_index_arrays(entities[0]['index'], [global_modified]) @@ -258,13 +418,13 @@ def test_time_index(notification, clean_mongo, crate_translator): r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 200 assert r.json().startswith('Notification successfully processed') - time.sleep(1) - crate_translator._refresh([entity_type]) - entities = crate_translator.query() + time.sleep(SLEEP_TIME) + r = requests.get(entities_url, params=None, headers=query_header(service)) + entities = r.json() assert len(entities) == 1 obtained = entities[0]['index'] assert_equal_time_index_arrays(obtained, [global_modified, newer]) @@ -275,19 +435,21 @@ def test_time_index(notification, clean_mongo, crate_translator): notification['data'][0]['temperature'].pop('metadata') r = requests.post('{}'.format(notify_url), data=json.dumps(notification), - headers=HEADERS_PUT) + headers=notify_header(service)) assert r.status_code == 200 assert r.json().startswith('Notification successfully processed') - time.sleep(1) - crate_translator._refresh([entity_type]) - entities = crate_translator.query() + time.sleep(SLEEP_TIME) + r = requests.get(entities_url, params=None, headers=query_header(service)) + entities = r.json() assert len(entities) == 1 obtained = entities[0]['index'] assert obtained[-1].startswith("{}".format(current.year)) + delete_entity_type(service, notification['data'][0]['type']) -def test_no_value_in_notification(notification): +@pytest.mark.parametrize("service", services) +def test_no_value_in_notification(service, notification): # No value notification['data'][0] = { 'id': '299531', @@ -298,7 +460,7 @@ def test_no_value_in_notification(notification): 'pm25': {'type': 'string', 'value': '5', 'metadata': {}}, } url = '{}'.format(notify_url) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 # Empty value @@ -310,11 +472,13 @@ def test_no_value_in_notification(notification): 'pm25': {'type': 'string', 'value': '', 'metadata': {}}, } url = '{}'.format(notify_url) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 + delete_entity_type(service, notification['data'][0]['type']) + - -def test_no_value_for_attributes(notification): +@pytest.mark.parametrize("service", services) +def test_no_value_for_attributes(service, notification): # with empty value notification['data'][0] = { 'id': '299531', @@ -326,9 +490,11 @@ def test_no_value_for_attributes(notification): url = '{}'.format(notify_url) get_url = "{}/entities/299531".format(QL_URL) url_new = '{}'.format(get_url) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 - res_get = requests.get(url_new, headers=HEADERS_PUT) + # Give time for notification to be processed + time.sleep(SLEEP_TIME) + res_get = requests.get(url_new, headers=notify_header(service)) assert res_get.status_code == 404 # entity with missing value string notification['data'][0] = { @@ -341,9 +507,11 @@ def test_no_value_for_attributes(notification): url = '{}'.format(notify_url) get_url = "{}/entities/299531/attrs/p/value".format(QL_URL) url_new = '{}'.format(get_url) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 - res_get = requests.get(url_new, headers=HEADERS_PUT) + # Give time for notification to be processed + time.sleep(SLEEP_TIME) + res_get = requests.get(url_new, headers=query_header(service)) assert res_get.status_code == 404 # entity has both valid and empty attributes notification['data'][0] = { @@ -355,15 +523,18 @@ def test_no_value_for_attributes(notification): url = '{}'.format(notify_url) get_url_new = "{}/entities/299531/attrs/pm10/value".format(QL_URL) url_new = '{}'.format(get_url_new) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 # Give time for notification to be processed - time.sleep(3) - res_get = requests.get(url_new, headers=HEADERS_PUT) + time.sleep(SLEEP_TIME) + res_get = requests.get(url_new, headers=query_header(service)) assert res_get.status_code == 200 - assert res_get.json()['values'][2] == '10' + assert res_get.json()['values'][0] == '10' + delete_entity_type(service, notification['data'][0]['type']) + -def test_no_value_no_type_for_attributes(notification): +@pytest.mark.parametrize("service", services) +def test_no_value_no_type_for_attributes(service, notification): # entity with no value and no type notification['data'][0] = { 'id': 'Room1', @@ -375,16 +546,16 @@ def test_no_value_no_type_for_attributes(notification): url = '{}'.format(notify_url) get_url = "{}/entities/Room1/attrs/temperature/value".format(QL_URL) url_new = '{}'.format(get_url) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 # Give time for notification to be processed - time.sleep(3) - res_get = requests.get(url_new, headers=HEADERS_PUT) + time.sleep(SLEEP_TIME) + res_get = requests.get(url_new, headers=query_header(service)) assert res_get.status_code == 404 # Get value of attribute having value get_url_new = "{}/entities/Room1/attrs/pressure/value".format(QL_URL) url_new = '{}'.format(get_url_new) - res_get = requests.get(url_new, headers=HEADERS_PUT) + res_get = requests.get(url_new, headers=query_header(service)) assert res_get.status_code == 200 assert res_get.json()['values'][0] == 26 @@ -397,16 +568,18 @@ def test_no_value_no_type_for_attributes(notification): url = '{}'.format(notify_url) get_url = "{}/entities/Room1/attrs/temperature/value".format(QL_URL) url_new = '{}'.format(get_url) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 # Give time for notification to be processed - time.sleep(3) - res_get = requests.get(url_new, headers=HEADERS_PUT) + time.sleep(SLEEP_TIME) + res_get = requests.get(url_new, headers=query_header(service)) assert res_get.status_code == 200 assert res_get.json()['values'][1] == '25' + delete_entity_type(service, notification['data'][0]['type']) -def test_with_value_no_type_for_attributes(notification): +@pytest.mark.parametrize("service", services) +def test_with_value_no_type_for_attributes(service, notification): # entity with value and no type notification['data'][0] = { 'id': 'Kitchen1', @@ -418,15 +591,18 @@ def test_with_value_no_type_for_attributes(notification): url = '{}'.format(notify_url) get_url = "{}/entities/Kitchen1/attrs/temperature/value".format(QL_URL) url_new = '{}'.format(get_url) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 # Give time for notification to be processed - time.sleep(3) - res_get = requests.get(url_new, headers=HEADERS_PUT) + time.sleep(SLEEP_TIME) + res_get = requests.get(url_new, headers=query_header(service)) assert res_get.status_code == 200 assert res_get.json()['values'][0] == '25' + delete_entity_type(service, notification['data'][0]['type']) -def test_no_value_with_type_for_attributes(notification): + +@pytest.mark.parametrize("service", services) +def test_no_value_with_type_for_attributes(service, notification): # entity with one Null value and no type notification['data'][0] = { 'id': 'Hall1', @@ -438,12 +614,11 @@ def test_no_value_with_type_for_attributes(notification): url = '{}'.format(notify_url) get_url = "{}/entities/Hall1/attrs/temperature/value".format(QL_URL) url_new = '{}'.format(get_url) - r = requests.post(url, data=json.dumps(notification), headers=HEADERS_PUT) + r = requests.post(url, data=json.dumps(notification), headers=notify_header(service)) assert r.status_code == 200 # Give time for notification to be processed - time.sleep(3) - res_get = requests.get(url_new, headers=HEADERS_PUT) + time.sleep(SLEEP_TIME) + res_get = requests.get(url_new, headers=query_header(service)) assert res_get.status_code == 200 assert res_get.json()['values'][0] == None - - + delete_entity_type(service, notification['data'][0]['type']) \ No newline at end of file diff --git a/src/reporter/tests/test_subscribe.py b/src/reporter/tests/test_subscribe.py index d3c2aa17..af514bdc 100644 --- a/src/reporter/tests/test_subscribe.py +++ b/src/reporter/tests/test_subscribe.py @@ -13,7 +13,7 @@ def test_invalid_wrong_orion_url(clean_mongo, clean_crate): assert r.status_code == 400 assert r.json() == { "error": "Bad Request", - "description": "Orion is not reachable by QuantumLeap at blabla" + "description": "Orion is not reachable at blabla" } diff --git a/src/reporter/tests/utils.py b/src/reporter/tests/utils.py index ac7aa908..b2df17e1 100644 --- a/src/reporter/tests/utils.py +++ b/src/reporter/tests/utils.py @@ -85,7 +85,9 @@ def insert_test_data(service, entity_types, n_entities=1, index_size=30, def delete_entity_type(service, entity_type): - h = {'Fiware-Service': service} + h = {} + if service: + h = {'Fiware-Service': service} url = '{}/types/{}'.format(QL_URL, entity_type) r = requests.delete(url, headers=h) diff --git a/src/tests/run_load_tests.sh b/src/tests/run_load_tests.sh index 530300c0..02900efe 100644 --- a/src/tests/run_load_tests.sh +++ b/src/tests/run_load_tests.sh @@ -15,4 +15,4 @@ docker run -i --rm loadimpact/k6 run --vus 100 --duration 120s - < notify-load-t sleep 10 -docker-compose down +docker-compose down -v diff --git a/src/tests/run_tests.sh b/src/tests/run_tests.sh index 210097d1..f0e36a3a 100644 --- a/src/tests/run_tests.sh +++ b/src/tests/run_tests.sh @@ -8,6 +8,8 @@ CRATE_VERSION=${PREV_CRATE} docker-compose pull --ignore-pull-failures tot=0 # Launch services with previous CRATE and QL version +echo "\n" +echo "Launch services with previous CRATE and QL version" CRATE_VERSION=${PREV_CRATE} QL_IMAGE=${QL_PREV_IMAGE} docker-compose up -d sleep 10 @@ -16,9 +18,13 @@ ORION_HOST=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}} QUANTUMLEAP_HOST=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps | grep "8668" | awk '{ print $1 }')` # Load data +echo "\n" +echo "Load data" docker run -ti --rm --network tests_default \ -e ORION_URL="http://$ORION_HOST:1026" \ -e QL_URL="http://$QUANTUMLEAP_HOST:8668" \ + --entrypoint "" \ + -e USE_FLASK=TRUE \ smartsdk/quantumleap python tests/common.py # Restart QL on development version and CRATE on current version @@ -27,11 +33,15 @@ CRATE_VERSION=${CRATE_VERSION} QL_IMAGE=smartsdk/quantumleap docker-compose up - sleep 40 # Backwards Compatibility Test +echo "\n" +echo "Backwards Compatibility Test" cd ../../ pytest src/tests/test_bc.py --cov-report= --cov-config=.coveragerc --cov-append --cov=src/ tot=$? # Integration Test +echo "\n" +echo "Integration Test" pytest src/tests/test_integration.py --cov-report= --cov-config=.coveragerc --cov-append --cov=src/ loc=$? if [ "$tot" -eq 0 ]; then diff --git a/src/translators/sql_translator.py b/src/translators/sql_translator.py index 15d62131..b4825364 100644 --- a/src/translators/sql_translator.py +++ b/src/translators/sql_translator.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta, timezone from geocoding.slf.geotypes import * from exceptions.exceptions import AmbiguousNGSIIdError, UnsupportedOption, \ - NGSIUsageError, InvalidParameterValue + NGSIUsageError, InvalidParameterValue, InvalidHeaderValue from translators import base_translator from translators.config import SQLTranslatorConfig from utils.common import iter_entity_attrs @@ -13,7 +13,6 @@ from typing import Any, List, Optional from uuid import uuid4 - # NGSI TYPES # Based on Orion output because official docs don't say much about these :( NGSI_DATETIME = 'DateTime' @@ -147,7 +146,8 @@ def _et2tn(self, entity_type, fiware_service=None): """ et = '"{}{}"'.format(TYPE_PREFIX, entity_type.lower()) if fiware_service: - return '"{}{}".{}'.format(TENANT_PREFIX, fiware_service.lower(), et) + return '"{}{}".{}'.format(TENANT_PREFIX, fiware_service.lower(), + et) return et def _ea2cn(self, entity_attr): @@ -170,16 +170,46 @@ def insert(self, entities, fiware_service=None, fiware_servicepath=None): msg = "Entities expected to be of type list, but got {}" raise TypeError(msg.format(type(entities))) - entities_by_type = {} - for e in entities: - entities_by_type.setdefault(e['type'], []).append(e) - - res = None - for et in entities_by_type.keys(): - res = self._insert_entities_of_type(et, - entities_by_type[et], - fiware_service, - fiware_servicepath) + service_paths = [] + if fiware_servicepath: + clean_fiware_servicepath = fiware_servicepath.replace(" ", "") + service_paths = clean_fiware_servicepath.split(",") + else: + service_paths = [fiware_servicepath] + if len(service_paths) == 1: + entities_by_type = {} + for e in entities: + entities_by_type.setdefault(e['type'], []).append(e) + + res = None + for et in entities_by_type.keys(): + res = self._insert_entities_of_type(et, + entities_by_type[et], + fiware_service, + service_paths[0]) + elif len(service_paths) == len(entities): + entities_by_service_path = {} + for idx, path in enumerate(service_paths): + entities_by_service_path.setdefault(path, []).append( + entities[idx]) + res = None + for path in entities_by_service_path.keys(): + entities_by_type = {} + for e in entities_by_service_path[path]: + entities_by_type.setdefault(e['type'], []).append(e) + + res = None + for et in entities_by_type.keys(): + res = self._insert_entities_of_type(et, + entities_by_type[et], + fiware_service, + path) + else: + msg = 'Multiple servicePath are allowed only ' \ + 'if their size is match the size of entities' + raise InvalidHeaderValue('Fiware-ServicePath', + fiware_servicepath, msg) + return res def _insert_entities_of_type(self, @@ -256,7 +286,8 @@ def _insert_entities_of_type(self, "Please use any of the following: {}. " "Falling back to {}.") self.logger.warning(msg.format( - attr_t, attr, entity_id, supported_types, NGSI_TEXT)) + attr_t, attr, entity_id, supported_types, + NGSI_TEXT)) table[col] = self.NGSI_TO_SQL[NGSI_TEXT] @@ -281,7 +312,8 @@ def _insert_entities_of_type(self, col_names = sorted(table.keys()) entries = [] # raw values in same order as column names for e in entities: - values = self._preprocess_values(e, table, col_names, fiware_servicepath) + values = self._preprocess_values(e, table, col_names, + fiware_servicepath) entries.append(values) # Insert entities data @@ -320,6 +352,7 @@ def _build_insert_params_and_values( col_list = ', '.join(['"{}"'.format(c.lower()) for c in col_names]) placeholders = ','.join(['?'] * len(col_names)) return col_list, placeholders, rows + # NOTE. Brittle code. # This code, like the rest of the insert workflow implicitly assumes # 1. col_names[k] <-> rows[k] <-> entities[k] @@ -344,7 +377,8 @@ def _build_original_data_value(self, entity: dict, def _to_db_ngsi_structured_value(data: dict) -> Any: return data - def _should_insert_original_entities(self, insert_error: Exception) -> bool: + def _should_insert_original_entities(self, + insert_error: Exception) -> bool: raise NotImplementedError def _insert_original_entities_in_failed_batch( @@ -450,7 +484,8 @@ def _get_select_clause(self, attr_names, aggr_method, aggr_period): if aggr_period: attrs.append( "DATE_TRUNC('{}',{}) as {}".format( - aggr_period, self.TIME_INDEX_NAME, self.TIME_INDEX_NAME) + aggr_period, self.TIME_INDEX_NAME, + self.TIME_INDEX_NAME) ) # TODO: https://github.com/smartsdk/ngsi-timeseries-api/issues/106 m = '{}("{}") as "{}"' @@ -492,11 +527,13 @@ def _get_where_clause(self, entity_ids, from_date, to_date, fiware_sp=None, clauses.append(" {} >= '{}'".format(self.TIME_INDEX_NAME, self._parse_date(from_date))) if to_date: - clauses.append(" {} <= '{}'".format(self.TIME_INDEX_NAME, self._parse_date(to_date))) + clauses.append(" {} <= '{}'".format(self.TIME_INDEX_NAME, + self._parse_date(to_date))) if fiware_sp: # Match prefix of fiware service path - clauses.append(" " + FIWARE_SERVICEPATH + " ~* '" + fiware_sp + "($|/.*)'") + clauses.append( + " " + FIWARE_SERVICEPATH + " ~* '" + fiware_sp + "($|/.*)'") else: # Match prefix of fiware service path clauses.append(" " + FIWARE_SERVICEPATH + " = ''") @@ -549,7 +586,8 @@ def _get_order_group_clause(self, aggr_method, aggr_period, if aggr_period: # consider always ordering by entity_id also order_by.extend(["entity_type", "entity_id"]) - order_by.append("{} {}".format(self.TIME_INDEX_NAME, direction)) + order_by.append( + "{} {}".format(self.TIME_INDEX_NAME, direction)) else: order_by.append("{} {}".format(self.TIME_INDEX_NAME, direction))