From 098e3e5eb4718e2687b0cdfd056fcbc7ea211925 Mon Sep 17 00:00:00 2001 From: c0c0n3 Date: Fri, 11 Sep 2020 12:36:48 +0200 Subject: [PATCH] Gunicorn plumbing (#357) * port notify load test from #354. * make sure load test uses latest ql code and removes containers to avoid undeletable dangling images. * port gunicorn setup from #354. * get rid of supervisord. * move gunicorn dep to pipfile; flexible wgsi launcher; use gthread instead of gevent; consolidate gunicorn config. * make test scripts give crate enough time to get 100% functional before starting to hammer it. * polish gunicorn runner; make config easily overridable in docker container. * skip broken geocoding test as it has nothing to do w/ this pr. --- Dockerfile | 24 ++- Pipfile | 1 + Pipfile.lock | 262 +++++++++++++------------- src/app.py | 21 ++- src/geocoding/tests/test_geocoding.py | 2 + src/reporter/tests/run_tests.sh | 2 +- src/server/__init__.py | 3 + src/server/gconfig.py | 74 ++++++++ src/server/grunner.py | 81 ++++++++ src/server/wsgi.py | 54 ++++++ src/tests/docker-compose.yml | 6 +- src/tests/notify-load-test.js | 49 +++++ src/tests/run_load_tests.sh | 18 ++ src/tests/run_tests.sh | 2 +- src/utils/hosts.py | 1 - 15 files changed, 459 insertions(+), 141 deletions(-) create mode 100644 src/server/__init__.py create mode 100644 src/server/gconfig.py create mode 100644 src/server/grunner.py create mode 100644 src/server/wsgi.py create mode 100644 src/tests/notify-load-test.js create mode 100644 src/tests/run_load_tests.sh delete mode 100644 src/utils/hosts.py diff --git a/Dockerfile b/Dockerfile index 61bdb600..b3e898f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,4 +16,26 @@ COPY . /src/ngsi-timeseries-api/ WORKDIR /src/ngsi-timeseries-api/src ENV PYTHONPATH=$PWD:$PYTHONPATH -CMD python app.py +EXPOSE 8668 +ENTRYPOINT ["python", "app.py"] +# NOTE. +# The above is basically the same as running: +# +# gunicorn server.wsgi --config server/gconfig.py +# +# You can also pass any valid Gunicorn option as container command arguments +# to add or override options in server/gconfig.py---see `server.grunner` for +# the details. +# In particular, a convenient way to reconfigure Gunicorn is to mount a config +# file on the container and then run the container with the following option +# +# --config /path/to/where/you/mounted/your/gunicorn.conf.py +# +# as in the below example +# +# $ echo 'workers = 2' > gunicorn.conf.py +# $ docker run -it --rm \ +# -p 8668:8668 \ +# -v $(pwd)/gunicorn.conf.py:/gunicorn.conf.py +# smartsdk/quantumleap --config /gunicorn.conf.py +# \ No newline at end of file diff --git a/Pipfile b/Pipfile index ca4c4da7..824b0198 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ flask = "~=1.1.0" geocoder = "~=1.33" geojson = "~=2.4" geomet = "~=0.2" +gunicorn = "~=20.0.4" influxdb = "~=4.0" pg8000 = ">=1.15" pymongo = "~=3.4" diff --git a/Pipfile.lock b/Pipfile.lock index 29d1379a..467126e5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a9a75af80a8be871c634db17f56aad540cb9a9b827b4824e5f52b1a4a90f9360" + "sha256": "4e58db5aca3cf9edf61c5649f8ef5e6e696e41845ebabf7b6f01288f7d48fbd6" }, "pipfile-spec": 6, "requires": { @@ -25,10 +25,10 @@ }, "attrs": { "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", + "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" ], - "version": "==19.3.0" + "version": "==20.2.0" }, "certifi": { "hashes": [ @@ -72,55 +72,58 @@ }, "coverage": { "hashes": [ - "sha256:00f1d23f4336efc3b311ed0d807feb45098fc86dee1ca13b3d6768cdab187c8a", - "sha256:01333e1bd22c59713ba8a79f088b3955946e293114479bbfc2e37d522be03355", - "sha256:0cb4be7e784dcdc050fc58ef05b71aa8e89b7e6636b99967fadbdba694cf2b65", - "sha256:0e61d9803d5851849c24f78227939c701ced6704f337cad0a91e0972c51c1ee7", - "sha256:1601e480b9b99697a570cea7ef749e88123c04b92d84cedaa01e117436b4a0a9", - "sha256:2742c7515b9eb368718cd091bad1a1b44135cc72468c731302b3d641895b83d1", - "sha256:2d27a3f742c98e5c6b461ee6ef7287400a1956c11421eb574d843d9ec1f772f0", - "sha256:402e1744733df483b93abbf209283898e9f0d67470707e3c7516d84f48524f55", - "sha256:5c542d1e62eece33c306d66fe0a5c4f7f7b3c08fecc46ead86d7916684b36d6c", - "sha256:5f2294dbf7875b991c381e3d5af2bcc3494d836affa52b809c91697449d0eda6", - "sha256:6402bd2fdedabbdb63a316308142597534ea8e1895f4e7d8bf7476c5e8751fef", - "sha256:66460ab1599d3cf894bb6baee8c684788819b71a5dc1e8fa2ecc152e5d752019", - "sha256:782caea581a6e9ff75eccda79287daefd1d2631cc09d642b6ee2d6da21fc0a4e", - "sha256:79a3cfd6346ce6c13145731d39db47b7a7b859c0272f02cdb89a3bdcbae233a0", - "sha256:7a5bdad4edec57b5fb8dae7d3ee58622d626fd3a0be0dfceda162a7035885ecf", - "sha256:8fa0cbc7ecad630e5b0f4f35b0f6ad419246b02bc750de7ac66db92667996d24", - "sha256:a027ef0492ede1e03a8054e3c37b8def89a1e3c471482e9f046906ba4f2aafd2", - "sha256:a3f3654d5734a3ece152636aad89f58afc9213c6520062db3978239db122f03c", - "sha256:a82b92b04a23d3c8a581fc049228bafde988abacba397d57ce95fe95e0338ab4", - "sha256:acf3763ed01af8410fc36afea23707d4ea58ba7e86a8ee915dfb9ceff9ef69d0", - "sha256:adeb4c5b608574a3d647011af36f7586811a2c1197c861aedb548dd2453b41cd", - "sha256:b83835506dfc185a319031cf853fa4bb1b3974b1f913f5bb1a0f3d98bdcded04", - "sha256:bb28a7245de68bf29f6fb199545d072d1036a1917dca17a1e75bbb919e14ee8e", - "sha256:bf9cb9a9fd8891e7efd2d44deb24b86d647394b9705b744ff6f8261e6f29a730", - "sha256:c317eaf5ff46a34305b202e73404f55f7389ef834b8dbf4da09b9b9b37f76dd2", - "sha256:dbe8c6ae7534b5b024296464f387d57c13caa942f6d8e6e0346f27e509f0f768", - "sha256:de807ae933cfb7f0c7d9d981a053772452217df2bf38e7e6267c9cbf9545a796", - "sha256:dead2ddede4c7ba6cb3a721870f5141c97dc7d85a079edb4bd8d88c3ad5b20c7", - "sha256:dec5202bfe6f672d4511086e125db035a52b00f1648d6407cc8e526912c0353a", - "sha256:e1ea316102ea1e1770724db01998d1603ed921c54a86a2efcb03428d5417e489", - "sha256:f90bfc4ad18450c80b024036eaf91e4a246ae287701aaa88eaebebf150868052" - ], - "version": "==5.1" + "sha256:098a703d913be6fbd146a8c50cc76513d726b022d170e5e98dc56d958fd592fb", + "sha256:16042dc7f8e632e0dcd5206a5095ebd18cb1d005f4c89694f7f8aafd96dd43a3", + "sha256:1adb6be0dcef0cf9434619d3b892772fdb48e793300f9d762e480e043bd8e716", + "sha256:27ca5a2bc04d68f0776f2cdcb8bbd508bbe430a7bf9c02315cd05fb1d86d0034", + "sha256:28f42dc5172ebdc32622a2c3f7ead1b836cdbf253569ae5673f499e35db0bac3", + "sha256:2fcc8b58953d74d199a1a4d633df8146f0ac36c4e720b4a1997e9b6327af43a8", + "sha256:304fbe451698373dc6653772c72c5d5e883a4aadaf20343592a7abb2e643dae0", + "sha256:30bc103587e0d3df9e52cd9da1dd915265a22fad0b72afe54daf840c984b564f", + "sha256:40f70f81be4d34f8d491e55936904db5c527b0711b2a46513641a5729783c2e4", + "sha256:4186fc95c9febeab5681bc3248553d5ec8c2999b8424d4fc3a39c9cba5796962", + "sha256:46794c815e56f1431c66d81943fa90721bb858375fb36e5903697d5eef88627d", + "sha256:4869ab1c1ed33953bb2433ce7b894a28d724b7aa76c19b11e2878034a4e4680b", + "sha256:4f6428b55d2916a69f8d6453e48a505c07b2245653b0aa9f0dee38785939f5e4", + "sha256:52f185ffd3291196dc1aae506b42e178a592b0b60a8610b108e6ad892cfc1bb3", + "sha256:538f2fd5eb64366f37c97fdb3077d665fa946d2b6d95447622292f38407f9258", + "sha256:64c4f340338c68c463f1b56e3f2f0423f7b17ba6c3febae80b81f0e093077f59", + "sha256:675192fca634f0df69af3493a48224f211f8db4e84452b08d5fcebb9167adb01", + "sha256:700997b77cfab016533b3e7dbc03b71d33ee4df1d79f2463a318ca0263fc29dd", + "sha256:8505e614c983834239f865da2dd336dcf9d72776b951d5dfa5ac36b987726e1b", + "sha256:962c44070c281d86398aeb8f64e1bf37816a4dfc6f4c0f114756b14fc575621d", + "sha256:9e536783a5acee79a9b308be97d3952b662748c4037b6a24cbb339dc7ed8eb89", + "sha256:9ea749fd447ce7fb1ac71f7616371f04054d969d412d37611716721931e36efd", + "sha256:a34cb28e0747ea15e82d13e14de606747e9e484fb28d63c999483f5d5188e89b", + "sha256:a3ee9c793ffefe2944d3a2bd928a0e436cd0ac2d9e3723152d6fd5398838ce7d", + "sha256:aab75d99f3f2874733946a7648ce87a50019eb90baef931698f96b76b6769a46", + "sha256:b1ed2bdb27b4c9fc87058a1cb751c4df8752002143ed393899edb82b131e0546", + "sha256:b360d8fd88d2bad01cb953d81fd2edd4be539df7bfec41e8753fe9f4456a5082", + "sha256:b8f58c7db64d8f27078cbf2a4391af6aa4e4767cc08b37555c4ae064b8558d9b", + "sha256:c1bbb628ed5192124889b51204de27c575b3ffc05a5a91307e7640eff1d48da4", + "sha256:c2ff24df02a125b7b346c4c9078c8936da06964cc2d276292c357d64378158f8", + "sha256:c890728a93fffd0407d7d37c1e6083ff3f9f211c83b4316fae3778417eab9811", + "sha256:c96472b8ca5dc135fb0aa62f79b033f02aa434fb03a8b190600a5ae4102df1fd", + "sha256:ce7866f29d3025b5b34c2e944e66ebef0d92e4a4f2463f7266daa03a1332a651", + "sha256:e26c993bd4b220429d4ec8c1468eca445a4064a61c74ca08da7429af9bc53bb0" + ], + "version": "==5.2.1" }, "coveralls": { "hashes": [ - "sha256:41bd57b60321dfd5b56e990ab3f7ed876090691c21a9e3b005e1f6e42e6ba4b9", - "sha256:d213f5edd49053d03f0db316ccabfe17725f2758147afc9a37eaca9d8e8602b5" + "sha256:4430b862baabb3cf090d36d84d331966615e4288d8a8c5957e0fd456d0dd8bd6", + "sha256:b3b60c17b03a0dee61952a91aed6f131e0b2ac8bd5da909389c53137811409e1" ], "index": "pypi", - "version": "==2.0.0" + "version": "==2.1.2" }, "crate": { "hashes": [ - "sha256:5602663ae407f329dc07a58bd8669908c3226a2b5ba3234a8b30df4aeac9cbe8", - "sha256:5e416ffeee74b839e297d60d930202c11fefe8855f4dae0a0a78628f434e1809" + "sha256:23e525cfe83aa2e00c8c00bd2c4f7b3b7038bd65e27bd347d24491e42c42554a", + "sha256:2de19674271e3a2feae8380fd9418bae536f5d246e93cd68dbb7a932f52c9c19" ], "index": "pypi", - "version": "==0.24.0" + "version": "==0.25.0" }, "decorator": { "hashes": [ @@ -173,19 +176,27 @@ "index": "pypi", "version": "==0.2.1.post1" }, + "gunicorn": { + "hashes": [ + "sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626", + "sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c" + ], + "index": "pypi", + "version": "==20.0.4" + }, "idna": { "hashes": [ - "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", - "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], - "version": "==2.9" + "version": "==2.10" }, "inflection": { "hashes": [ - "sha256:88b101b2668a1d81d6d72d4c2018e53bc6c7fc544c987849da1c7f77545c3bc9", - "sha256:f576e85132d34f5bf7df5183c2c6f94cfb32e528f53065345cf71329ba0b8924" + "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", + "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2" ], - "version": "==0.5.0" + "version": "==0.5.1" }, "influxdb": { "hashes": [ @@ -256,26 +267,26 @@ }, "more-itertools": { "hashes": [ - "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5", - "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2" + "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", + "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" ], - "version": "==8.4.0" + "version": "==8.5.0" }, "openapi-spec-validator": { "hashes": [ - "sha256:0caacd9829e9e3051e830165367bf58d436d9487b29a09220fa7edb9f47ff81b", - "sha256:d4da8aef72bf5be40cf0df444abd20009a41baf9048a8e03750c07a934f1bdd8", - "sha256:e489c7a273284bc78277ac22791482e8058d323b4a265015e9fcddf6a8045bcd" + "sha256:6dd75e50c94f1bb454d0e374a56418e7e06a07affb2c7f1df88564c5d728dac3", + "sha256:79381a69b33423ee400ae1624a461dae7725e450e2e306e32f2dd8d16a4d85cb", + "sha256:ec1b01a00e20955a527358886991ae34b4b791b253027ee9f7df5f84b59d91c7" ], - "version": "==0.2.8" + "version": "==0.2.9" }, "pg8000": { "hashes": [ - "sha256:79d2e761343e582dec6698cf7c06d49c33255cbafba29ff298dd4690fffb7d80", - "sha256:af97353076b8e5d271d91c64c8ca806e2157d11b7862c90ff6f0e23be0fc217d" + "sha256:3d646b11227d94a3130a765a981dc6323bc959a3cd6ed54421d174b2ef256087", + "sha256:8af70cdfcc1fadafa32468a6af563e1c0b5271c4dcc99a4490030a128cb295a3" ], "index": "pypi", - "version": "==1.15.3" + "version": "==1.16.5" }, "pluggy": { "hashes": [ @@ -286,69 +297,69 @@ }, "py": { "hashes": [ - "sha256:a673fa23d7000440cc885c17dbd34fafcb7d7a6e230b29f6766400de36a33c44", - "sha256:f3b3a4c36512a4c4f024041ab51866f11761cc169670204b235f6b20523d4e6b" + "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", + "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" ], - "version": "==1.8.2" + "version": "==1.9.0" }, "pymongo": { "hashes": [ - "sha256:01b4e10027aef5bb9ecefbc26f5df3368ce34aef81df43850f701e716e3fe16d", - "sha256:0fc5aa1b1acf7f61af46fe0414e6a4d0c234b339db4c03a63da48599acf1cbfc", - "sha256:1396eb7151e0558b1f817e4b9d7697d5599e5c40d839a9f7270bd90af994ad82", - "sha256:18e84a3ec5e73adcb4187b8e5541b2ad61d716026ed9863267e650300d8bea33", - "sha256:19adf2848b80cb349b9891cc854581bbf24c338be9a3260e73159bdeb2264464", - "sha256:20ee0475aa2ba437b0a14806f125d696f90a8433d820fb558fdd6f052acde103", - "sha256:26798795097bdeb571f13942beef7e0b60125397811c75b7aa9214d89880dd1d", - "sha256:26e707a4eb851ec27bb969b5f1413b9b2eac28fe34271fa72329100317ea7c73", - "sha256:2a3c7ad01553b27ec553688a1e6445e7f40355fb37d925c11fcb50b504e367f8", - "sha256:2f07b27dbf303ea53f4147a7922ce91a26b34a0011131471d8aaf73151fdee9a", - "sha256:316f0cf543013d0c085e15a2c8abe0db70f93c9722c0f99b6f3318ff69477d70", - "sha256:31d11a600eea0c60de22c8bdcb58cda63c762891facdcb74248c36713240987f", - "sha256:334ef3ffd0df87ea83a0054454336159f8ad9c1b389e19c0032d9cb8410660e6", - "sha256:358ba4693c01022d507b96a980ded855a32dbdccc3c9331d0667be5e967f30ed", - "sha256:3a6568bc53103df260f5c7d2da36dffc5202b9a36c85540bba1836a774943794", - "sha256:444bf2f44264578c4085bb04493bfed0e5c1b4fe7c2704504d769f955cc78fe4", - "sha256:47a00b22c52ee59dffc2aad02d0bbfb20c26ec5b8de8900492bf13ad6901cf35", - "sha256:4c067db43b331fc709080d441cb2e157114fec60749667d12186cc3fc8e7a951", - "sha256:4c092310f804a5d45a1bcaa4191d6d016c457b6ed3982a622c35f729ff1c7f6b", - "sha256:53b711b33134e292ef8499835a3df10909c58df53a2a0308f598c432e9a62892", - "sha256:568d6bee70652d8a5af1cd3eec48b4ca1696fb1773b80719ebbd2925b72cb8f6", - "sha256:56fa55032782b7f8e0bf6956420d11e2d4e9860598dfe9c504edec53af0fc372", - "sha256:5a2c492680c61b440272341294172fa3b3751797b1ab983533a770e4fb0a67ac", - "sha256:61235cc39b5b2f593086d1d38f3fc130b2d125bd8fc8621d35bc5b6bdeb92bd2", - "sha256:619ac9aaf681434b4d4718d1b31aa2f0fce64f2b3f8435688fcbdc0c818b6c54", - "sha256:6238ac1f483494011abde5286282afdfacd8926659e222ba9b74c67008d3a58c", - "sha256:63752a72ca4d4e1386278bd43d14232f51718b409e7ac86bcf8810826b531113", - "sha256:6fdc5ccb43864065d40dd838437952e9e3da9821b7eac605ba46ada77f846bdf", - "sha256:7abc3a6825a346fa4621a6f63e3b662bbb9e0f6ffc32d30a459d695f20fb1a8b", - "sha256:7aef381bb9ae8a3821abd7f9d4d93978dbd99072b48522e181baeffcd95b56ae", - "sha256:80df3caf251fe61a3f0c9614adc6e2bfcffd1cd3345280896766712fb4b4d6d7", - "sha256:95f970f34b59987dee6f360d2e7d30e181d58957b85dff929eee4423739bd151", - "sha256:993257f6ca3cde55332af1f62af3e04ca89ce63c08b56a387cdd46136c72f2fa", - "sha256:9c0a57390549affc2b5dda24a38de03a5c7cbc58750cd161ff5d106c3c6eec80", - "sha256:a0794e987d55d2f719cc95fcf980fc62d12b80e287e6a761c4be14c60bd9fecc", - "sha256:a3b98121e68bf370dd8ea09df67e916f93ea95b52fc010902312168c4d1aff5d", - "sha256:a60756d55f0887023b3899e6c2923ba5f0042fb11b1d17810b4e07395404f33e", - "sha256:a676bd2fbc2309092b9bbb0083d35718b5420af3a42135ebb1e4c3633f56604d", - "sha256:a732838c78554c1257ff2492f5c8c4c7312d0aecd7f732149e255f3749edd5ee", - "sha256:ae65d65fde4135ef423a2608587c9ef585a3551fc2e4e431e7c7e527047581be", - "sha256:b070a4f064a9edb70f921bfdc270725cff7a78c22036dd37a767c51393fb956f", - "sha256:b6da85949aa91e9f8c521681344bd2e163de894a5492337fba8b05c409225a4f", - "sha256:bbf47110765b2a999803a7de457567389253f8670f7daafb98e059c899ce9764", - "sha256:c06b3f998d2d7160db58db69adfb807d2ec307e883e2f17f6b87a1ef6c723f11", - "sha256:c318fb70542be16d3d4063cde6010b1e4d328993a793529c15a619251f517c39", - "sha256:c4aef42e5fa4c9d5a99f751fb79caa880dac7eaf8a65121549318b984676a1b7", - "sha256:c9ca545e93a9c2a3bdaa2e6e21f7a43267ff0813e8055adf2b591c13164c0c57", - "sha256:da2c3220eb55c4239dd8b982e213da0b79023cac59fe54ca09365f2bc7e4ad32", - "sha256:dd8055da300535eefd446b30995c0813cc4394873c9509323762a93e97c04c03", - "sha256:e2b46e092ea54b732d98c476720386ff2ccd126de1e52076b470b117bff7e409", - "sha256:e334c4f39a2863a239d38b5829e442a87f241a92da9941861ee6ec5d6380b7fe", - "sha256:e5c54f04ca42bbb5153aec5d4f2e3d9f81e316945220ac318abd4083308143f5", - "sha256:f96333f9d2517c752c20a35ff95de5fc2763ac8cdb1653df0f6f45d281620606" + "sha256:03dc64a9aa7a5d405aea5c56db95835f6a2fa31b3502c5af1760e0e99210be30", + "sha256:05fcc6f9c60e6efe5219fbb5a30258adb3d3e5cbd317068f3d73c09727f2abb6", + "sha256:076a7f2f7c251635cf6116ac8e45eefac77758ee5a77ab7bd2f63999e957613b", + "sha256:137e6fa718c7eff270dbd2fc4b90d94b1a69c9e9eb3f3de9e850a7fd33c822dc", + "sha256:1f865b1d1c191d785106f54df9abdc7d2f45a946b45fd1ea0a641b4f982a2a77", + "sha256:213c445fe7e654621c6309e874627c35354b46ef3ee807f5a1927dc4b30e1a67", + "sha256:25e617daf47d8dfd4e152c880cd0741cbdb48e51f54b8de9ddbfe74ecd87dd16", + "sha256:3d9bb1ba935a90ec4809a8031efd988bdb13cdba05d9e9a3e9bf151bf759ecde", + "sha256:40696a9a53faa7d85aaa6fd7bef1cae08f7882640bad08c350fb59dee7ad069b", + "sha256:421aa1b92c291c429668bd8d8d8ec2bd00f183483a756928e3afbf2b6f941f00", + "sha256:4437300eb3a5e9cc1a73b07d22c77302f872f339caca97e9bf8cf45eca8fa0d2", + "sha256:455f4deb00158d5ec8b1d3092df6abb681b225774ab8a59b3510293b4c8530e3", + "sha256:475a34a0745c456ceffaec4ce86b7e0983478f1b6140890dff7b161e7bcd895b", + "sha256:4797c0080f41eba90404335e5ded3aa66731d303293a675ff097ce4ea3025bb9", + "sha256:4ae23fbbe9eadf61279a26eba866bbf161a6f7e2ffad14a42cf20e9cb8e94166", + "sha256:4b32744901ee9990aa8cd488ec85634f443526def1e5190a407dc107148249d7", + "sha256:50127b13b38e8e586d5e97d342689405edbd74ad0bd891d97ee126a8c7b6e45f", + "sha256:50531caa7b4be1c4ed5e2d5793a4e51cc9bd62a919a6fd3299ef7c902e206eab", + "sha256:68220b81850de8e966d4667d5c325a96c6ac0d6adb3d18935d6e3d325d441f48", + "sha256:689142dc0c150e9cb7c012d84cac2c346d40beb891323afb6caf18ec4caafae0", + "sha256:6a15e2bee5c4188369a87ed6f02de804651152634a46cca91966a11c8abd2550", + "sha256:7122ffe597b531fb065d3314e704a6fe152b81820ca5f38543e70ffcc95ecfd4", + "sha256:7307024b18266b302f4265da84bb1effb5d18999ef35b30d17592959568d5c0a", + "sha256:7a4a6f5b818988a3917ec4baa91d1143242bdfece8d38305020463955961266a", + "sha256:83c5a3ecd96a9f3f11cfe6dfcbcec7323265340eb24cc996acaecea129865a3a", + "sha256:890b0f1e18dbd898aeb0ab9eae1ab159c6bcbe87f0abb065b0044581d8614062", + "sha256:8deda1f7b4c03242f2a8037706d9584e703f3d8c74d6d9cac5833db36fe16c42", + "sha256:8ea13d0348b4c96b437d944d7068d59ed4a6c98aaa6c40d8537a2981313f1c66", + "sha256:91e96bf85b7c07c827d339a386e8a3cf2e90ef098c42595227f729922d0851df", + "sha256:96782ebb3c9e91e174c333208b272ea144ed2a684413afb1038e3b3342230d72", + "sha256:9755c726aa6788f076114dfdc03b92b03ff8860316cca00902cce88bcdb5fedd", + "sha256:9dbab90c348c512e03f146e93a5e2610acec76df391043ecd46b6b775d5397e6", + "sha256:9ee0eef254e340cc11c379f797af3977992a7f2c176f1a658740c94bf677e13c", + "sha256:9fc17fdac8f1973850d42e51e8ba6149d93b1993ed6768a24f352f926dd3d587", + "sha256:a2787319dc69854acdfd6452e6a8ba8f929aeb20843c7f090e04159fc18e6245", + "sha256:b7c522292407fa04d8195032493aac937e253ad9ae524aab43b9d9d242571f03", + "sha256:bd312794f51e37dcf77f013d40650fe4fbb211dd55ef2863839c37480bd44369", + "sha256:c0d660a186e36c526366edf8a64391874fe53cf8b7039224137aee0163c046df", + "sha256:c4869141e20769b65d2d72686e7a7eb141ce9f3168106bed3e7dcced54eb2422", + "sha256:cc4057f692ac35bbe82a0a908d42ce3a281c9e913290fac37d7fa3bd01307dfb", + "sha256:cccf1e7806f12300e3a3b48f219e111000c2538483e85c869c35c1ae591e6ce9", + "sha256:ce208f80f398522e49d9db789065c8ad2cd37b21bd6b23d30053474b7416af11", + "sha256:d0565481dc196986c484a7fb13214fc6402201f7fb55c65fd215b3324962fe6c", + "sha256:d1b3366329c45a474b3bbc9b9c95d4c686e03f35da7fd12bc144626d1f2a7c04", + "sha256:d226e0d4b9192d95079a9a29c04dd81816b1ce8903b8c174a39224fe978547cb", + "sha256:d38b35f6eef4237b1d0d8e845fc1546dad85c55eba447e28c211da8c7ef9697c", + "sha256:d64c98277ea80e4484f1332ab107e8dfd173a7dcf1bdbf10a9cccc97aaab145f", + "sha256:d9de8427a5601799784eb0e7fa1b031aa64086ce04de29df775a8ca37eedac41", + "sha256:e6a15cf8f887d9f578dd49c6fb3a99d53e1d922fdd67a245a67488d77bf56eb2", + "sha256:e8c446882cbb3774cd78c738c9f58220606b702b7c1655f1423357dc51674054", + "sha256:e8d188ee39bd0ffe76603da887706e4e7b471f613625899ddf1e27867dc6a0d3", + "sha256:ef76535776c0708a85258f6dc51d36a2df12633c735f6d197ed7dfcaa7449b99", + "sha256:f6efca006a81e1197b925a7d7b16b8f61980697bb6746587aad8842865233218" ], "index": "pypi", - "version": "==3.10.1" + "version": "==3.11.0" }, "pyrsistent": { "hashes": [ @@ -429,11 +440,11 @@ }, "requests": { "hashes": [ - "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", - "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" + "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", + "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" ], "index": "pypi", - "version": "==2.23.0" + "version": "==2.24.0" }, "rethinkdb": { "hashes": [ @@ -458,18 +469,17 @@ }, "swagger-ui-bundle": { "hashes": [ - "sha256:49d2e12d60a6499e9d37ea37953b5d700f4e114edc7520fe918bae5eb693a20e", - "sha256:c5373b683487b1b914dccd23bcd9a3016afa2c2d1cda10f8713c0a9af0f91dd3", - "sha256:f776811855092c086dbb08216c8810a84accef8c76c796a135caa13645c5cc68" + "sha256:f5255f786cde67a2638111f4a7d04355836743198a83c4ecbe815d9fc384b0c8", + "sha256:f5691167f2e9f73ecbe8229a89454ae5ea958f90bb0d4583ed7adaae598c4122" ], - "version": "==0.0.6" + "version": "==0.0.8" }, "urllib3": { "hashes": [ - "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527", - "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115" + "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", + "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" ], - "version": "==1.25.9" + "version": "==1.25.10" }, "werkzeug": { "hashes": [ diff --git a/src/app.py b/src/app.py index 7d88971f..9266f430 100644 --- a/src/app.py +++ b/src/app.py @@ -1,12 +1,15 @@ -from utils.hosts import LOCAL +import server.wsgi as flask +import server.grunner as gunicorn +from utils.cfgreader import EnvReader, BoolVar + + +def use_flask() -> bool: + env_var = BoolVar('USE_FLASK', False) + return EnvReader().safe_read(env_var) if __name__ == '__main__': - import connexion - app = connexion.FlaskApp(__name__, specification_dir='../specification/') - app.add_api('quantumleap.yml', - arguments={'title': 'QuantumLeap V2 API'}, - pythonic_params=True, - # validate_responses=True, strict_validation=True - ) - app.run(host=LOCAL, port=8668) + if use_flask(): # dev mode, run the WSGI app in Flask dev server + flask.run() + else: # prod mode, run the WSGI app in Gunicorn + gunicorn.run() diff --git a/src/geocoding/tests/test_geocoding.py b/src/geocoding/tests/test_geocoding.py index cc356396..b71b2985 100644 --- a/src/geocoding/tests/test_geocoding.py +++ b/src/geocoding/tests/test_geocoding.py @@ -94,6 +94,8 @@ def test_entity_add_street_line(air_quality_observed): assert len(geo['coordinates']) > 1 +# TODO: see #358 +@pytest.mark.skip(reason="see #358") def test_entity_add_city_shape(air_quality_observed): air_quality_observed.pop('location') diff --git a/src/reporter/tests/run_tests.sh b/src/reporter/tests/run_tests.sh index 55286e01..1c7cc246 100644 --- a/src/reporter/tests/run_tests.sh +++ b/src/reporter/tests/run_tests.sh @@ -3,7 +3,7 @@ docker build -t smartsdk/quantumleap ../../../ docker-compose up -d -sleep 12 +sleep 20 cd ../../../ pytest src/reporter/ --cov-report= --cov-config=.coveragerc --cov-append --cov=src/ diff --git a/src/server/__init__.py b/src/server/__init__.py new file mode 100644 index 00000000..470fa983 --- /dev/null +++ b/src/server/__init__.py @@ -0,0 +1,3 @@ + +DEFAULT_HOST = '0.0.0.0' # bind to all available network interfaces +DEFAULT_PORT = 8668 diff --git a/src/server/gconfig.py b/src/server/gconfig.py new file mode 100644 index 00000000..3b55429b --- /dev/null +++ b/src/server/gconfig.py @@ -0,0 +1,74 @@ +# +# Gunicorn settings to run QuantumLeap. +# To make configuration more manageable, we keep all our Gunicorn settings +# in this file and start Gunincorn from the Docker container without any +# command line args except for the app module and the path to this config +# file, e.g. +# +# gunicorn server.wsgi --config server/gconfig.py +# +# Settings spec: +# - https://docs.gunicorn.org/en/stable/settings.html +# + +import multiprocessing + +import server + + +# +# Server config section. +# + +bind = f"{server.DEFAULT_HOST}:{server.DEFAULT_PORT}" + + +# +# Worker processes config section. +# Read: https://docs.gunicorn.org/en/latest/design.html +# + + +# Number of worker processes for handling requests. +# We set it to the max Gunicorn recommends. +workers = multiprocessing.cpu_count() * 4 + 1 + +# QuantumLeap does alot of network IO, so we configure worker processes +# to use multi-threading (`gthread`) to improve performance. With this +# setting, each request gets handled in its own thread taken from a +# thread pool. +# In our tests, the `gthread` worker type had better throughput and +# latency than `gevent` but `gevent` used up less memory, most likely +# because of the difference in actual OS threads. So for now we go with +# `gthread` and a low number of threads. This has the advantage of better +# performance, reasonable memory consumption, and keeps us from accidentally +# falling into the `gevent` monkey patching rabbit hole. Also notice that +# according to Gunicorn docs, when using `gevent`, Psycopg (Timescale +# driver) needs psycogreen properly configured to take full advantage +# of async IO. (Not sure what to do for the Crate driver!) +worker_class = 'gthread' + +# The size of each process's thread pool. +# So here's the surprise. In our tests, w/r/t to throughput `gthread` +# outperformed `gevent`---27% better. Latency was pretty much the same +# though. But the funny thing is that we used exactly the same number +# of worker processes and the default number of threads per process, +# which is, wait for it, 1. Yes, 1. +# +# TODO: proper benchmarking. +# We did some initial quick & dirty benchmarking to get these results. +# We'll likely have to measure better and also understand better the +# way the various Gunicorn worker types actually work. (Pun intended.) +threads = 1 + + +# +# Logging config section. +# + +loglevel = 'debug' + + +# TODO: other settings. +# Review gunicorn default settings with an eye on security and performance. +# We might need to set more options than the above. diff --git a/src/server/grunner.py b/src/server/grunner.py new file mode 100644 index 00000000..06a41808 --- /dev/null +++ b/src/server/grunner.py @@ -0,0 +1,81 @@ +import sys +from typing import Any, Dict + +from gunicorn.app.base import Application +from gunicorn.config import make_settings + +import server.gconfig +from server.wsgi import application + + +def quantumleap_base_config() -> Dict[str, Any]: + """ + Read the base QuantumLeap configuration from the ``server.gconfig`` + module. + + :return: the dictionary with the base QuantumLeap settings. + """ + gunicorn_setting_names = [k for k, v in make_settings().items()] + server_config_vars = vars(server.gconfig).items() + return { + k: v + for k, v in server_config_vars if k in gunicorn_setting_names + } + + +class GuantumLeap(Application): + """ + Gunicorn server runner. + + This class is a fully-fledged Gunicorn server WSGI runner, just like + ``WSGIApplication`` from ``gunicorn.app.wsgiapp``, except the WSGI + app to run is fixed (QuantumLeap) and the content of ``server.gconfig`` + is used as initial configuration. Notice you can override these base + config settings using CLI args or/and a config file as you'd normally + do with Gunicorn, but you can't run any WSGI app other than QuantumLeap. + """ + + def init(self, parser, opts, args): + return quantumleap_base_config() + + def load(self): + return application + + +def run(): + """ + Start a fully-fledged Gunicorn server to run QuantumLeap. + + If you pass no CLI args, this is the same as running + + ``$ gunicorn server.wsgi --config server/gconfig.py`` + + which starts Gunicorn to run the QuantumLeap WSGI Flask app with + the Gunicorn settings in our ``server.gconfig`` module. + You can specify any valid Gunicorn CLI option and it'll take + precedence over any setting with the same name in ``server.gconfig``. + Also you can specify a different config module and options in that + module will override those with the same name in ``server.gconfig``. + The only thing you won't be able to change is the WSGI app to run, + QuantumLeap. + """ + gunicorn = GuantumLeap('%(prog)s [OPTIONS] [APP_MODULE]') # (1) + sys.argv[0] = 'quantumleap' # (2) + sys.exit(gunicorn.run()) # (2) + +# NOTE. +# 1. We keep the same usage as in `gunicorn.app.wsgiapp.run`. +# 2. We basically start Gunicorn in the same way as the Gunicorn launcher +# would: +# +# $ cat $(which gunicorn) +# +# #!/Users/andrea/.local/share/virtualenvs/ngsi-timeseries-api-MeJ80LMF/bin/python +# # -*- coding: utf-8 -*- +# import re +# import sys +# from gunicorn.app.wsgiapp import run +# if __name__ == '__main__': +# sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) +# sys.exit(run()) +# diff --git a/src/server/wsgi.py b/src/server/wsgi.py new file mode 100644 index 00000000..07fd3c41 --- /dev/null +++ b/src/server/wsgi.py @@ -0,0 +1,54 @@ +from connexion import FlaskApp + +import server + + +SPEC_DIR = '../../specification/' +SPEC = 'quantumleap.yml' + + +def new_wrapper() -> FlaskApp: + """ + Factory function to build a Connexion wrapper to manage the Flask + application in which QuantumLeap runs. + + :return: the Connexion wrapper. + """ + wrapper = FlaskApp(__name__, + specification_dir=SPEC_DIR) + wrapper.add_api(SPEC, + arguments={'title': 'QuantumLeap V2 API'}, + pythonic_params=True, + # validate_responses=True, strict_validation=True + ) + return wrapper + + +quantumleap = new_wrapper() +""" +Singleton Connexion wrapper that manages the QuantumLeap Flask app. +""" + +application = quantumleap.app +""" +The WSGI callable to run QuantumLeap in a WSGI container of your choice, +e.g. Gunicorn, uWSGI. +Notice that Gunicorn will look for a WSGI callable named `application` if +no variable name follows the module name given on the command line. So one +way to run QuantumLeap in Gunicorn would be + + gunicorn server.wsgi --config server/gconfig.py + +An even more convenient way is to use our Gunicorn standalone server, +see `server.grunner` module. +""" + + +def run(): + """ + Runs the bare-bones QuantumLeap WSGI app. + Notice the app will run in the Flask dev server, so it's only good + for development, not prod! + """ + quantumleap.run(host=server.DEFAULT_HOST, + port=server.DEFAULT_PORT) diff --git a/src/tests/docker-compose.yml b/src/tests/docker-compose.yml index 527f5e4a..c41f178d 100644 --- a/src/tests/docker-compose.yml +++ b/src/tests/docker-compose.yml @@ -21,7 +21,9 @@ services: - "27017:27017" quantumleap: - image: ${QL_IMAGE:-smartsdk/quantumleap} + image: ${QL_IMAGE:-smartsdk/quantumleap:latest} + sysctls: + net.core.somaxconn: 4096 ports: - "8668:8668" depends_on: @@ -33,7 +35,7 @@ services: - USE_GEOCODING=True - REDIS_HOST=redis - REDIS_PORT=6379 - - LOGLEVEL=DEBUG + - LOGLEVEL=ERROR crate: image: crate:${CRATE_VERSION:-4.1.4} diff --git a/src/tests/notify-load-test.js b/src/tests/notify-load-test.js new file mode 100644 index 00000000..3cae34c4 --- /dev/null +++ b/src/tests/notify-load-test.js @@ -0,0 +1,49 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; + +export default function() { + var url = 'http://192.0.0.1:8668/v2/notify'; + const before = new Date().getTime(); + const T = 30; // time needed to complete a VU iteration + + + for (var i = 0; i < 100; i++){ + var data = { + "id": "Room:1", + "type": "Room", + "temperature": { + "value": 23, + "type": "Float" + }, + "pressure": { + "value": 720, + "type": "Integer" + } + } + var array = []; + array.push(data); + + var payload = { + "data" : array + } + var payload = JSON.stringify(payload); + + var params = { + headers: { + 'Content-Type': 'application/json', + } + }; + let res = http.post(url, payload, params); + check(res, { 'status was 200': r => r.status == 200 }); + } + const after = new Date().getTime(); + const diff = (after - before) / 1000; + const remainder = T - diff; + if (remainder > 0) { + sleep(remainder); + } else { + console.warn( + `Timer exhausted! The execution time of the test took longer than ${T} seconds` + ); + } +} diff --git a/src/tests/run_load_tests.sh b/src/tests/run_load_tests.sh new file mode 100644 index 00000000..530300c0 --- /dev/null +++ b/src/tests/run_load_tests.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +docker build --cache-from smartsdk/quantumleap -t smartsdk/quantumleap ../../ + +docker-compose up -d +docker-compose stop orion +docker-compose stop mongo +sleep 10 + +docker run -i --rm loadimpact/k6 run --vus 10 --duration 60s - < notify-load-test.js + +sleep 10 + +docker run -i --rm loadimpact/k6 run --vus 100 --duration 120s - < notify-load-test.js + +sleep 10 + +docker-compose down diff --git a/src/tests/run_tests.sh b/src/tests/run_tests.sh index 54dea935..210097d1 100644 --- a/src/tests/run_tests.sh +++ b/src/tests/run_tests.sh @@ -24,7 +24,7 @@ docker run -ti --rm --network tests_default \ # Restart QL on development version and CRATE on current version docker-compose stop quantumleap CRATE_VERSION=${CRATE_VERSION} QL_IMAGE=smartsdk/quantumleap docker-compose up -d -sleep 30 +sleep 40 # Backwards Compatibility Test cd ../../ diff --git a/src/utils/hosts.py b/src/utils/hosts.py deleted file mode 100644 index 9a08be55..00000000 --- a/src/utils/hosts.py +++ /dev/null @@ -1 +0,0 @@ -LOCAL = "0.0.0.0"