diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a64d3d5..9a85fbcf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,4 +4,7 @@ repos: hooks: - id: black language_version: python3.7 - +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'master' + hooks: + - id: mypy diff --git a/examples/custom_fdw/README.md b/examples/custom_fdw/README.md index 2ff570b1..ea218a81 100644 --- a/examples/custom_fdw/README.md +++ b/examples/custom_fdw/README.md @@ -77,28 +77,29 @@ Mounting jobstories... You can now run ordinary SQL queries against these tables: ``` -$ sgr sql -s hackernews "SELECT id, title, url, score FROM showstories LIMIT 10" - -23573474 Show HN: OBS-web, control OBS from the browser https://github.com/Niek/obs-web 49 -23574723 Launch HN: Mighty Health (YC S19) – Health coaching for people over 50 17 -23575326 Show HN: Jargon-free financial advice via text 3 -23574926 Show HN: Cedreo easy Floor Plan Software. Instant 3D floor plans from 2D drawing https://cedreo.com/floor-plan-software/ 2 -23574842 Show HN: A QEMU port for iOS that you can sideload https://getutm.app/ 2 -23569912 Show HN: Noodel.js – User interface for beautiful, dynamic content trees https://github.com/zlu883/Noodel 17 -23562165 Show HN: Poica – Algebraic data types and type introspection for pure C https://github.com/Hirrolot/poica 101 -23560046 Show HN: My Book on Evolutionary Algorithms, Written in Python Notebooks https://shahinrostami.com/posts/search-and-optimisation/practical-evolutionary-algorithms/preface/ 59 -23572342 Show HN: Portable Serverless without the complexity of Kubernetes https://www.openfaas.com/blog/introducing-faasd/ 6 -23561904 Show HN: Flatend – Quickly build microservices using P2P networking in Node/Go https://github.com/lithdew/flatend 134 +$ sgr sql -s hackernews "SELECT id, title, url, score FROM topstories LIMIT 10" + +23648942 Amazon to pay $1B+ for Zoox https://www.axios.com/report-amazon-to-pay-1-billion-for-self-driving-tech-firm-zoox-719d293b-3799-4315-a573-a226a58bb004.html 55 +23646158 When you type realty.com into Safari it takes you to realtor.com https://www.facebook.com/story.php?story_fbid=10157161487396994&id=501751993 653 +23648864 Turn recipe websites into plain text https://plainoldrecipe.com/ 30 +23644253 Olympus quits camera business after 84 years https://www.bbc.com/news/technology-53165293 548 +23648217 Boston bans use of facial recognition technology https://www.wbur.org/news/2020/06/23/boston-facial-recognition-ban 51 +23646953 Curl Wttr.in https://github.com/chubin/wttr.in 190 +23646164 Quora goes permanently remote-first https://twitter.com/adamdangelo/status/1276210618786168833 267 +23646395 Dwarf Fortress Creator Explains Its Complexity and Origins [video] https://www.youtube.com/watch?v=VAhHkJQ3KgY 152 +23645305 Blackballed by PayPal, Sci-Hub switches to Bitcoin https://www.coindesk.com/blackballed-by-paypal-scientific-paper-pirate-takes-bitcoin-donations 479 +23646028 The Acorn Archimedes was the first desktop to use the ARM architecture https://spectrum.ieee.org/tech-talk/consumer-electronics/gadgets/why-wait-for-apple-try-out-the-original-arm-desktop-experience-today-with-a-raspberry-pi 111 + ``` Since actual query planning and filtering is done by PostgreSQL and the foreign data wrapper only fetches tuples from the API, all PostgreSQL constructs are supported: ``` -$ sgr sql -s hackernews "SELECT id, title, url, score FROM topstories \ - WHERE title LIKE '%Python%' ORDER BY score DESC LIMIT 5" - -23549273 Practical Python Programming https://github.com/dabeaz-course/practical-python 341 -23559680 PyPy: A Faster Python Implementation https://www.pypy.org/index.html 71 -23560046 Show HN: My Book on Evolutionary Algorithms, Written in Python Notebooks https://shahinrostami.com/posts/search-and-optimisation/practical-evolutionary-algorithms/preface/ 59 +$ sgr sql -s hackernews "SELECT id, title, url, score FROM showstories ORDER BY score DESC LIMIT 5" +23643096 Show HN: Aviary.sh – A tiny Bash alternative to Ansible https://github.com/team-video/aviary.sh 235 +23626167 Show HN: HN Deck – An alternative way to browse Hacker News https://hndeck.sagunshrestha.com/ 110 +23640069 Show HN: Sourceful – Crowdsourcing the best public Google docs https://sourceful.co.uk 102 +23627066 Show HN: Splitgraph - Build and share data with Postgres, inspired by Docker/Git http://www.splitgraph.com 79 +23629125 Show HN: Deta – A cloud platform for building and deploying apps https://www.deta.sh/ 78 ``` diff --git a/poetry.lock b/poetry.lock index ffbd7c02..4b131060 100644 --- a/poetry.lock +++ b/poetry.lock @@ -303,7 +303,7 @@ marker = "python_version < \"3.8\"" name = "importlib-metadata" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -version = "1.6.1" +version = "1.7.0" [package.dependencies] zipp = ">=0.5" @@ -414,7 +414,7 @@ description = "Optional static typing for Python" name = "mypy" optional = false python-versions = ">=3.5" -version = "0.781" +version = "0.782" [package.dependencies] mypy-extensions = ">=0.4.3,<0.5.0" @@ -564,7 +564,7 @@ description = "library with cross-python path, ini-parsing, io, code, log facili name = "py" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.8.2" +version = "1.9.0" [[package]] category = "dev" @@ -894,7 +894,7 @@ description = "Database Abstraction Library" name = "sqlalchemy" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -version = "1.3.17" +version = "1.3.18" [package.extras] mssql = ["pyodbc"] @@ -973,7 +973,7 @@ description = "Virtual Python Environment builder" name = "virtualenv" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" -version = "20.0.24" +version = "20.0.25" [package.dependencies] appdirs = ">=1.4.3,<2" @@ -999,7 +999,7 @@ description = "Measures the displayed width of unicode strings in a terminal" name = "wcwidth" optional = false python-versions = "*" -version = "0.2.4" +version = "0.2.5" [[package]] category = "main" @@ -1184,8 +1184,8 @@ imagesize = [ {file = "imagesize-1.2.0.tar.gz", hash = "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"}, ] importlib-metadata = [ - {file = "importlib_metadata-1.6.1-py2.py3-none-any.whl", hash = "sha256:15ec6c0fd909e893e3a08b3a7c76ecb149122fb14b7efe1199ddd4c7c57ea958"}, - {file = "importlib_metadata-1.6.1.tar.gz", hash = "sha256:0505dd08068cfec00f53a74a0ad927676d7757da81b7436a6eefe4c7cf75c545"}, + {file = "importlib_metadata-1.7.0-py2.py3-none-any.whl", hash = "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070"}, + {file = "importlib_metadata-1.7.0.tar.gz", hash = "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83"}, ] importlib-resources = [ {file = "importlib_resources-2.0.1-py2.py3-none-any.whl", hash = "sha256:83985739b3a6679702f9ab33f0ad016ad564664d0568a31ac14d7c64789453e6"}, @@ -1271,20 +1271,20 @@ more-itertools = [ {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"}, ] mypy = [ - {file = "mypy-0.781-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:43335f3ff3288eb877de6d186ec08e10ea61093405189678756e14c7ccee4a9b"}, - {file = "mypy-0.781-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:d7c9255ba6626e1745bd68e1b85e3d7888844eaf38252fefb8157194e55fd3e9"}, - {file = "mypy-0.781-cp35-cp35m-win_amd64.whl", hash = "sha256:1fe322a51df7ec60e4060c358422732359dda4467c1bd240f2172433b26b39be"}, - {file = "mypy-0.781-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:e60674723cad7b7c7fc4e9075f7a9d5d927d23d290e30cf86aeb987ef135ca1d"}, - {file = "mypy-0.781-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:e2f193a2076e4508a88c93c25348a1cea5e6b717ee50b4422a001e9ac819c3d5"}, - {file = "mypy-0.781-cp36-cp36m-win_amd64.whl", hash = "sha256:845abd8a9537da01e6b96f091e2d6d4ece18e52339f81f9fbbac1b6db2aa8d2d"}, - {file = "mypy-0.781-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:2bc1fe8f793f1b1809afd6f57c2d9b948e1bf525a78e4c2957392a383c86a4a4"}, - {file = "mypy-0.781-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ca56382e6ebdeb3cb928ae4ce87031cc752a1eca40ff86abc14dc712def41798"}, - {file = "mypy-0.781-cp37-cp37m-win_amd64.whl", hash = "sha256:738a8df84da4e9ce1f59cbdebc7d1d03310149405df9a95308d4f2a62a6d7c13"}, - {file = "mypy-0.781-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b0c6ae0cb991a7a11013d0cc7edf8343184465b6e2dcbb9e44265c7bb3fde3af"}, - {file = "mypy-0.781-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:ec45f2a5935b291d86974a24e09676e467ac108d0c7ce94de44d7650c43a5805"}, - {file = "mypy-0.781-cp38-cp38-win_amd64.whl", hash = "sha256:d6e9611ae026be70604672cd71bee468cb231078d8d9b3d6b43f8dcbd4d9b776"}, - {file = "mypy-0.781-py3-none-any.whl", hash = "sha256:5c75603ea44bffad0356df333bd1b411facad321e0692d6b0687d65d94fcd8e1"}, - {file = "mypy-0.781.tar.gz", hash = "sha256:94bb664868b5cf4ca1147d875a4c77883d8c605cf2e916853006e4c6194f1e84"}, + {file = "mypy-0.782-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:2c6cde8aa3426c1682d35190b59b71f661237d74b053822ea3d748e2c9578a7c"}, + {file = "mypy-0.782-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9c7a9a7ceb2871ba4bac1cf7217a7dd9ccd44c27c2950edbc6dc08530f32ad4e"}, + {file = "mypy-0.782-cp35-cp35m-win_amd64.whl", hash = "sha256:c05b9e4fb1d8a41d41dec8786c94f3b95d3c5f528298d769eb8e73d293abc48d"}, + {file = "mypy-0.782-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:6731603dfe0ce4352c555c6284c6db0dc935b685e9ce2e4cf220abe1e14386fd"}, + {file = "mypy-0.782-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:f05644db6779387ccdb468cc47a44b4356fc2ffa9287135d05b70a98dc83b89a"}, + {file = "mypy-0.782-cp36-cp36m-win_amd64.whl", hash = "sha256:b7fbfabdbcc78c4f6fc4712544b9b0d6bf171069c6e0e3cb82440dd10ced3406"}, + {file = "mypy-0.782-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:3fdda71c067d3ddfb21da4b80e2686b71e9e5c72cca65fa216d207a358827f86"}, + {file = "mypy-0.782-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d7df6eddb6054d21ca4d3c6249cae5578cb4602951fd2b6ee2f5510ffb098707"}, + {file = "mypy-0.782-cp37-cp37m-win_amd64.whl", hash = "sha256:a4a2cbcfc4cbf45cd126f531dedda8485671545b43107ded25ce952aac6fb308"}, + {file = "mypy-0.782-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6bb93479caa6619d21d6e7160c552c1193f6952f0668cdda2f851156e85186fc"}, + {file = "mypy-0.782-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:81c7908b94239c4010e16642c9102bfc958ab14e36048fa77d0be3289dda76ea"}, + {file = "mypy-0.782-cp38-cp38-win_amd64.whl", hash = "sha256:5dd13ff1f2a97f94540fd37a49e5d255950ebcdf446fb597463a40d0df3fac8b"}, + {file = "mypy-0.782-py3-none-any.whl", hash = "sha256:e0b61738ab504e656d1fe4ff0c0601387a5489ca122d55390ade31f9ca0e252d"}, + {file = "mypy-0.782.tar.gz", hash = "sha256:eff7d4a85e9eea55afa34888dfeaccde99e7520b51f867ac28a48492c0b1130c"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, @@ -1413,8 +1413,8 @@ psycopg2-binary = [ {file = "psycopg2_binary-2.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:fa466306fcf6b39b8a61d003123d442b23707d635a5cb05ac4e1b62cc79105cd"}, ] py = [ - {file = "py-1.8.2-py2.py3-none-any.whl", hash = "sha256:a673fa23d7000440cc885c17dbd34fafcb7d7a6e230b29f6766400de36a33c44"}, - {file = "py-1.8.2.tar.gz", hash = "sha256:f3b3a4c36512a4c4f024041ab51866f11761cc169670204b235f6b20523d4e6b"}, + {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, + {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, ] pyfakefs = [ {file = "pyfakefs-4.0.2-py3-none-any.whl", hash = "sha256:42cf165adc821fc9e205d3fc14033d45e0b8224e1d2fea4f67b487c6b7b3230e"}, @@ -1557,34 +1557,34 @@ sphinxcontrib-serializinghtml = [ {file = "sphinxcontrib_serializinghtml-1.1.4-py2.py3-none-any.whl", hash = "sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a"}, ] sqlalchemy = [ - {file = "SQLAlchemy-1.3.17-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:fe01bac7226499aedf472c62fa3b85b2c619365f3f14dd222ffe4f3aa91e5f98"}, - {file = "SQLAlchemy-1.3.17-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:b50f45d0e82b4562f59f0e0ca511f65e412f2a97d790eea5f60e34e5f1aabc9a"}, - {file = "SQLAlchemy-1.3.17-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:ce2646e4c0807f3461be0653502bb48c6e91a5171d6e450367082c79e12868bf"}, - {file = "SQLAlchemy-1.3.17-cp27-cp27m-win32.whl", hash = "sha256:e4e2664232005bd306f878b0f167a31f944a07c4de0152c444f8c61bbe3cfb38"}, - {file = "SQLAlchemy-1.3.17-cp27-cp27m-win_amd64.whl", hash = "sha256:925b4fe5e7c03ed76912b75a9a41dfd682d59c0be43bce88d3b27f7f5ba028fb"}, - {file = "SQLAlchemy-1.3.17-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:869bbb637de58ab0a912b7f20e9192132f9fbc47fc6b5111cd1e0f6cdf5cf9b0"}, - {file = "SQLAlchemy-1.3.17-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:703c002277f0fbc3c04d0ae4989a174753a7554b2963c584ce2ec0cddcf2bc53"}, - {file = "SQLAlchemy-1.3.17-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:eb4fcf7105bf071c71068c6eee47499ab8d4b8f5a11fc35147c934f0faa60f23"}, - {file = "SQLAlchemy-1.3.17-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:8d01e949a5d22e5c4800d59b50617c56125fc187fbeb8fa423e99858546de616"}, - {file = "SQLAlchemy-1.3.17-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:a9e75e49a0f1583eee0ce93270232b8e7bb4b1edc89cc70b07600d525aef4f43"}, - {file = "SQLAlchemy-1.3.17-cp35-cp35m-win32.whl", hash = "sha256:a87d496884f40c94c85a647c385f4fd5887941d2609f71043e2b73f2436d9c65"}, - {file = "SQLAlchemy-1.3.17-cp35-cp35m-win_amd64.whl", hash = "sha256:6cd157ce74a911325e164441ff2d9b4e244659a25b3146310518d83202f15f7a"}, - {file = "SQLAlchemy-1.3.17-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:27e2efc8f77661c9af2681755974205e7462f1ae126f498f4fe12a8b24761d15"}, - {file = "SQLAlchemy-1.3.17-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:31c043d5211aa0e0773821fcc318eb5cbe2ec916dfbc4c6eea0c5188971988eb"}, - {file = "SQLAlchemy-1.3.17-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:a9030cd30caf848a13a192c5e45367e3c6f363726569a56e75dc1151ee26d859"}, - {file = "SQLAlchemy-1.3.17-cp36-cp36m-win32.whl", hash = "sha256:f502ef245c492b391e0e23e94cba030ab91722dcc56963c85bfd7f3441ea2bbe"}, - {file = "SQLAlchemy-1.3.17-cp36-cp36m-win_amd64.whl", hash = "sha256:128bc917ed20d78143a45024455ff0aed7d3b96772eba13d5dbaf9cc57e5c41b"}, - {file = "SQLAlchemy-1.3.17-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:2a12f8be25b9ea3d1d5b165202181f2b7da4b3395289000284e5bb86154ce87c"}, - {file = "SQLAlchemy-1.3.17-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e8aa395482728de8bdcca9cc0faf3765ab483e81e01923aaa736b42f0294f570"}, - {file = "SQLAlchemy-1.3.17-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f35248f7e0d63b234a109dd72fbfb4b5cb6cb6840b221d0df0ecbf54ab087654"}, - {file = "SQLAlchemy-1.3.17-cp37-cp37m-win32.whl", hash = "sha256:ce1ddaadee913543ff0154021d31b134551f63428065168e756d90bdc4c686f5"}, - {file = "SQLAlchemy-1.3.17-cp37-cp37m-win_amd64.whl", hash = "sha256:9cb1819008f0225a7c066cac8bb0cf90847b2c4a6eb9ebb7431dbd00c56c06c5"}, - {file = "SQLAlchemy-1.3.17-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:65eb3b03229f684af0cf0ad3bcc771970c1260a82a791a8d07bffb63d8c95bcc"}, - {file = "SQLAlchemy-1.3.17-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8a0e0cd21da047ea10267c37caf12add400a92f0620c8bc09e4a6531a765d6d7"}, - {file = "SQLAlchemy-1.3.17-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b7878e59ec31f12d54b3797689402ee3b5cfcb5598f2ebf26491732758751908"}, - {file = "SQLAlchemy-1.3.17-cp38-cp38-win32.whl", hash = "sha256:ce6c3d18b2a8ce364013d47b9cad71db815df31d55918403f8db7d890c9d07ae"}, - {file = "SQLAlchemy-1.3.17-cp38-cp38-win_amd64.whl", hash = "sha256:ed375a79f06cad285166e5be74745df1ed6845c5624aafadec4b7a29c25866ef"}, - {file = "SQLAlchemy-1.3.17.tar.gz", hash = "sha256:156a27548ba4e1fed944ff9fcdc150633e61d350d673ae7baaf6c25c04ac1f71"}, + {file = "SQLAlchemy-1.3.18-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:f11c2437fb5f812d020932119ba02d9e2bc29a6eca01a055233a8b449e3e1e7d"}, + {file = "SQLAlchemy-1.3.18-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:0ec575db1b54909750332c2e335c2bb11257883914a03bc5a3306a4488ecc772"}, + {file = "SQLAlchemy-1.3.18-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:f57be5673e12763dd400fea568608700a63ce1c6bd5bdbc3cc3a2c5fdb045274"}, + {file = "SQLAlchemy-1.3.18-cp27-cp27m-win32.whl", hash = "sha256:8cac7bb373a5f1423e28de3fd5fc8063b9c8ffe8957dc1b1a59cb90453db6da1"}, + {file = "SQLAlchemy-1.3.18-cp27-cp27m-win_amd64.whl", hash = "sha256:adad60eea2c4c2a1875eb6305a0b6e61a83163f8e233586a4d6a55221ef984fe"}, + {file = "SQLAlchemy-1.3.18-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:57aa843b783179ab72e863512e14bdcba186641daf69e4e3a5761d705dcc35b1"}, + {file = "SQLAlchemy-1.3.18-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:621f58cd921cd71ba6215c42954ffaa8a918eecd8c535d97befa1a8acad986dd"}, + {file = "SQLAlchemy-1.3.18-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:fc728ece3d5c772c196fd338a99798e7efac7a04f9cb6416299a3638ee9a94cd"}, + {file = "SQLAlchemy-1.3.18-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:736d41cfebedecc6f159fc4ac0769dc89528a989471dc1d378ba07d29a60ba1c"}, + {file = "SQLAlchemy-1.3.18-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:427273b08efc16a85aa2b39892817e78e3ed074fcb89b2a51c4979bae7e7ba98"}, + {file = "SQLAlchemy-1.3.18-cp35-cp35m-win32.whl", hash = "sha256:cbe1324ef52ff26ccde2cb84b8593c8bf930069dfc06c1e616f1bfd4e47f48a3"}, + {file = "SQLAlchemy-1.3.18-cp35-cp35m-win_amd64.whl", hash = "sha256:8fd452dc3d49b3cc54483e033de6c006c304432e6f84b74d7b2c68afa2569ae5"}, + {file = "SQLAlchemy-1.3.18-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:e89e0d9e106f8a9180a4ca92a6adde60c58b1b0299e1b43bd5e0312f535fbf33"}, + {file = "SQLAlchemy-1.3.18-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6ac2558631a81b85e7fb7a44e5035347938b0a73f5fdc27a8566777d0792a6a4"}, + {file = "SQLAlchemy-1.3.18-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:87fad64529cde4f1914a5b9c383628e1a8f9e3930304c09cf22c2ae118a1280e"}, + {file = "SQLAlchemy-1.3.18-cp36-cp36m-win32.whl", hash = "sha256:e4624d7edb2576cd72bb83636cd71c8ce544d8e272f308bd80885056972ca299"}, + {file = "SQLAlchemy-1.3.18-cp36-cp36m-win_amd64.whl", hash = "sha256:89494df7f93b1836cae210c42864b292f9b31eeabca4810193761990dc689cce"}, + {file = "SQLAlchemy-1.3.18-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:716754d0b5490bdcf68e1e4925edc02ac07209883314ad01a137642ddb2056f1"}, + {file = "SQLAlchemy-1.3.18-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:50c4ee32f0e1581828843267d8de35c3298e86ceecd5e9017dc45788be70a864"}, + {file = "SQLAlchemy-1.3.18-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:d98bc827a1293ae767c8f2f18be3bb5151fd37ddcd7da2a5f9581baeeb7a3fa1"}, + {file = "SQLAlchemy-1.3.18-cp37-cp37m-win32.whl", hash = "sha256:0942a3a0df3f6131580eddd26d99071b48cfe5aaf3eab2783076fbc5a1c1882e"}, + {file = "SQLAlchemy-1.3.18-cp37-cp37m-win_amd64.whl", hash = "sha256:16593fd748944726540cd20f7e83afec816c2ac96b082e26ae226e8f7e9688cf"}, + {file = "SQLAlchemy-1.3.18-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:c26f95e7609b821b5f08a72dab929baa0d685406b953efd7c89423a511d5c413"}, + {file = "SQLAlchemy-1.3.18-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:512a85c3c8c3995cc91af3e90f38f460da5d3cade8dc3a229c8e0879037547c9"}, + {file = "SQLAlchemy-1.3.18-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:d05c4adae06bd0c7f696ae3ec8d993ed8ffcc4e11a76b1b35a5af8a099bd2284"}, + {file = "SQLAlchemy-1.3.18-cp38-cp38-win32.whl", hash = "sha256:109581ccc8915001e8037b73c29590e78ce74be49ca0a3630a23831f9e3ed6c7"}, + {file = "SQLAlchemy-1.3.18-cp38-cp38-win_amd64.whl", hash = "sha256:8619b86cb68b185a778635be5b3e6018623c0761dde4df2f112896424aa27bd8"}, + {file = "SQLAlchemy-1.3.18.tar.gz", hash = "sha256:da2fb75f64792c1fc64c82313a00c728a7c301efe6a60b7a9fe35b16b4368ce7"}, ] tabulate = [ {file = "tabulate-0.8.7-py3-none-any.whl", hash = "sha256:ac64cb76d53b1231d364babcd72abbb16855adac7de6665122f97b593f1eb2ba"}, @@ -1631,12 +1631,12 @@ urllib3 = [ {file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"}, ] virtualenv = [ - {file = "virtualenv-20.0.24-py2.py3-none-any.whl", hash = "sha256:1b253c5d0e76afe24c76ce288cdd5dd01ac7deaa55f644998c74e5a799349522"}, - {file = "virtualenv-20.0.24.tar.gz", hash = "sha256:680011aa2995fb8b7c2bbea7da7afd8dcaf382def7a3e02a1b1fba4b402aa8cf"}, + {file = "virtualenv-20.0.25-py2.py3-none-any.whl", hash = "sha256:ffffcb3c78a671bb3d590ac3bc67c081ea2188befeeb058870cba13e7f82911b"}, + {file = "virtualenv-20.0.25.tar.gz", hash = "sha256:f332ba0b2dfbac9f6b1da9f11224f0036b05cdb4df23b228527c2a2d5504aeed"}, ] wcwidth = [ - {file = "wcwidth-0.2.4-py2.py3-none-any.whl", hash = "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f"}, - {file = "wcwidth-0.2.4.tar.gz", hash = "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f"}, + {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, + {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] websocket-client = [ {file = "websocket_client-0.57.0-py2.py3-none-any.whl", hash = "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549"}, diff --git a/splitgraph/core/table.py b/splitgraph/core/table.py index 3fab045d..fa8da9dc 100644 --- a/splitgraph/core/table.py +++ b/splitgraph/core/table.py @@ -22,13 +22,13 @@ from splitgraph.config import SPLITGRAPH_META_SCHEMA, SPLITGRAPH_API_SCHEMA, SG_CMD_ASCII from splitgraph.core.common import Tracer -from splitgraph.core.output import pluralise, truncate_list from splitgraph.core.fragment_manager import ( get_temporary_table_id, get_chunk_groups, ExtraIndexInfo, ) from splitgraph.core.indexing.range import quals_to_sql +from splitgraph.core.output import pluralise, truncate_list from splitgraph.core.sql import select from splitgraph.core.types import TableSchema, Quals from splitgraph.engine import ResultShape @@ -107,13 +107,20 @@ def create_foreign_table( table_name: str, schema_spec: TableSchema, internal_table_name: Optional[str] = None, + extra_options: Optional[Dict[str, str]] = None, ): + table_options = extra_options or {} + table_options.update({"table": internal_table_name or table_name}) + + table_opts, table_optvals = zip(*table_options.items()) + query = SQL("CREATE FOREIGN TABLE {}.{} (").format(Identifier(schema), Identifier(table_name)) query += SQL(",".join("{} %s " % col.pg_type for col in schema_spec)).format( *(Identifier(col.name) for col in schema_spec) ) - query += SQL(") SERVER {} OPTIONS (table %s);").format(Identifier(server)) - args = [internal_table_name or table_name] + query += SQL(") SERVER {} OPTIONS (").format(Identifier(server)) + query += SQL(",").join(Identifier(o) + SQL(" %s") for o in table_opts) + SQL(");") + args = list(table_optvals) for col in schema_spec: if col.comment: query += SQL("COMMENT ON COLUMN {}.{}.{} IS %s;").format( diff --git a/splitgraph/ingestion/socrata/fdw.py b/splitgraph/ingestion/socrata/fdw.py index c56bf0e8..f5db878f 100644 --- a/splitgraph/ingestion/socrata/fdw.py +++ b/splitgraph/ingestion/socrata/fdw.py @@ -21,10 +21,10 @@ _PG_LOGLEVEL = logging.INFO -def to_json(row, columns): +def to_json(row, columns, column_map): result = {} for col in columns: - val = row.get(col) + val = row.get(column_map.get(col, col)) if isinstance(val, (dict, list)): val = json.dumps(val) result[col] = val @@ -63,7 +63,7 @@ def get_rel_size(self, quals, columns): A tuple of the form (expected_number_of_rows, avg_row_width (in bytes)) """ try: - return estimate_socrata_rows_width(columns, self.table_meta) + return estimate_socrata_rows_width(columns, self.table_meta, self.column_map) except Exception: logging.exception("Failed planning Socrata query, returning dummy values") return 1000000, len(columns) * 10 @@ -73,9 +73,9 @@ def explain(self, quals, columns, sortkeys=None, verbose=False): def execute(self, quals, columns, sortkeys=None): """Main Multicorn entry point.""" - query = quals_to_socrata(quals) - select = cols_to_socrata(columns) - order = sortkeys_to_socrata(sortkeys) + query = quals_to_socrata(quals, self.column_map) + select = cols_to_socrata(columns, self.column_map) + order = sortkeys_to_socrata(sortkeys, self.column_map) logging.debug("Socrata query: %r, select: %r, order: %r", query, select, order) @@ -89,7 +89,7 @@ def execute(self, quals, columns, sortkeys=None): ) for r in result: - r = to_json(r, columns) + r = to_json(r, columns, self.column_map) yield r @property @@ -123,6 +123,10 @@ def __init__(self, fdw_options, fdw_columns): self.fdw_columns = fdw_columns self.table = self.fdw_options["table"] + + # Mappings from SG to Socrata columns (for query building) + self.column_map = json.loads(self.fdw_options.get("column_map") or "{}") + self.app_token = self.fdw_options.get("app_token") self.domain = self.fdw_options["domain"] self.batch_size = int(self.fdw_options.get("batch_size", 1000)) diff --git a/splitgraph/ingestion/socrata/mount.py b/splitgraph/ingestion/socrata/mount.py index 2104c41b..6521ad38 100644 --- a/splitgraph/ingestion/socrata/mount.py +++ b/splitgraph/ingestion/socrata/mount.py @@ -1,4 +1,5 @@ """Splitgraph mount handler for Socrata datasets""" +import json import logging from typing import Optional, Dict, Any @@ -107,13 +108,14 @@ def generate_socrata_mount_queries(sought_ids, datasets, mountpoint, server_id, table_name = tables_inv.get(socrata_id) or slugify( dataset["resource"]["name"] ) + "_" + socrata_id.replace("-", "_") - schema_spec = socrata_to_sg_schema(dataset) + schema_spec, column_map = socrata_to_sg_schema(dataset) sql, args = create_foreign_table( schema=mountpoint, server=server_id, table_name=table_name, schema_spec=schema_spec, internal_table_name=socrata_id, + extra_options={"column_map": json.dumps(column_map)}, ) description = dataset["resource"].get("description") diff --git a/splitgraph/ingestion/socrata/querying.py b/splitgraph/ingestion/socrata/querying.py index b8041e82..1f346aee 100644 --- a/splitgraph/ingestion/socrata/querying.py +++ b/splitgraph/ingestion/socrata/querying.py @@ -1,5 +1,6 @@ -from typing import Dict, Any, List, Tuple +from typing import Dict, Any, List, Tuple, Optional +from splitgraph.core.sql import POSTGRES_MAX_IDENTIFIER from splitgraph.core.types import TableSchema, TableColumn try: @@ -72,11 +73,19 @@ def dedupe_sg_schema(schema_spec: TableSchema, prefix_len: int = 59) -> TableSch ) ) else: - result.append(column) + result.append( + TableColumn( + column.ordinal, + column.name[:POSTGRES_MAX_IDENTIFIER], + column.pg_type, + column.is_pk, + column.comment, + ) + ) return result -def socrata_to_sg_schema(metadata: Dict[str, Any]) -> TableSchema: +def socrata_to_sg_schema(metadata: Dict[str, Any]) -> Tuple[TableSchema, Dict[str, str]]: try: col_names = metadata["resource"]["columns_field_name"] col_types = metadata["resource"]["columns_datatype"] @@ -95,12 +104,21 @@ def socrata_to_sg_schema(metadata: Dict[str, Any]) -> TableSchema: for i, (n, t, d) in enumerate(zip(col_names, col_types, col_desc)) ] - return dedupe_sg_schema(result) + # Truncate Socrata column names to 63 characters and calculate + # a map of Splitgraph columns to Socrata columns. + result_deduped = dedupe_sg_schema(result) + + sg_to_socrata_cols = { + d.name: r.name for r, d in zip(result, result_deduped) if d.name != r.name + } + + return result_deduped, sg_to_socrata_cols -def estimate_socrata_rows_width(columns, metadata): +def estimate_socrata_rows_width(columns, metadata, column_map=None): """Estimate number of rows required for a query and each row's width from the table metadata.""" + column_map = column_map or {} # We currently don't use qualifiers for this, they get passed to Socrata # directly. Socrata's metadata does contain some statistics for each column @@ -112,13 +130,14 @@ def estimate_socrata_rows_width(columns, metadata): # Socrata doesn't return the Socrata Row ID width in its metadata but we know # how big it usually is (row-XXXX.XXXX_XXXX). - row_width = sum(column_widths[c] if c != ":id" else 18 for c in columns) + row_width = sum(column_widths[column_map.get(c, c)] if c != ":id" else 18 for c in columns) return cardinality, row_width -def _emit_col(col): - return f"`{col}`" +def _emit_col(col, column_map: Optional[Dict[str, str]] = None): + column_map = column_map or {} + return f"`{column_map.get(col, col)}`" def _emit_val(val): @@ -138,40 +157,41 @@ def _convert_op(op): return None -def _base_qual_to_socrata(col, op, value): +def _base_qual_to_socrata(col, op, value, column_map=None): soql_op = _convert_op(op) if not soql_op: return "TRUE" else: - return f"{_emit_col(col)} {soql_op} {_emit_val(value)}" + return f"{_emit_col(col, column_map)} {soql_op} {_emit_val(value)}" -def _qual_to_socrata(qual): +def _qual_to_socrata(qual, column_map=None): if qual.is_list_operator: if qual.list_any_or_all == ANY: # Convert col op ANY([a,b,c]) into (cop op a) OR (col op b)... return " OR ".join( - f"({_base_qual_to_socrata(qual.field_name, qual.operator[0], v)})" + f"({_base_qual_to_socrata(qual.field_name, qual.operator[0], v, column_map)})" for v in qual.value ) # Convert col op ALL(ARRAY[a,b,c...]) into (cop op a) AND (col op b)... return " AND ".join( - f"({_base_qual_to_socrata(qual.field_name, qual.operator[0], v)})" for v in qual.value + f"({_base_qual_to_socrata(qual.field_name, qual.operator[0], v, column_map)})" + for v in qual.value ) else: - return f"{_base_qual_to_socrata(qual.field_name, qual.operator[0], qual.value)}" + return f"{_base_qual_to_socrata(qual.field_name, qual.operator[0], qual.value, column_map)}" -def quals_to_socrata(quals): +def quals_to_socrata(quals, column_map: Optional[Dict[str, str]] = None): """Convert a list of Multicorn quals to a SoQL query""" - return " AND ".join(f"({_qual_to_socrata(q)})" for q in quals) + return " AND ".join(f"({_qual_to_socrata(q, column_map)})" for q in quals) -def cols_to_socrata(cols): - return ",".join(f"{_emit_col(c)}" for c in cols) +def cols_to_socrata(cols, column_map: Optional[Dict[str, str]] = None): + return ",".join(f"{_emit_col(c, column_map)}" for c in cols) -def sortkeys_to_socrata(sortkeys): +def sortkeys_to_socrata(sortkeys, column_map: Optional[Dict[str, str]] = None): if not sortkeys: # Always sort on ID for stable paging return ":id" @@ -181,5 +201,5 @@ def sortkeys_to_socrata(sortkeys): if key.nulls_first != key.is_reversed: raise ValueError("Unsupported SortKey %s" % str(key)) order = "DESC" if key.is_reversed else "ASC" - clauses.append(f"{_emit_col(key.attname)} {order}") + clauses.append(f"{_emit_col(key.attname, column_map)} {order}") return ",".join(clauses) diff --git a/test/resources/ingestion/socrata/dataset_metadata.json b/test/resources/ingestion/socrata/dataset_metadata.json index d969c935..04a2bad2 100644 --- a/test/resources/ingestion/socrata/dataset_metadata.json +++ b/test/resources/ingestion/socrata/dataset_metadata.json @@ -57,7 +57,7 @@ "name": "Name", "dataTypeName": "text", "description": "Name of employee", - "fieldName": "name", + "fieldName": "name_long_column_to_test_that_we_map_to_socrata_column_names_correctly", "position": 1, "rdfProperties": "http://xmlns.com/foaf/0.1/name", "renderTypeName": "text", diff --git a/test/resources/ingestion/socrata/find_datasets.json b/test/resources/ingestion/socrata/find_datasets.json index 91488594..1b8096d7 100644 --- a/test/resources/ingestion/socrata/find_datasets.json +++ b/test/resources/ingestion/socrata/find_datasets.json @@ -38,7 +38,7 @@ "job_titles", "typical_hours", "annual_salary", - "name", + "name_long_column_to_test_that_we_map_to_socrata_column_names_correctly", "department" ], "columns_datatype": [ diff --git a/test/splitgraph/ingestion/test_socrata.py b/test/splitgraph/ingestion/test_socrata.py index 4019a138..e756e640 100644 --- a/test/splitgraph/ingestion/test_socrata.py +++ b/test/splitgraph/ingestion/test_socrata.py @@ -39,6 +39,11 @@ class S(NamedTuple): is_reversed: bool = False +_long_name_col = "name_long_column_to_test_that_we_map_to_socrata_column_names_correctly" +_long_name_col_sg = "name_long_column_to_test_that_we_map_to_socrata_column_names_co" +_col_map = {_long_name_col_sg: _long_name_col} + + @pytest.mark.parametrize( "quals,expected", [ @@ -96,10 +101,10 @@ def test_socrata_get_rel_size(): 162, ) assert estimate_socrata_rows_width( - columns=["annual_salary", "name"], metadata=socrata_meta + columns=["annual_salary", _long_name_col_sg], metadata=socrata_meta, column_map=_col_map ) == (33702, 380) assert estimate_socrata_rows_width( - columns=["name", "annual_salary"], metadata=socrata_meta + columns=[_long_name_col_sg, "annual_salary"], metadata=socrata_meta, column_map=_col_map ) == (33702, 380) @@ -152,7 +157,9 @@ def test_socrata_mounting(local_engine_empty): TableColumn( ordinal=7, name="annual_salary", pg_type="numeric", is_pk=False, comment=mock.ANY ), - TableColumn(ordinal=8, name="name", pg_type="text", is_pk=False, comment=mock.ANY), + TableColumn( + ordinal=8, name=_long_name_col_sg, pg_type="text", is_pk=False, comment=mock.ANY + ), TableColumn( ordinal=9, name="department", @@ -162,6 +169,13 @@ def test_socrata_mounting(local_engine_empty): ), ] + assert local_engine_empty.run_sql( + "SELECT option_value FROM information_schema.foreign_table_options " + "WHERE foreign_table_name = 'some_table' " + "AND foreign_table_schema = 'test/pg_mount' " + "AND option_name = 'column_map'" + ) == [(f'{{"{_long_name_col_sg}": "{_long_name_col}"}}',)] + def test_socrata_mounting_error(): socrata = MagicMock(spec=Socrata) @@ -226,8 +240,8 @@ def test_socrata_fdw(): socrata = MagicMock(spec=Socrata) socrata.get_metadata.return_value = socrata_meta socrata.get_all.return_value = [ - {"name": "Test", "job_titles": "Test Title", "annual_salary": 123456.0}, - {"name": "Test2", "job_titles": "Test Title 2", "annual_salary": 789101.0}, + {_long_name_col: "Test", "job_titles": "Test Title", "annual_salary": 123456.0}, + {_long_name_col: "Test2", "job_titles": "Test Title 2", "annual_salary": 789101.0}, ] with mock.patch("sodapy.Socrata", return_value=socrata): @@ -239,14 +253,15 @@ def test_socrata_fdw(): "domain": "data.cityofchicago.gov", "app_token": "SOME_TOKEN", "batch_size": "4200", + "column_map": json.dumps(_col_map), }, - fdw_columns=["name", "job_titles", "annual_salary"], + fdw_columns=[_long_name_col_sg, "job_titles", "annual_salary"], ) - assert fdw.get_rel_size([], ["annual_salary", "name"]) == (33702, 380) - assert fdw.can_sort([S("name"), S("salary", nulls_first=False, is_reversed=True)]) == [ - S("name") - ] + assert fdw.get_rel_size([], ["annual_salary", _long_name_col_sg]) == (33702, 380) + assert fdw.can_sort( + [S(_long_name_col_sg), S("salary", nulls_first=False, is_reversed=True)] + ) == [S(_long_name_col_sg)] assert fdw.explain([], []) == [ "Socrata query to data.cityofchicago.gov", "Socrata dataset ID: xzkq-xp2w", @@ -255,21 +270,21 @@ def test_socrata_fdw(): assert list( fdw.execute( quals=[Q("salary", ">", 42)], - columns=["name", "job_titles", "annual_salary"], - sortkeys=[S("name")], + columns=[_long_name_col_sg, "job_titles", "annual_salary"], + sortkeys=[S(_long_name_col_sg)], ) ) == [ - {"name": "Test", "job_titles": "Test Title", "annual_salary": 123456.0}, - {"name": "Test2", "job_titles": "Test Title 2", "annual_salary": 789101.0}, + {_long_name_col_sg: "Test", "job_titles": "Test Title", "annual_salary": 123456.0}, + {_long_name_col_sg: "Test2", "job_titles": "Test Title 2", "annual_salary": 789101.0}, ] assert socrata.get_all.mock_calls == [ call( dataset_identifier="xzkq-xp2w", where="(`salary` > 42)", - select="`name`,`job_titles`,`annual_salary`", + select=f"`{_long_name_col}`,`job_titles`,`annual_salary`", limit=4200, - order="`name` ASC", + order=f"`{_long_name_col}` ASC", ) ] @@ -309,7 +324,7 @@ def test_socrata_column_deduplication(): ), TableColumn( ordinal=3, - name="long_col_but_still_uniquelong_col_but_still_uniquelong_col_but_still_unique", + name="long_col_but_still_uniquelong_col_but_still_uniquelong_col_but_", pg_type="some_type", is_pk=False, comment=None,