From e0f51168181a43d5565a0d8ba11b2a28dbbd3c22 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2020 15:43:38 +0000 Subject: [PATCH 1/2] Bump pylint from 2.5.3 to 2.6.0 Bumps [pylint](https://github.com/PyCQA/pylint) from 2.5.3 to 2.6.0. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.5.3...pylint-2.6.0) Signed-off-by: dependabot-preview[bot] --- Pipfile | 2 +- Pipfile.lock | 132 ++++++++++++++++++--------------------------------- 2 files changed, 48 insertions(+), 86 deletions(-) diff --git a/Pipfile b/Pipfile index a767d1a..838947f 100644 --- a/Pipfile +++ b/Pipfile @@ -11,7 +11,7 @@ meilisearch = "==0.12.3" requests-iap = "==0.2.0" [dev-packages] -pylint = "==2.5.3" +pylint = "==2.6.0" [requires] python_version = "3.8" diff --git a/Pipfile.lock b/Pipfile.lock index 8050f9b..3dca712 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9b5b87fc284e2472e9f61023ae7b582a675f79f78b4034cb2ebc0dd4b98813de" + "sha256": "202a84e8719605b103448e46146355d86642b3847edeee72a2de507feb1cc231" }, "pipfile-spec": 6, "requires": { @@ -21,7 +21,6 @@ "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594", "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.2.0" }, "automat": { @@ -40,36 +39,44 @@ }, "cffi": { "hashes": [ - "sha256:0da50dcbccd7cb7e6c741ab7912b2eff48e85af217d72b57f80ebc616257125e", - "sha256:12a453e03124069b6896107ee133ae3ab04c624bb10683e1ed1c1663df17c13c", - "sha256:15419020b0e812b40d96ec9d369b2bc8109cc3295eac6e013d3261343580cc7e", - "sha256:15a5f59a4808f82d8ec7364cbace851df591c2d43bc76bcbe5c4543a7ddd1bf1", - "sha256:23e44937d7695c27c66a54d793dd4b45889a81b35c0751ba91040fe825ec59c4", - "sha256:29c4688ace466a365b85a51dcc5e3c853c1d283f293dfcc12f7a77e498f160d2", - "sha256:57214fa5430399dffd54f4be37b56fe22cedb2b98862550d43cc085fb698dc2c", - "sha256:577791f948d34d569acb2d1add5831731c59d5a0c50a6d9f629ae1cefd9ca4a0", - "sha256:6539314d84c4d36f28d73adc1b45e9f4ee2a89cdc7e5d2b0a6dbacba31906798", - "sha256:65867d63f0fd1b500fa343d7798fa64e9e681b594e0a07dc934c13e76ee28fb1", - "sha256:672b539db20fef6b03d6f7a14b5825d57c98e4026401fce838849f8de73fe4d4", - "sha256:6843db0343e12e3f52cc58430ad559d850a53684f5b352540ca3f1bc56df0731", - "sha256:7057613efefd36cacabbdbcef010e0a9c20a88fc07eb3e616019ea1692fa5df4", - "sha256:76ada88d62eb24de7051c5157a1a78fd853cca9b91c0713c2e973e4196271d0c", - "sha256:837398c2ec00228679513802e3744d1e8e3cb1204aa6ad408b6aff081e99a487", - "sha256:8662aabfeab00cea149a3d1c2999b0731e70c6b5bac596d95d13f643e76d3d4e", - "sha256:95e9094162fa712f18b4f60896e34b621df99147c2cee216cfa8f022294e8e9f", - "sha256:99cc66b33c418cd579c0f03b77b94263c305c389cb0c6972dac420f24b3bf123", - "sha256:9b219511d8b64d3fa14261963933be34028ea0e57455baf6781fe399c2c3206c", - "sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b", - "sha256:b9aa9d8818c2e917fa2c105ad538e222a5bce59777133840b93134022a7ce650", - "sha256:bf44a9a0141a082e89c90e8d785b212a872db793a0080c20f6ae6e2a0ebf82ad", - "sha256:c0b48b98d79cf795b0916c57bebbc6d16bb43b9fc9b8c9f57f4cf05881904c75", - "sha256:da9d3c506f43e220336433dffe643fbfa40096d408cb9b7f2477892f369d5f82", - "sha256:e4082d832e36e7f9b2278bc774886ca8207346b99f278e54c9de4834f17232f7", - "sha256:e4b9b7af398c32e408c00eb4e0d33ced2f9121fd9fb978e6c1b57edd014a7d15", - "sha256:e613514a82539fc48291d01933951a13ae93b6b444a88782480be32245ed4afa", - "sha256:f5033952def24172e60493b68717792e3aebb387a8d186c43c020d9363ee7281" - ], - "version": "==1.14.2" + "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d", + "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b", + "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4", + "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f", + "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3", + "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579", + "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537", + "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e", + "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05", + "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171", + "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca", + "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522", + "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c", + "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc", + "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d", + "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808", + "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828", + "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869", + "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d", + "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9", + "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0", + "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc", + "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15", + "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c", + "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a", + "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3", + "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1", + "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768", + "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d", + "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b", + "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e", + "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d", + "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730", + "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394", + "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1", + "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591" + ], + "version": "==1.14.3" }, "chardet": { "hashes": [ @@ -110,7 +117,6 @@ "sha256:eb80a288e3cfc08f679f95da72d2ef90cb74f6d8a8ba69d2f215c5e110b2ca32", "sha256:fa7fbcc40e2210aca26c7ac8a39467eae444d90a2c346cbcffd9133a166bcc67" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==3.1" }, "cssselect": { @@ -118,7 +124,6 @@ "sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf", "sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "hyperlink": { @@ -133,7 +138,6 @@ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.10" }, "incremental": { @@ -162,7 +166,6 @@ "sha256:a7803a1c27177d73329a0cc83a9c10de50fa4d4f37970e0194e8ae24b1fb7066", "sha256:d8f92a93d0cc9f5a7f72f01562539cb7030f7741758e764a0a8716f9b0210f7a" ], - "markers": "python_version >= '3.5'", "version": "==1.0.3" }, "jmespath": { @@ -170,7 +173,6 @@ "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9", "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.0" }, "lxml": { @@ -207,7 +209,6 @@ "sha256:ecc930ae559ea8a43377e8b60ca6f8d61ac532fc57efb915d899de4a67928efd", "sha256:f161af26f596131b63b236372e4ce40f3167c1b5b5d459b29d2514bd8c9dc9ee" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==4.5.2" }, "meilisearch": { @@ -223,7 +224,6 @@ "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" ], - "markers": "python_version >= '3.5'", "version": "==8.5.0" }, "packaging": { @@ -231,7 +231,6 @@ "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.4" }, "parsel": { @@ -246,14 +245,12 @@ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.13.1" }, "protego": { "hashes": [ "sha256:a682771bc7b51b2ff41466460896c1a5a653f9a1e71639ef365a72e66d8734b4" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.1.16" }, "py": { @@ -261,42 +258,19 @@ "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.9.0" }, "pyasn1": { "hashes": [ - "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359", - "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576", - "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf", - "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7", "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", - "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00", - "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8", - "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86", - "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12", - "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776", - "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", - "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2", - "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3" + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba" ], "version": "==0.4.8" }, "pyasn1-modules": { "hashes": [ - "sha256:0845a5582f6a02bb3e1bde9ecfc4bfcae6ec3210dd270522fee602365430c3f8", - "sha256:0fe1b68d1e486a1ed5473f1302bd991c1611d319bba158e98b106ff86e1d7199", - "sha256:15b7c67fabc7fc240d87fb9aabf999cf82311a6d6fb2c70d00d3d0604878c811", - "sha256:426edb7a5e8879f1ec54a1864f16b882c2837bfd06eee62f2c982315ee2473ed", - "sha256:65cebbaffc913f4fe9e4808735c95ea22d7a7775646ab690518c056784bc21b4", "sha256:905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e", - "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74", - "sha256:a99324196732f53093a84c4369c996713eb8c89d360a496b599fb1a9c47fc3eb", - "sha256:b80486a6c77252ea3a3e9b1e360bc9cf28eaac41263d173c032581ad2f20fe45", - "sha256:c29a5e5cc7a3f05926aff34e097e84f8589cd790ce0ed41b67aed6857b26aafd", - "sha256:cbac4bc38d117f2a49aeedec4407d23e8866ea4ac27ff2cf7fb3e5b570df19e0", - "sha256:f39edd8c4ecaa4556e989147ebf219227e2cd2e8a43c7e7fcb1f1c18c5fd6a3d", - "sha256:fe0644d9ab041506b62782e92b06b8c68cca799e1a9636ec398675459e031405" + "sha256:a50b808ffeb97cb3601dd25981f6b016cbb3d31fbf57a8b8a87428e6158d0c74" ], "version": "==0.2.8" }, @@ -305,7 +279,6 @@ "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.20" }, "pydispatcher": { @@ -320,7 +293,6 @@ "sha256:412e00137858f04bde0729913874a48485665f2d36fe9ee449f26be864af9316", "sha256:7ead136e03655af85069b6f47b23eb7c3e5c221aa9f022a4fbb499f5b7308f29" ], - "markers": "python_version >= '3.5'", "version": "==2.0.2" }, "pyjwt": { @@ -342,7 +314,6 @@ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pytest": { @@ -365,7 +336,6 @@ "sha256:b3559a131db72c33ee969480840fff4bb6dd111de7dd27c8ee1f820f4f00231b", "sha256:fe75cc94a9443b9246fc7049224f75604b113c36acb93f87b80ed42c44cbb898" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==2.24.0" }, "requests-iap": { @@ -404,7 +374,6 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "toml": { @@ -440,7 +409,6 @@ "sha256:f058bd0168271de4dcdc39845b52dd0a4a2fecf5f1246335f13f5e96eaebb467", "sha256:f3c19e5bd42bbe4bf345704ad7c326c74d3fd7a1b3844987853bef180be638d4" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==20.3.0" }, "urllib3": { @@ -448,7 +416,6 @@ "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a", "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.25.10" }, "w3lib": { @@ -501,7 +468,6 @@ "sha256:f68bf937f113b88c866d090fea0bc52a098695173fc613b055a17ff0cf9683b6", "sha256:fb55c182a3f7b84c1a2d6de5fa7b1a05d4660d866b91dbf8d74549c57a1499e8" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==5.1.0" } }, @@ -511,16 +477,14 @@ "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703", "sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386" ], - "markers": "python_version >= '3.5'", "version": "==2.4.2" }, "isort": { "hashes": [ - "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", - "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + "sha256:6187a9f1ce8784cbc6d1b88790a43e6083a6302f03e9ae482acc0f232a98c843", + "sha256:c16eaa7432a1c004c585d79b12ad080c6c421dd18fe27982ca11f95e6898e432" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==4.3.21" + "version": "==5.5.3" }, "lazy-object-proxy": { "hashes": [ @@ -546,7 +510,6 @@ "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.4.3" }, "mccabe": { @@ -558,18 +521,17 @@ }, "pylint": { "hashes": [ - "sha256:7dd78437f2d8d019717dbf287772d0b2dbdfd13fc016aa7faa08d67bccc46adc", - "sha256:d0ece7d223fe422088b0e8f13fa0a1e8eb745ebffcb8ed53d3e95394b6101a1c" + "sha256:bb4a908c9dadbc3aac18860550e870f58e1a02c9f2c204fdf5693d73be061210", + "sha256:bfe68f020f8a0fece830a22dd4d5dddb4ecc6137db04face4c3420a46a52239f" ], "index": "pypi", - "version": "==2.5.3" + "version": "==2.6.0" }, "six": { "hashes": [ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "toml": { From 3a0263dda22f57d60ecdd5e1d3c9f2ffe44c40cc Mon Sep 17 00:00:00 2001 From: Clementine Urquizar Date: Mon, 21 Sep 2020 20:21:39 +0200 Subject: [PATCH 2/2] Fix linter errors --- scraper/src/config/config_loader.py | 4 ++-- scraper/src/custom_dupefilter.py | 2 +- scraper/src/documentation_spider.py | 4 ++-- scraper/src/js_executor.py | 4 ++-- scraper/src/strategies/default_strategy.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scraper/src/config/config_loader.py b/scraper/src/config/config_loader.py index 9bcfc1a..1aea2ad 100644 --- a/scraper/src/config/config_loader.py +++ b/scraper/src/config/config_loader.py @@ -100,8 +100,8 @@ def _load_config(self, config): self.config_content = copy.deepcopy(data) return data - except ValueError: - raise ValueError('CONFIG is not a valid JSON') + except ValueError as value_error: + raise ValueError('CONFIG is not a valid JSON') from value_error sys.exit(EXIT_CODE_WRONG_CONFIG) def _parse(self): diff --git a/scraper/src/custom_dupefilter.py b/scraper/src/custom_dupefilter.py index 4a7ecab..f53321a 100644 --- a/scraper/src/custom_dupefilter.py +++ b/scraper/src/custom_dupefilter.py @@ -57,7 +57,7 @@ def custom_request_fingerprint(self, request, include_headers=None, return cache[include_headers] def __init__(self, path=None, debug=False, use_anchors=False): - super(CustomDupeFilter, self).__init__(path=path, debug=debug) + super().__init__(path=path, debug=debug) # Spread config bool self.use_anchors = use_anchors self.fingerprints_with_scheme = set() # This set will not be scheme agnostic diff --git a/scraper/src/documentation_spider.py b/scraper/src/documentation_spider.py index 58c3eb8..ac840e7 100644 --- a/scraper/src/documentation_spider.py +++ b/scraper/src/documentation_spider.py @@ -80,7 +80,7 @@ def __init__(self, config, meilisearch_helper, strategy, *args, **kwargs): self.remove_get_params = config.remove_get_params self.strict_redirect = config.strict_redirect self.nb_hits_max = config.nb_hits_max - super(DocumentationSpider, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) # Get rid of scheme consideration # Start_urls must stays authentic URL in order to be reached, we build agnostic scheme regex based on those URL @@ -120,7 +120,7 @@ def __init__(self, config, meilisearch_helper, strategy, *args, **kwargs): self.force_sitemap_urls_crawling = config.force_sitemap_urls_crawling # END _init_ part from SitemapSpider - super(DocumentationSpider, self)._compile_rules() + super()._compile_rules() def start_requests(self): # We crawl according to the sitemap diff --git a/scraper/src/js_executor.py b/scraper/src/js_executor.py index f98ceec..9c782c4 100644 --- a/scraper/src/js_executor.py +++ b/scraper/src/js_executor.py @@ -18,5 +18,5 @@ def execute(self, url, js): try: parsed_result = json.loads(result) return parsed_result - except ValueError: - raise ValueError('CONFIG is not a valid JSON') + except ValueError as value_error: + raise ValueError('CONFIG is not a valid JSON') from value_error diff --git a/scraper/src/strategies/default_strategy.py b/scraper/src/strategies/default_strategy.py index 76b8be5..bb2b00c 100644 --- a/scraper/src/strategies/default_strategy.py +++ b/scraper/src/strategies/default_strategy.py @@ -20,7 +20,7 @@ class DefaultStrategy(AbstractStrategy): dom = None def __init__(self, config): - super(DefaultStrategy, self).__init__(config) + super().__init__(config) self.levels = ['lvl0', 'lvl1', 'lvl2', 'lvl3', 'lvl4', 'lvl5', 'lvl6'] self.global_content = {} self.page_rank = {}