From 31ed2ed4b40948ffb19bf517f0e2b163b1a83f3a Mon Sep 17 00:00:00 2001 From: Abhi Vaidyanatha Date: Thu, 29 Apr 2021 20:09:37 -0700 Subject: [PATCH 1/8] Add toy source for the PokeAPI --- .../connectors/source-pokeapi/.dockerignore | 6 ++ .../connectors/source-pokeapi/Dockerfile | 16 +++ .../connectors/source-pokeapi/README.md | 100 ++++++++++++++++++ .../connectors/source-pokeapi/build.gradle | 32 ++++++ .../connectors/source-pokeapi/main_dev.py | 32 ++++++ .../source-pokeapi/requirements.txt | 4 + .../source-pokeapi/sample_files/config.json | 1 + .../sample_files/configured_catalog.json | 21 ++++ .../sample_files/invalid_config.json | 1 + .../connectors/source-pokeapi/setup.py | 35 ++++++ .../source-pokeapi/source_pokeapi/__init__.py | 27 +++++ .../source_pokeapi/schemas/TODO.md | 25 +++++ .../source_pokeapi/schemas/pokemon.json | 9 ++ .../source-pokeapi/source_pokeapi/source.py | 84 +++++++++++++++ .../source-pokeapi/source_pokeapi/spec.json | 16 +++ .../connectors/source-pokeapi/spec.json | 18 ++++ .../source-pokeapi/unit_tests/unit_test.py | 29 +++++ 17 files changed, 456 insertions(+) create mode 100644 airbyte-integrations/connectors/source-pokeapi/.dockerignore create mode 100644 airbyte-integrations/connectors/source-pokeapi/Dockerfile create mode 100644 airbyte-integrations/connectors/source-pokeapi/README.md create mode 100644 airbyte-integrations/connectors/source-pokeapi/build.gradle create mode 100644 airbyte-integrations/connectors/source-pokeapi/main_dev.py create mode 100644 airbyte-integrations/connectors/source-pokeapi/requirements.txt create mode 100644 airbyte-integrations/connectors/source-pokeapi/sample_files/config.json create mode 100644 airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-pokeapi/sample_files/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-pokeapi/setup.py create mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py create mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json create mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py create mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json create mode 100644 airbyte-integrations/connectors/source-pokeapi/spec.json create mode 100644 airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/source-pokeapi/.dockerignore b/airbyte-integrations/connectors/source-pokeapi/.dockerignore new file mode 100644 index 0000000000000..04e7483747252 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!Dockerfile.test +!source_pokeapi +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-pokeapi/Dockerfile b/airbyte-integrations/connectors/source-pokeapi/Dockerfile new file mode 100644 index 0000000000000..e6c477a5ce5c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/Dockerfile @@ -0,0 +1,16 @@ +FROM airbyte/integration-base-python:0.1.7 + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="source_pokeapi" +ENV AIRBYTE_IMPL_MODULE="source_pokeapi" +ENV AIRBYTE_IMPL_PATH="SourcePokeapi" + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install . + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-pokeapi diff --git a/airbyte-integrations/connectors/source-pokeapi/README.md b/airbyte-integrations/connectors/source-pokeapi/README.md new file mode 100644 index 0000000000000..13edd8fd8caba --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/README.md @@ -0,0 +1,100 @@ +# Pokeapi Source + +This is the repository for the Pokeapi source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pokeapi). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-pokeapi:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pokeapi) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pokeapi/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pokeapi test creds` +and place them into `secrets/config.json`. + + +### Locally running the connector +``` +python main_dev.py spec +python main_dev.py check --config secrets/config.json +python main_dev.py discover --config secrets/config.json +python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-pokeapi:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-pokeapi:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-pokeapi:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pokeapi:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pokeapi:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-pokeapi:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +``` + +### Integration Tests +1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-pokeapi:integrationTest` to run the standard integration test suite. +1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). +1. Create a Pull Request +1. Pat yourself on the back for being an awesome contributor +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master diff --git a/airbyte-integrations/connectors/source-pokeapi/build.gradle b/airbyte-integrations/connectors/source-pokeapi/build.gradle new file mode 100644 index 0000000000000..55922454bdbfc --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/build.gradle @@ -0,0 +1,32 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-standard-source-test-file' +} + +airbytePython { + moduleDirectory 'source_pokeapi' +} + +airbyteStandardSourceTestFile { + // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors + + // All these input paths must live inside this connector's directory (or subdirectories) + // TODO update the spec JSON file + specPath = "source_pokeapi/spec.json" + + // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file + // there (in case it contains any credentials) + // TODO update the config file to contain actual credentials + configPath = "secrets/config.json" + // TODO update the sample configured_catalog JSON for use in testing + // Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured + // for incremental syncing (e.g. include cursor fields, etc). + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + +dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-pokeapi/main_dev.py b/airbyte-integrations/connectors/source-pokeapi/main_dev.py new file mode 100644 index 0000000000000..deb0263b964e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/main_dev.py @@ -0,0 +1,32 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import sys + +from base_python.entrypoint import launch +from source_pokeapi import SourcePokeapi + +if __name__ == "__main__": + source = SourcePokeapi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pokeapi/requirements.txt b/airbyte-integrations/connectors/source-pokeapi/requirements.txt new file mode 100644 index 0000000000000..dd447512e620a --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/requirements.txt @@ -0,0 +1,4 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/airbyte-protocol +-e ../../bases/base-python +-e . diff --git a/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json b/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json new file mode 100644 index 0000000000000..b6be551d52dd2 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json @@ -0,0 +1 @@ +{"name": "ditto"} diff --git a/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json new file mode 100644 index 0000000000000..50b6decb33e73 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json @@ -0,0 +1,21 @@ +{ + "streams": [ + { + "stream": { + "name": "pokemon", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "name": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pokeapi/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-pokeapi/sample_files/invalid_config.json new file mode 100644 index 0000000000000..09d9db11e96c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/sample_files/invalid_config.json @@ -0,0 +1 @@ +{"name": "datto"} diff --git a/airbyte-integrations/connectors/source-pokeapi/setup.py b/airbyte-integrations/connectors/source-pokeapi/setup.py new file mode 100644 index 0000000000000..3598ea5da3c40 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/setup.py @@ -0,0 +1,35 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from setuptools import find_packages, setup + +setup( + name="source_pokeapi", + description="Source implementation for Pokeapi.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=["airbyte-protocol", "base-python", "pytest==6.1.2"], + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]} +) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py new file mode 100644 index 0000000000000..fccde64cd0e05 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourcePokeapi + +__all__ = ["SourcePokeapi"] diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/TODO.md b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/TODO.md new file mode 100644 index 0000000000000..cf1efadb3c9c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json new file mode 100644 index 0000000000000..a90ed1fede997 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py new file mode 100644 index 0000000000000..55eb0ad8b3504 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -0,0 +1,84 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests + +from base_python import AbstractSource, HttpStream, Stream + +POKEMON_LIST = ['bulbasaur','ivysaur','venusaur','charmander','charmeleon','charizard','squirtle','wartortle','blastoise','caterpie','metapod','butterfree','weedle','kakuna','beedrill','pidgey','pidgeotto','pidgeot','rattata','raticate','spearow','fearow','ekans','arbok','pikachu','raichu','sandshrew','sandslash','nidoranf','nidorina','nidoqueen','nidoranm','nidorino','nidoking','clefairy','clefable','vulpix','ninetales','jigglypuff','wigglytuff','zubat','golbat','oddish','gloom','vileplume','paras','parasect','venonat','venomoth','diglett','dugtrio','meowth','persian','psyduck','golduck','mankey','primeape','growlithe','arcanine','poliwag','poliwhirl','poliwrath','abra','kadabra','alakazam','machop','machoke','machamp','bellsprout','weepinbell','victreebel','tentacool','tentacruel','geodude','graveler','golem','ponyta','rapidash','slowpoke','slowbro','magnemite','magneton','farfetchd','doduo','dodrio','seel','dewgong','grimer','muk','shellder','cloyster','gastly','haunter','gengar','onix','drowzee','hypno','krabby','kingler','voltorb','electrode','exeggcute','exeggutor','cubone','marowak','hitmonlee','hitmonchan','lickitung','koffing','weezing','rhyhorn','rhydon','chansey','tangela','kangaskhan','horsea','seadra','goldeen','seaking','staryu','starmie','mrmime','scyther','jynx','electabuzz','magmar','pinsir','tauros','magikarp','gyarados','lapras','ditto','eevee','vaporeon','jolteon','flareon','porygon','omanyte','omastar','kabuto','kabutops','aerodactyl','snorlax','articuno','zapdos','moltres','dratini','dragonair','dragonite','mewtwo','mew','chikorita','bayleef','meganium','cyndaquil','quilava','typhlosion','totodile','croconaw','feraligatr','sentret','furret','hoothoot','noctowl','ledyba','ledian','spinarak','ariados','crobat','chinchou','lanturn','pichu','cleffa','igglybuff','togepi','togetic','natu','xatu','mareep','flaaffy','ampharos','bellossom','marill','azumarill','sudowoodo','politoed','hoppip','skiploom','jumpluff','aipom','sunkern','sunflora','yanma','wooper','quagsire','espeon','umbreon','murkrow','slowking','misdreavus','unown','wobbuffet','girafarig','pineco','forretress','dunsparce','gligar','steelix','snubbull','granbull','qwilfish','scizor','shuckle','heracross','sneasel','teddiursa','ursaring','slugma','magcargo','swinub','piloswine','corsola','remoraid','octillery','delibird','mantine','skarmory','houndour','houndoom','kingdra','phanpy','donphan','porygon2','stantler','smeargle','tyrogue','hitmontop','smoochum','elekid','magby','miltank','blissey','raikou','entei','suicune','larvitar','pupitar','tyranitar','lugia','ho-oh','celebi','treecko','grovyle','sceptile','torchic','combusken','blaziken','mudkip','marshtomp','swampert','poochyena','mightyena','zigzagoon','linoone','wurmple','silcoon','beautifly','cascoon','dustox','lotad','lombre','ludicolo','seedot','nuzleaf','shiftry','taillow','swellow','wingull','pelipper','ralts','kirlia','gardevoir','surskit','masquerain','shroomish','breloom','slakoth','vigoroth','slaking','nincada','ninjask','shedinja','whismur','loudred','exploud','makuhita','hariyama','azurill','nosepass','skitty','delcatty','sableye','mawile','aron','lairon','aggron','meditite','medicham','electrike','manectric','plusle','minun','volbeat','illumise','roselia','gulpin','swalot','carvanha','sharpedo','wailmer','wailord','numel','camerupt','torkoal','spoink','grumpig','spinda','trapinch','vibrava','flygon','cacnea','cacturne','swablu','altaria','zangoose','seviper','lunatone','solrock','barboach','whiscash','corphish','crawdaunt','baltoy','claydol','lileep','cradily','anorith','armaldo','feebas','milotic','castform','kecleon','shuppet','banette','duskull','dusclops','tropius','chimecho','absol','wynaut','snorunt','glalie','spheal','sealeo','walrein','clamperl','huntail','gorebyss','relicanth','luvdisc','bagon','shelgon','salamence','beldum','metang','metagross','regirock','regice','registeel','latias','latios','kyogre','groudon','rayquaza','jirachi','deoxys','turtwig','grotle','torterra','chimchar','monferno','infernape','piplup','prinplup','empoleon','starly','staravia','staraptor','bidoof','bibarel','kricketot','kricketune','shinx','luxio','luxray','budew','roserade','cranidos','rampardos','shieldon','bastiodon','burmy','wormadam','mothim','combee','vespiquen','pachirisu','buizel','floatzel','cherubi','cherrim','shellos','gastrodon','ambipom','drifloon','drifblim','buneary','lopunny','mismagius','honchkrow','glameow','purugly','chingling','stunky','skuntank','bronzor','bronzong','bonsly','mimejr','happiny','chatot','spiritomb','gible','gabite','garchomp','munchlax','riolu','lucario','hippopotas','hippowdon','skorupi','drapion','croagunk','toxicroak','carnivine','finneon','lumineon','mantyke','snover','abomasnow','weavile','magnezone','lickilicky','rhyperior','tangrowth','electivire','magmortar','togekiss','yanmega','leafeon','glaceon','gliscor','mamoswine','porygon-z','gallade','probopass','dusknoir','froslass','rotom','uxie','mesprit','azelf','dialga','palkia','heatran','regigigas','giratina','cresselia','phione','manaphy','darkrai','shaymin','arceus','victini','snivy','servine','serperior','tepig','pignite','emboar','oshawott','dewott','samurott','patrat','watchog','lillipup','herdier','stoutland','purrloin','liepard','pansage','simisage','pansear','simisear','panpour','simipour','munna','musharna','pidove','tranquill','unfezant','blitzle','zebstrika','roggenrola','boldore','gigalith','woobat','swoobat','drilbur','excadrill','audino','timburr','gurdurr','conkeldurr','tympole','palpitoad','seismitoad','throh','sawk','sewaddle','swadloon','leavanny','venipede','whirlipede','scolipede','cottonee','whimsicott','petilil','lilligant','basculin','sandile','krokorok','krookodile','darumaka','darmanitan','maractus','dwebble','crustle','scraggy','scrafty','sigilyph','yamask','cofagrigus','tirtouga','carracosta','archen','archeops','trubbish','garbodor','zorua','zoroark','minccino','cinccino','gothita','gothorita','gothitelle','solosis','duosion','reuniclus','ducklett','swanna','vanillite','vanillish','vanilluxe','deerling','sawsbuck','emolga','karrablast','escavalier','foongus','amoonguss','frillish','jellicent','alomomola','joltik','galvantula','ferroseed','ferrothorn','klink','klang','klinklang','tynamo','eelektrik','eelektross','elgyem','beheeyem','litwick','lampent','chandelure','axew','fraxure','haxorus','cubchoo','beartic','cryogonal','shelmet','accelgor','stunfisk','mienfoo','mienshao','druddigon','golett','golurk','pawniard','bisharp','bouffalant','rufflet','braviary','vullaby','mandibuzz','heatmor','durant','deino','zweilous','hydreigon','larvesta','volcarona','cobalion','terrakion','virizion','tornadus','thundurus','reshiram','zekrom','landorus','kyurem','keldeo','meloetta','genesect','chespin','quilladin','chesnaught','fennekin','braixen','delphox','froakie','frogadier','greninja','bunnelby','diggersby','fletchling','fletchinder','talonflame','scatterbug','spewpa','vivillon','litleo','pyroar','flabebe','floette','florges','skiddo','gogoat','pancham','pangoro','furfrou','espurr','meowstic','honedge','doublade','aegislash','spritzee','aromatisse','swirlix','slurpuff','inkay','malamar','binacle','barbaracle','skrelp','dragalge','clauncher','clawitzer','helioptile','heliolisk','tyrunt','tyrantrum','amaura','aurorus','sylveon','hawlucha','dedenne','carbink','goomy','sliggoo','goodra','klefki','phantump','trevenant','pumpkaboo','gourgeist','bergmite','avalugg','noibat','noivern','xerneas','yveltal','zygarde','diancie','hoopa','volcanion','rowlet','dartrix','decidueye','litten','torracat','incineroar','popplio','brionne','primarina','pikipek','trumbeak','toucannon','yungoos','gumshoos','grubbin','charjabug','vikavolt','crabrawler','crabominable','oricorio','cutiefly','ribombee','rockruff','lycanroc','wishiwashi','mareanie','toxapex','mudbray','mudsdale','dewpider','araquanid','fomantis','lurantis','morelull','shiinotic','salandit','salazzle','stufful','bewear','bounsweet','steenee','tsareena','comfey','oranguru','passimian','wimpod','golisopod','sandygast','palossand','pyukumuku','typenull','silvally','minior','komala','turtonator','togedemaru','mimikyu','bruxish','drampa','dhelmise','jangmo-o','hakamo-o','kommo-o','tapukoko','tapulele','tapubulu','tapufini','cosmog','cosmoem','solgaleo','lunala','nihilego','buzzwole','pheromosa','xurkitree','celesteela','kartana','guzzlord','necrozma','magearna','marshadow','poipole','naganadel','stakataka','blacephalon','zeraora','meltan','melmetal','grookey','thwackey','rillaboom','scorbunny','raboot','cinderace','sobble','drizzile','inteleon','skwovet','greedent','rookidee','corvisquire','corviknight','blipbug','dottler','orbeetle','nickit','thievul','gossifleur','eldegoss','wooloo','dubwool','chewtle','drednaw','yamper','boltund','rolycoly','carkol','coalossal','applin','flapple','appletun','silicobra','sandaconda','cramorant','arrokuda','barraskewda','toxel','toxtricity','sizzlipede','centiskorch','clobbopus','grapploct','sinistea','polteageist','hatenna','hattrem','hatterene','impidimp','morgrem','grimmsnarl','obstagoon','perrserker','cursola','sirfetchd','mrrime','runerigus','milcery','alcremie','falinks','pincurchin','snom','frosmoth','stonjourner','eiscue','indeedee','morpeko','cufant','copperajah','dracozolt','arctozolt','dracovish','arctovish','duraludon','dreepy','drakloak','dragapult','zacian','zamazenta','eternatus','kubfu','urshifu','zarude','regieleki','regidrago','glastrier','spectrier','calyrex'] + +# Basic full refresh stream +class PokeapiStream(HttpStream): + url_base = 'https://pokeapi.co/api/v2/' + + def __init__(self, name: str, **kwargs): + super().__init__() + self._name = name + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + # The api requires that we include the base currency as a query param so we do that in this method + return {"name": self.name} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + # The response is a simple JSON whose schema matches our stream's schema exactly, + # so we just return a list containing the response + return [response.json()] + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + # The API does not offer pagination, + # so we return None to indicate there are no more pages in the response + return None + + +class Pokemon(PokeapiStream): + def path(self, **kwargs) -> str: + pokemon_name = self._name + return f'pokemon/{pokemon_name}' + +# Source +class SourcePokeapi(AbstractSource): + + def check_connection(self, logger, config) -> Tuple[bool, any]: + input_pokemon = config['name'] + if input_pokemon not in POKEMON_LIST: + return False, f"Input Pokemon {input_pokemon} is invalid. Please check your spelling our input a valid Pokemon." + else: + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + args = {"name": config["name"]} + return [Pokemon(**args)] diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json new file mode 100644 index 0000000000000..eb3f55c30a6c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json @@ -0,0 +1,16 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Pokeapi Spec", + "type": "object", + "required": ["TODO"], + "additionalProperties": false, + "properties": { + "TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.": { + "type": "string", + "description": "describe me" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pokeapi/spec.json b/airbyte-integrations/connectors/source-pokeapi/spec.json new file mode 100644 index 0000000000000..8d2b88858e65a --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/spec.json @@ -0,0 +1,18 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/pokeapi", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Pokeapi Spec", + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Pokemon requested from the API.", + "pattern": "^[a-z0-9_\\-]+$", + "examples": ["ditto, luxray, snorlax"] + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py new file mode 100644 index 0000000000000..2854c429a7db5 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py @@ -0,0 +1,29 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +# format anchor + + +def test_example_method(): + assert True From f2cc885f8cdb5dc2dff90918a399c6fe6565ec3e Mon Sep 17 00:00:00 2001 From: Abhi Vaidyanatha Date: Fri, 30 Apr 2021 17:52:06 -0700 Subject: [PATCH 2/8] Address review comments. --- .../source_pokeapi/schemas/pokemon.json | 262 ++++++++++++++++++ .../source-pokeapi/source_pokeapi/source.py | 14 +- .../source-pokeapi/source_pokeapi/spec.json | 12 +- .../connectors/source-pokeapi/spec.json | 18 -- 4 files changed, 276 insertions(+), 30 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-pokeapi/spec.json diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json index a90ed1fede997..978d311d0be1a 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json @@ -2,8 +2,270 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "id": { + "type": ["null", "integer"] + }, "name": { "type": ["null", "string"] + }, + "base_experience": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + }, + "is_default ": { + "type": ["null", "boolean"] + }, + "order": { + "type": ["null", "integer"] + }, + "weight": { + "type": ["null", "integer"] + }, + "abilities": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "is_hidden": { + "type": ["null", "boolean"] + }, + "slot": { + "type": ["null", "integer"] + }, + "ability": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + }, + "forms": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + }, + "game_indices": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "game_index": { + "type": ["null", "integer"] + }, + "version": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + }, + "held_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "item": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "version_details": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "version": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "rarity": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "location_area_encounters": { + "type": ["null", "string"] + }, + "moves": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "move": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "version_group_details": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "move_learn_method": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "version_group": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "level_learned_at": { + "type": ["null", "integer"] + } + } + } + } + } + } + }, + "sprites": { + "type": ["null", "object"], + "properties": { + "front_default": { + "type": ["null", "string"] + }, + "front_shiny": { + "type": ["null", "string"] + }, + "front_female": { + "type": ["null", "string"] + }, + "front_shiny_female": { + "type": ["null", "string"] + }, + "back_default": { + "type": ["null", "string"] + }, + "back_shiny": { + "type": ["null", "string"] + }, + "back_female": { + "type": ["null", "string"] + }, + "back_shiny_female": { + "type": ["null", "string"] + } + } + }, + "species": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "stats": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "stat": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "effort": { + "type": ["null", "integer"] + }, + "base_stat": { + "type": ["null", "integer"] + } + } + } + }, + "types": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "slot": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 55eb0ad8b3504..263e1fc303894 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -34,9 +34,9 @@ class PokeapiStream(HttpStream): url_base = 'https://pokeapi.co/api/v2/' - def __init__(self, name: str, **kwargs): - super().__init__() - self._name = name + def __init__(self, pokemon_name: str, **kwargs): + super().__init__(**kwargs) + self.pokemon_name = pokemon_name def request_params( self, @@ -44,8 +44,8 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - # The api requires that we include the base currency as a query param so we do that in this method - return {"name": self.name} + # The api requires that we include the Pokemon name as a query param so we do that in this method + return {"pokemon_name": self.pokemon_name} def parse_response( self, @@ -66,7 +66,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, class Pokemon(PokeapiStream): def path(self, **kwargs) -> str: - pokemon_name = self._name + pokemon_name = self.pokemon_name return f'pokemon/{pokemon_name}' # Source @@ -81,4 +81,4 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: args = {"name": config["name"]} - return [Pokemon(**args)] + return [Pokemon(name=config["name"])] diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json index eb3f55c30a6c4..8d2b88858e65a 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json @@ -1,16 +1,18 @@ { - "documentationUrl": "https://docsurl.com", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/pokeapi", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Pokeapi Spec", "type": "object", - "required": ["TODO"], + "required": ["name"], "additionalProperties": false, "properties": { - "TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.": { + "name": { "type": "string", - "description": "describe me" + "description": "Pokemon requested from the API.", + "pattern": "^[a-z0-9_\\-]+$", + "examples": ["ditto, luxray, snorlax"] } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pokeapi/spec.json b/airbyte-integrations/connectors/source-pokeapi/spec.json deleted file mode 100644 index 8d2b88858e65a..0000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/spec.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/pokeapi", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pokeapi Spec", - "type": "object", - "required": ["name"], - "additionalProperties": false, - "properties": { - "name": { - "type": "string", - "description": "Pokemon requested from the API.", - "pattern": "^[a-z0-9_\\-]+$", - "examples": ["ditto, luxray, snorlax"] - } - } - } -} \ No newline at end of file From 8c936cc85fb743fecd9651f1075e8ff0a1975cc2 Mon Sep 17 00:00:00 2001 From: Abhi Vaidyanatha Date: Fri, 30 Apr 2021 18:29:38 -0700 Subject: [PATCH 3/8] Run gradlew format --- .../connectors/source-pokeapi/main_dev.py | 23 + .../source-pokeapi/sample_files/config.json | 2 +- .../sample_files/configured_catalog.json | 2 +- .../sample_files/invalid_config.json | 2 +- .../connectors/source-pokeapi/setup.py | 25 +- .../source-pokeapi/source_pokeapi/source.py | 954 +++++++++++++++++- .../source-pokeapi/source_pokeapi/spec.json | 2 +- .../source-pokeapi/unit_tests/unit_test.py | 23 + 8 files changed, 1011 insertions(+), 22 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/main_dev.py b/airbyte-integrations/connectors/source-pokeapi/main_dev.py index deb0263b964e4..eb61216f69660 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main_dev.py +++ b/airbyte-integrations/connectors/source-pokeapi/main_dev.py @@ -1,3 +1,26 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + """ MIT License diff --git a/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json b/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json index b6be551d52dd2..4111fdb6585a5 100644 --- a/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json +++ b/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json @@ -1 +1 @@ -{"name": "ditto"} +{ "name": "ditto" } diff --git a/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json index 50b6decb33e73..6169d1c68cd5c 100644 --- a/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json @@ -18,4 +18,4 @@ "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-pokeapi/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-pokeapi/sample_files/invalid_config.json index 09d9db11e96c9..0d853925b60a1 100644 --- a/airbyte-integrations/connectors/source-pokeapi/sample_files/invalid_config.json +++ b/airbyte-integrations/connectors/source-pokeapi/sample_files/invalid_config.json @@ -1 +1 @@ -{"name": "datto"} +{ "name": "datto" } diff --git a/airbyte-integrations/connectors/source-pokeapi/setup.py b/airbyte-integrations/connectors/source-pokeapi/setup.py index 3598ea5da3c40..d5ba4bbfaec5f 100644 --- a/airbyte-integrations/connectors/source-pokeapi/setup.py +++ b/airbyte-integrations/connectors/source-pokeapi/setup.py @@ -1,3 +1,26 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + """ MIT License @@ -31,5 +54,5 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=["airbyte-protocol", "base-python", "pytest==6.1.2"], - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]} + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, ) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 263e1fc303894..4efcd0f8d72de 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -1,3 +1,26 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + """ MIT License @@ -25,34 +48,932 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests - from base_python import AbstractSource, HttpStream, Stream -POKEMON_LIST = ['bulbasaur','ivysaur','venusaur','charmander','charmeleon','charizard','squirtle','wartortle','blastoise','caterpie','metapod','butterfree','weedle','kakuna','beedrill','pidgey','pidgeotto','pidgeot','rattata','raticate','spearow','fearow','ekans','arbok','pikachu','raichu','sandshrew','sandslash','nidoranf','nidorina','nidoqueen','nidoranm','nidorino','nidoking','clefairy','clefable','vulpix','ninetales','jigglypuff','wigglytuff','zubat','golbat','oddish','gloom','vileplume','paras','parasect','venonat','venomoth','diglett','dugtrio','meowth','persian','psyduck','golduck','mankey','primeape','growlithe','arcanine','poliwag','poliwhirl','poliwrath','abra','kadabra','alakazam','machop','machoke','machamp','bellsprout','weepinbell','victreebel','tentacool','tentacruel','geodude','graveler','golem','ponyta','rapidash','slowpoke','slowbro','magnemite','magneton','farfetchd','doduo','dodrio','seel','dewgong','grimer','muk','shellder','cloyster','gastly','haunter','gengar','onix','drowzee','hypno','krabby','kingler','voltorb','electrode','exeggcute','exeggutor','cubone','marowak','hitmonlee','hitmonchan','lickitung','koffing','weezing','rhyhorn','rhydon','chansey','tangela','kangaskhan','horsea','seadra','goldeen','seaking','staryu','starmie','mrmime','scyther','jynx','electabuzz','magmar','pinsir','tauros','magikarp','gyarados','lapras','ditto','eevee','vaporeon','jolteon','flareon','porygon','omanyte','omastar','kabuto','kabutops','aerodactyl','snorlax','articuno','zapdos','moltres','dratini','dragonair','dragonite','mewtwo','mew','chikorita','bayleef','meganium','cyndaquil','quilava','typhlosion','totodile','croconaw','feraligatr','sentret','furret','hoothoot','noctowl','ledyba','ledian','spinarak','ariados','crobat','chinchou','lanturn','pichu','cleffa','igglybuff','togepi','togetic','natu','xatu','mareep','flaaffy','ampharos','bellossom','marill','azumarill','sudowoodo','politoed','hoppip','skiploom','jumpluff','aipom','sunkern','sunflora','yanma','wooper','quagsire','espeon','umbreon','murkrow','slowking','misdreavus','unown','wobbuffet','girafarig','pineco','forretress','dunsparce','gligar','steelix','snubbull','granbull','qwilfish','scizor','shuckle','heracross','sneasel','teddiursa','ursaring','slugma','magcargo','swinub','piloswine','corsola','remoraid','octillery','delibird','mantine','skarmory','houndour','houndoom','kingdra','phanpy','donphan','porygon2','stantler','smeargle','tyrogue','hitmontop','smoochum','elekid','magby','miltank','blissey','raikou','entei','suicune','larvitar','pupitar','tyranitar','lugia','ho-oh','celebi','treecko','grovyle','sceptile','torchic','combusken','blaziken','mudkip','marshtomp','swampert','poochyena','mightyena','zigzagoon','linoone','wurmple','silcoon','beautifly','cascoon','dustox','lotad','lombre','ludicolo','seedot','nuzleaf','shiftry','taillow','swellow','wingull','pelipper','ralts','kirlia','gardevoir','surskit','masquerain','shroomish','breloom','slakoth','vigoroth','slaking','nincada','ninjask','shedinja','whismur','loudred','exploud','makuhita','hariyama','azurill','nosepass','skitty','delcatty','sableye','mawile','aron','lairon','aggron','meditite','medicham','electrike','manectric','plusle','minun','volbeat','illumise','roselia','gulpin','swalot','carvanha','sharpedo','wailmer','wailord','numel','camerupt','torkoal','spoink','grumpig','spinda','trapinch','vibrava','flygon','cacnea','cacturne','swablu','altaria','zangoose','seviper','lunatone','solrock','barboach','whiscash','corphish','crawdaunt','baltoy','claydol','lileep','cradily','anorith','armaldo','feebas','milotic','castform','kecleon','shuppet','banette','duskull','dusclops','tropius','chimecho','absol','wynaut','snorunt','glalie','spheal','sealeo','walrein','clamperl','huntail','gorebyss','relicanth','luvdisc','bagon','shelgon','salamence','beldum','metang','metagross','regirock','regice','registeel','latias','latios','kyogre','groudon','rayquaza','jirachi','deoxys','turtwig','grotle','torterra','chimchar','monferno','infernape','piplup','prinplup','empoleon','starly','staravia','staraptor','bidoof','bibarel','kricketot','kricketune','shinx','luxio','luxray','budew','roserade','cranidos','rampardos','shieldon','bastiodon','burmy','wormadam','mothim','combee','vespiquen','pachirisu','buizel','floatzel','cherubi','cherrim','shellos','gastrodon','ambipom','drifloon','drifblim','buneary','lopunny','mismagius','honchkrow','glameow','purugly','chingling','stunky','skuntank','bronzor','bronzong','bonsly','mimejr','happiny','chatot','spiritomb','gible','gabite','garchomp','munchlax','riolu','lucario','hippopotas','hippowdon','skorupi','drapion','croagunk','toxicroak','carnivine','finneon','lumineon','mantyke','snover','abomasnow','weavile','magnezone','lickilicky','rhyperior','tangrowth','electivire','magmortar','togekiss','yanmega','leafeon','glaceon','gliscor','mamoswine','porygon-z','gallade','probopass','dusknoir','froslass','rotom','uxie','mesprit','azelf','dialga','palkia','heatran','regigigas','giratina','cresselia','phione','manaphy','darkrai','shaymin','arceus','victini','snivy','servine','serperior','tepig','pignite','emboar','oshawott','dewott','samurott','patrat','watchog','lillipup','herdier','stoutland','purrloin','liepard','pansage','simisage','pansear','simisear','panpour','simipour','munna','musharna','pidove','tranquill','unfezant','blitzle','zebstrika','roggenrola','boldore','gigalith','woobat','swoobat','drilbur','excadrill','audino','timburr','gurdurr','conkeldurr','tympole','palpitoad','seismitoad','throh','sawk','sewaddle','swadloon','leavanny','venipede','whirlipede','scolipede','cottonee','whimsicott','petilil','lilligant','basculin','sandile','krokorok','krookodile','darumaka','darmanitan','maractus','dwebble','crustle','scraggy','scrafty','sigilyph','yamask','cofagrigus','tirtouga','carracosta','archen','archeops','trubbish','garbodor','zorua','zoroark','minccino','cinccino','gothita','gothorita','gothitelle','solosis','duosion','reuniclus','ducklett','swanna','vanillite','vanillish','vanilluxe','deerling','sawsbuck','emolga','karrablast','escavalier','foongus','amoonguss','frillish','jellicent','alomomola','joltik','galvantula','ferroseed','ferrothorn','klink','klang','klinklang','tynamo','eelektrik','eelektross','elgyem','beheeyem','litwick','lampent','chandelure','axew','fraxure','haxorus','cubchoo','beartic','cryogonal','shelmet','accelgor','stunfisk','mienfoo','mienshao','druddigon','golett','golurk','pawniard','bisharp','bouffalant','rufflet','braviary','vullaby','mandibuzz','heatmor','durant','deino','zweilous','hydreigon','larvesta','volcarona','cobalion','terrakion','virizion','tornadus','thundurus','reshiram','zekrom','landorus','kyurem','keldeo','meloetta','genesect','chespin','quilladin','chesnaught','fennekin','braixen','delphox','froakie','frogadier','greninja','bunnelby','diggersby','fletchling','fletchinder','talonflame','scatterbug','spewpa','vivillon','litleo','pyroar','flabebe','floette','florges','skiddo','gogoat','pancham','pangoro','furfrou','espurr','meowstic','honedge','doublade','aegislash','spritzee','aromatisse','swirlix','slurpuff','inkay','malamar','binacle','barbaracle','skrelp','dragalge','clauncher','clawitzer','helioptile','heliolisk','tyrunt','tyrantrum','amaura','aurorus','sylveon','hawlucha','dedenne','carbink','goomy','sliggoo','goodra','klefki','phantump','trevenant','pumpkaboo','gourgeist','bergmite','avalugg','noibat','noivern','xerneas','yveltal','zygarde','diancie','hoopa','volcanion','rowlet','dartrix','decidueye','litten','torracat','incineroar','popplio','brionne','primarina','pikipek','trumbeak','toucannon','yungoos','gumshoos','grubbin','charjabug','vikavolt','crabrawler','crabominable','oricorio','cutiefly','ribombee','rockruff','lycanroc','wishiwashi','mareanie','toxapex','mudbray','mudsdale','dewpider','araquanid','fomantis','lurantis','morelull','shiinotic','salandit','salazzle','stufful','bewear','bounsweet','steenee','tsareena','comfey','oranguru','passimian','wimpod','golisopod','sandygast','palossand','pyukumuku','typenull','silvally','minior','komala','turtonator','togedemaru','mimikyu','bruxish','drampa','dhelmise','jangmo-o','hakamo-o','kommo-o','tapukoko','tapulele','tapubulu','tapufini','cosmog','cosmoem','solgaleo','lunala','nihilego','buzzwole','pheromosa','xurkitree','celesteela','kartana','guzzlord','necrozma','magearna','marshadow','poipole','naganadel','stakataka','blacephalon','zeraora','meltan','melmetal','grookey','thwackey','rillaboom','scorbunny','raboot','cinderace','sobble','drizzile','inteleon','skwovet','greedent','rookidee','corvisquire','corviknight','blipbug','dottler','orbeetle','nickit','thievul','gossifleur','eldegoss','wooloo','dubwool','chewtle','drednaw','yamper','boltund','rolycoly','carkol','coalossal','applin','flapple','appletun','silicobra','sandaconda','cramorant','arrokuda','barraskewda','toxel','toxtricity','sizzlipede','centiskorch','clobbopus','grapploct','sinistea','polteageist','hatenna','hattrem','hatterene','impidimp','morgrem','grimmsnarl','obstagoon','perrserker','cursola','sirfetchd','mrrime','runerigus','milcery','alcremie','falinks','pincurchin','snom','frosmoth','stonjourner','eiscue','indeedee','morpeko','cufant','copperajah','dracozolt','arctozolt','dracovish','arctovish','duraludon','dreepy','drakloak','dragapult','zacian','zamazenta','eternatus','kubfu','urshifu','zarude','regieleki','regidrago','glastrier','spectrier','calyrex'] +POKEMON_LIST = [ + "bulbasaur", + "ivysaur", + "venusaur", + "charmander", + "charmeleon", + "charizard", + "squirtle", + "wartortle", + "blastoise", + "caterpie", + "metapod", + "butterfree", + "weedle", + "kakuna", + "beedrill", + "pidgey", + "pidgeotto", + "pidgeot", + "rattata", + "raticate", + "spearow", + "fearow", + "ekans", + "arbok", + "pikachu", + "raichu", + "sandshrew", + "sandslash", + "nidoranf", + "nidorina", + "nidoqueen", + "nidoranm", + "nidorino", + "nidoking", + "clefairy", + "clefable", + "vulpix", + "ninetales", + "jigglypuff", + "wigglytuff", + "zubat", + "golbat", + "oddish", + "gloom", + "vileplume", + "paras", + "parasect", + "venonat", + "venomoth", + "diglett", + "dugtrio", + "meowth", + "persian", + "psyduck", + "golduck", + "mankey", + "primeape", + "growlithe", + "arcanine", + "poliwag", + "poliwhirl", + "poliwrath", + "abra", + "kadabra", + "alakazam", + "machop", + "machoke", + "machamp", + "bellsprout", + "weepinbell", + "victreebel", + "tentacool", + "tentacruel", + "geodude", + "graveler", + "golem", + "ponyta", + "rapidash", + "slowpoke", + "slowbro", + "magnemite", + "magneton", + "farfetchd", + "doduo", + "dodrio", + "seel", + "dewgong", + "grimer", + "muk", + "shellder", + "cloyster", + "gastly", + "haunter", + "gengar", + "onix", + "drowzee", + "hypno", + "krabby", + "kingler", + "voltorb", + "electrode", + "exeggcute", + "exeggutor", + "cubone", + "marowak", + "hitmonlee", + "hitmonchan", + "lickitung", + "koffing", + "weezing", + "rhyhorn", + "rhydon", + "chansey", + "tangela", + "kangaskhan", + "horsea", + "seadra", + "goldeen", + "seaking", + "staryu", + "starmie", + "mrmime", + "scyther", + "jynx", + "electabuzz", + "magmar", + "pinsir", + "tauros", + "magikarp", + "gyarados", + "lapras", + "ditto", + "eevee", + "vaporeon", + "jolteon", + "flareon", + "porygon", + "omanyte", + "omastar", + "kabuto", + "kabutops", + "aerodactyl", + "snorlax", + "articuno", + "zapdos", + "moltres", + "dratini", + "dragonair", + "dragonite", + "mewtwo", + "mew", + "chikorita", + "bayleef", + "meganium", + "cyndaquil", + "quilava", + "typhlosion", + "totodile", + "croconaw", + "feraligatr", + "sentret", + "furret", + "hoothoot", + "noctowl", + "ledyba", + "ledian", + "spinarak", + "ariados", + "crobat", + "chinchou", + "lanturn", + "pichu", + "cleffa", + "igglybuff", + "togepi", + "togetic", + "natu", + "xatu", + "mareep", + "flaaffy", + "ampharos", + "bellossom", + "marill", + "azumarill", + "sudowoodo", + "politoed", + "hoppip", + "skiploom", + "jumpluff", + "aipom", + "sunkern", + "sunflora", + "yanma", + "wooper", + "quagsire", + "espeon", + "umbreon", + "murkrow", + "slowking", + "misdreavus", + "unown", + "wobbuffet", + "girafarig", + "pineco", + "forretress", + "dunsparce", + "gligar", + "steelix", + "snubbull", + "granbull", + "qwilfish", + "scizor", + "shuckle", + "heracross", + "sneasel", + "teddiursa", + "ursaring", + "slugma", + "magcargo", + "swinub", + "piloswine", + "corsola", + "remoraid", + "octillery", + "delibird", + "mantine", + "skarmory", + "houndour", + "houndoom", + "kingdra", + "phanpy", + "donphan", + "porygon2", + "stantler", + "smeargle", + "tyrogue", + "hitmontop", + "smoochum", + "elekid", + "magby", + "miltank", + "blissey", + "raikou", + "entei", + "suicune", + "larvitar", + "pupitar", + "tyranitar", + "lugia", + "ho-oh", + "celebi", + "treecko", + "grovyle", + "sceptile", + "torchic", + "combusken", + "blaziken", + "mudkip", + "marshtomp", + "swampert", + "poochyena", + "mightyena", + "zigzagoon", + "linoone", + "wurmple", + "silcoon", + "beautifly", + "cascoon", + "dustox", + "lotad", + "lombre", + "ludicolo", + "seedot", + "nuzleaf", + "shiftry", + "taillow", + "swellow", + "wingull", + "pelipper", + "ralts", + "kirlia", + "gardevoir", + "surskit", + "masquerain", + "shroomish", + "breloom", + "slakoth", + "vigoroth", + "slaking", + "nincada", + "ninjask", + "shedinja", + "whismur", + "loudred", + "exploud", + "makuhita", + "hariyama", + "azurill", + "nosepass", + "skitty", + "delcatty", + "sableye", + "mawile", + "aron", + "lairon", + "aggron", + "meditite", + "medicham", + "electrike", + "manectric", + "plusle", + "minun", + "volbeat", + "illumise", + "roselia", + "gulpin", + "swalot", + "carvanha", + "sharpedo", + "wailmer", + "wailord", + "numel", + "camerupt", + "torkoal", + "spoink", + "grumpig", + "spinda", + "trapinch", + "vibrava", + "flygon", + "cacnea", + "cacturne", + "swablu", + "altaria", + "zangoose", + "seviper", + "lunatone", + "solrock", + "barboach", + "whiscash", + "corphish", + "crawdaunt", + "baltoy", + "claydol", + "lileep", + "cradily", + "anorith", + "armaldo", + "feebas", + "milotic", + "castform", + "kecleon", + "shuppet", + "banette", + "duskull", + "dusclops", + "tropius", + "chimecho", + "absol", + "wynaut", + "snorunt", + "glalie", + "spheal", + "sealeo", + "walrein", + "clamperl", + "huntail", + "gorebyss", + "relicanth", + "luvdisc", + "bagon", + "shelgon", + "salamence", + "beldum", + "metang", + "metagross", + "regirock", + "regice", + "registeel", + "latias", + "latios", + "kyogre", + "groudon", + "rayquaza", + "jirachi", + "deoxys", + "turtwig", + "grotle", + "torterra", + "chimchar", + "monferno", + "infernape", + "piplup", + "prinplup", + "empoleon", + "starly", + "staravia", + "staraptor", + "bidoof", + "bibarel", + "kricketot", + "kricketune", + "shinx", + "luxio", + "luxray", + "budew", + "roserade", + "cranidos", + "rampardos", + "shieldon", + "bastiodon", + "burmy", + "wormadam", + "mothim", + "combee", + "vespiquen", + "pachirisu", + "buizel", + "floatzel", + "cherubi", + "cherrim", + "shellos", + "gastrodon", + "ambipom", + "drifloon", + "drifblim", + "buneary", + "lopunny", + "mismagius", + "honchkrow", + "glameow", + "purugly", + "chingling", + "stunky", + "skuntank", + "bronzor", + "bronzong", + "bonsly", + "mimejr", + "happiny", + "chatot", + "spiritomb", + "gible", + "gabite", + "garchomp", + "munchlax", + "riolu", + "lucario", + "hippopotas", + "hippowdon", + "skorupi", + "drapion", + "croagunk", + "toxicroak", + "carnivine", + "finneon", + "lumineon", + "mantyke", + "snover", + "abomasnow", + "weavile", + "magnezone", + "lickilicky", + "rhyperior", + "tangrowth", + "electivire", + "magmortar", + "togekiss", + "yanmega", + "leafeon", + "glaceon", + "gliscor", + "mamoswine", + "porygon-z", + "gallade", + "probopass", + "dusknoir", + "froslass", + "rotom", + "uxie", + "mesprit", + "azelf", + "dialga", + "palkia", + "heatran", + "regigigas", + "giratina", + "cresselia", + "phione", + "manaphy", + "darkrai", + "shaymin", + "arceus", + "victini", + "snivy", + "servine", + "serperior", + "tepig", + "pignite", + "emboar", + "oshawott", + "dewott", + "samurott", + "patrat", + "watchog", + "lillipup", + "herdier", + "stoutland", + "purrloin", + "liepard", + "pansage", + "simisage", + "pansear", + "simisear", + "panpour", + "simipour", + "munna", + "musharna", + "pidove", + "tranquill", + "unfezant", + "blitzle", + "zebstrika", + "roggenrola", + "boldore", + "gigalith", + "woobat", + "swoobat", + "drilbur", + "excadrill", + "audino", + "timburr", + "gurdurr", + "conkeldurr", + "tympole", + "palpitoad", + "seismitoad", + "throh", + "sawk", + "sewaddle", + "swadloon", + "leavanny", + "venipede", + "whirlipede", + "scolipede", + "cottonee", + "whimsicott", + "petilil", + "lilligant", + "basculin", + "sandile", + "krokorok", + "krookodile", + "darumaka", + "darmanitan", + "maractus", + "dwebble", + "crustle", + "scraggy", + "scrafty", + "sigilyph", + "yamask", + "cofagrigus", + "tirtouga", + "carracosta", + "archen", + "archeops", + "trubbish", + "garbodor", + "zorua", + "zoroark", + "minccino", + "cinccino", + "gothita", + "gothorita", + "gothitelle", + "solosis", + "duosion", + "reuniclus", + "ducklett", + "swanna", + "vanillite", + "vanillish", + "vanilluxe", + "deerling", + "sawsbuck", + "emolga", + "karrablast", + "escavalier", + "foongus", + "amoonguss", + "frillish", + "jellicent", + "alomomola", + "joltik", + "galvantula", + "ferroseed", + "ferrothorn", + "klink", + "klang", + "klinklang", + "tynamo", + "eelektrik", + "eelektross", + "elgyem", + "beheeyem", + "litwick", + "lampent", + "chandelure", + "axew", + "fraxure", + "haxorus", + "cubchoo", + "beartic", + "cryogonal", + "shelmet", + "accelgor", + "stunfisk", + "mienfoo", + "mienshao", + "druddigon", + "golett", + "golurk", + "pawniard", + "bisharp", + "bouffalant", + "rufflet", + "braviary", + "vullaby", + "mandibuzz", + "heatmor", + "durant", + "deino", + "zweilous", + "hydreigon", + "larvesta", + "volcarona", + "cobalion", + "terrakion", + "virizion", + "tornadus", + "thundurus", + "reshiram", + "zekrom", + "landorus", + "kyurem", + "keldeo", + "meloetta", + "genesect", + "chespin", + "quilladin", + "chesnaught", + "fennekin", + "braixen", + "delphox", + "froakie", + "frogadier", + "greninja", + "bunnelby", + "diggersby", + "fletchling", + "fletchinder", + "talonflame", + "scatterbug", + "spewpa", + "vivillon", + "litleo", + "pyroar", + "flabebe", + "floette", + "florges", + "skiddo", + "gogoat", + "pancham", + "pangoro", + "furfrou", + "espurr", + "meowstic", + "honedge", + "doublade", + "aegislash", + "spritzee", + "aromatisse", + "swirlix", + "slurpuff", + "inkay", + "malamar", + "binacle", + "barbaracle", + "skrelp", + "dragalge", + "clauncher", + "clawitzer", + "helioptile", + "heliolisk", + "tyrunt", + "tyrantrum", + "amaura", + "aurorus", + "sylveon", + "hawlucha", + "dedenne", + "carbink", + "goomy", + "sliggoo", + "goodra", + "klefki", + "phantump", + "trevenant", + "pumpkaboo", + "gourgeist", + "bergmite", + "avalugg", + "noibat", + "noivern", + "xerneas", + "yveltal", + "zygarde", + "diancie", + "hoopa", + "volcanion", + "rowlet", + "dartrix", + "decidueye", + "litten", + "torracat", + "incineroar", + "popplio", + "brionne", + "primarina", + "pikipek", + "trumbeak", + "toucannon", + "yungoos", + "gumshoos", + "grubbin", + "charjabug", + "vikavolt", + "crabrawler", + "crabominable", + "oricorio", + "cutiefly", + "ribombee", + "rockruff", + "lycanroc", + "wishiwashi", + "mareanie", + "toxapex", + "mudbray", + "mudsdale", + "dewpider", + "araquanid", + "fomantis", + "lurantis", + "morelull", + "shiinotic", + "salandit", + "salazzle", + "stufful", + "bewear", + "bounsweet", + "steenee", + "tsareena", + "comfey", + "oranguru", + "passimian", + "wimpod", + "golisopod", + "sandygast", + "palossand", + "pyukumuku", + "typenull", + "silvally", + "minior", + "komala", + "turtonator", + "togedemaru", + "mimikyu", + "bruxish", + "drampa", + "dhelmise", + "jangmo-o", + "hakamo-o", + "kommo-o", + "tapukoko", + "tapulele", + "tapubulu", + "tapufini", + "cosmog", + "cosmoem", + "solgaleo", + "lunala", + "nihilego", + "buzzwole", + "pheromosa", + "xurkitree", + "celesteela", + "kartana", + "guzzlord", + "necrozma", + "magearna", + "marshadow", + "poipole", + "naganadel", + "stakataka", + "blacephalon", + "zeraora", + "meltan", + "melmetal", + "grookey", + "thwackey", + "rillaboom", + "scorbunny", + "raboot", + "cinderace", + "sobble", + "drizzile", + "inteleon", + "skwovet", + "greedent", + "rookidee", + "corvisquire", + "corviknight", + "blipbug", + "dottler", + "orbeetle", + "nickit", + "thievul", + "gossifleur", + "eldegoss", + "wooloo", + "dubwool", + "chewtle", + "drednaw", + "yamper", + "boltund", + "rolycoly", + "carkol", + "coalossal", + "applin", + "flapple", + "appletun", + "silicobra", + "sandaconda", + "cramorant", + "arrokuda", + "barraskewda", + "toxel", + "toxtricity", + "sizzlipede", + "centiskorch", + "clobbopus", + "grapploct", + "sinistea", + "polteageist", + "hatenna", + "hattrem", + "hatterene", + "impidimp", + "morgrem", + "grimmsnarl", + "obstagoon", + "perrserker", + "cursola", + "sirfetchd", + "mrrime", + "runerigus", + "milcery", + "alcremie", + "falinks", + "pincurchin", + "snom", + "frosmoth", + "stonjourner", + "eiscue", + "indeedee", + "morpeko", + "cufant", + "copperajah", + "dracozolt", + "arctozolt", + "dracovish", + "arctovish", + "duraludon", + "dreepy", + "drakloak", + "dragapult", + "zacian", + "zamazenta", + "eternatus", + "kubfu", + "urshifu", + "zarude", + "regieleki", + "regidrago", + "glastrier", + "spectrier", + "calyrex", +] + -# Basic full refresh stream class PokeapiStream(HttpStream): - url_base = 'https://pokeapi.co/api/v2/' + url_base = "https://pokeapi.co/api/v2/" def __init__(self, pokemon_name: str, **kwargs): super().__init__(**kwargs) self.pokemon_name = pokemon_name def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: # The api requires that we include the Pokemon name as a query param so we do that in this method return {"pokemon_name": self.pokemon_name} def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> Iterable[Mapping]: # The response is a simple JSON whose schema matches our stream's schema exactly, # so we just return a list containing the response @@ -67,18 +988,17 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, class Pokemon(PokeapiStream): def path(self, **kwargs) -> str: pokemon_name = self.pokemon_name - return f'pokemon/{pokemon_name}' + return f"pokemon/{pokemon_name}" + # Source class SourcePokeapi(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - input_pokemon = config['name'] + input_pokemon = config["name"] if input_pokemon not in POKEMON_LIST: return False, f"Input Pokemon {input_pokemon} is invalid. Please check your spelling our input a valid Pokemon." else: return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = {"name": config["name"]} return [Pokemon(name=config["name"])] diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json index 8d2b88858e65a..a0d624df6d749 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json @@ -15,4 +15,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py index 2854c429a7db5..a3f12086dd5e7 100644 --- a/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py @@ -1,3 +1,26 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + """ MIT License From a641be0d07b9ba607f88ed92fcdf27ceb32f0dc0 Mon Sep 17 00:00:00 2001 From: Abhi Vaidyanatha Date: Fri, 30 Apr 2021 18:34:14 -0700 Subject: [PATCH 4/8] Move Pokemon list constant to its own file --- .../source_pokeapi/pokemon_list.py | 901 +++++++++++++++++ .../source-pokeapi/source_pokeapi/source.py | 902 +----------------- 2 files changed, 902 insertions(+), 901 deletions(-) create mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py new file mode 100644 index 0000000000000..970095e4bde9d --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py @@ -0,0 +1,901 @@ +# Store large list of all known Pokemon. +POKEMON_LIST = [ + "bulbasaur", + "ivysaur", + "venusaur", + "charmander", + "charmeleon", + "charizard", + "squirtle", + "wartortle", + "blastoise", + "caterpie", + "metapod", + "butterfree", + "weedle", + "kakuna", + "beedrill", + "pidgey", + "pidgeotto", + "pidgeot", + "rattata", + "raticate", + "spearow", + "fearow", + "ekans", + "arbok", + "pikachu", + "raichu", + "sandshrew", + "sandslash", + "nidoranf", + "nidorina", + "nidoqueen", + "nidoranm", + "nidorino", + "nidoking", + "clefairy", + "clefable", + "vulpix", + "ninetales", + "jigglypuff", + "wigglytuff", + "zubat", + "golbat", + "oddish", + "gloom", + "vileplume", + "paras", + "parasect", + "venonat", + "venomoth", + "diglett", + "dugtrio", + "meowth", + "persian", + "psyduck", + "golduck", + "mankey", + "primeape", + "growlithe", + "arcanine", + "poliwag", + "poliwhirl", + "poliwrath", + "abra", + "kadabra", + "alakazam", + "machop", + "machoke", + "machamp", + "bellsprout", + "weepinbell", + "victreebel", + "tentacool", + "tentacruel", + "geodude", + "graveler", + "golem", + "ponyta", + "rapidash", + "slowpoke", + "slowbro", + "magnemite", + "magneton", + "farfetchd", + "doduo", + "dodrio", + "seel", + "dewgong", + "grimer", + "muk", + "shellder", + "cloyster", + "gastly", + "haunter", + "gengar", + "onix", + "drowzee", + "hypno", + "krabby", + "kingler", + "voltorb", + "electrode", + "exeggcute", + "exeggutor", + "cubone", + "marowak", + "hitmonlee", + "hitmonchan", + "lickitung", + "koffing", + "weezing", + "rhyhorn", + "rhydon", + "chansey", + "tangela", + "kangaskhan", + "horsea", + "seadra", + "goldeen", + "seaking", + "staryu", + "starmie", + "mrmime", + "scyther", + "jynx", + "electabuzz", + "magmar", + "pinsir", + "tauros", + "magikarp", + "gyarados", + "lapras", + "ditto", + "eevee", + "vaporeon", + "jolteon", + "flareon", + "porygon", + "omanyte", + "omastar", + "kabuto", + "kabutops", + "aerodactyl", + "snorlax", + "articuno", + "zapdos", + "moltres", + "dratini", + "dragonair", + "dragonite", + "mewtwo", + "mew", + "chikorita", + "bayleef", + "meganium", + "cyndaquil", + "quilava", + "typhlosion", + "totodile", + "croconaw", + "feraligatr", + "sentret", + "furret", + "hoothoot", + "noctowl", + "ledyba", + "ledian", + "spinarak", + "ariados", + "crobat", + "chinchou", + "lanturn", + "pichu", + "cleffa", + "igglybuff", + "togepi", + "togetic", + "natu", + "xatu", + "mareep", + "flaaffy", + "ampharos", + "bellossom", + "marill", + "azumarill", + "sudowoodo", + "politoed", + "hoppip", + "skiploom", + "jumpluff", + "aipom", + "sunkern", + "sunflora", + "yanma", + "wooper", + "quagsire", + "espeon", + "umbreon", + "murkrow", + "slowking", + "misdreavus", + "unown", + "wobbuffet", + "girafarig", + "pineco", + "forretress", + "dunsparce", + "gligar", + "steelix", + "snubbull", + "granbull", + "qwilfish", + "scizor", + "shuckle", + "heracross", + "sneasel", + "teddiursa", + "ursaring", + "slugma", + "magcargo", + "swinub", + "piloswine", + "corsola", + "remoraid", + "octillery", + "delibird", + "mantine", + "skarmory", + "houndour", + "houndoom", + "kingdra", + "phanpy", + "donphan", + "porygon2", + "stantler", + "smeargle", + "tyrogue", + "hitmontop", + "smoochum", + "elekid", + "magby", + "miltank", + "blissey", + "raikou", + "entei", + "suicune", + "larvitar", + "pupitar", + "tyranitar", + "lugia", + "ho-oh", + "celebi", + "treecko", + "grovyle", + "sceptile", + "torchic", + "combusken", + "blaziken", + "mudkip", + "marshtomp", + "swampert", + "poochyena", + "mightyena", + "zigzagoon", + "linoone", + "wurmple", + "silcoon", + "beautifly", + "cascoon", + "dustox", + "lotad", + "lombre", + "ludicolo", + "seedot", + "nuzleaf", + "shiftry", + "taillow", + "swellow", + "wingull", + "pelipper", + "ralts", + "kirlia", + "gardevoir", + "surskit", + "masquerain", + "shroomish", + "breloom", + "slakoth", + "vigoroth", + "slaking", + "nincada", + "ninjask", + "shedinja", + "whismur", + "loudred", + "exploud", + "makuhita", + "hariyama", + "azurill", + "nosepass", + "skitty", + "delcatty", + "sableye", + "mawile", + "aron", + "lairon", + "aggron", + "meditite", + "medicham", + "electrike", + "manectric", + "plusle", + "minun", + "volbeat", + "illumise", + "roselia", + "gulpin", + "swalot", + "carvanha", + "sharpedo", + "wailmer", + "wailord", + "numel", + "camerupt", + "torkoal", + "spoink", + "grumpig", + "spinda", + "trapinch", + "vibrava", + "flygon", + "cacnea", + "cacturne", + "swablu", + "altaria", + "zangoose", + "seviper", + "lunatone", + "solrock", + "barboach", + "whiscash", + "corphish", + "crawdaunt", + "baltoy", + "claydol", + "lileep", + "cradily", + "anorith", + "armaldo", + "feebas", + "milotic", + "castform", + "kecleon", + "shuppet", + "banette", + "duskull", + "dusclops", + "tropius", + "chimecho", + "absol", + "wynaut", + "snorunt", + "glalie", + "spheal", + "sealeo", + "walrein", + "clamperl", + "huntail", + "gorebyss", + "relicanth", + "luvdisc", + "bagon", + "shelgon", + "salamence", + "beldum", + "metang", + "metagross", + "regirock", + "regice", + "registeel", + "latias", + "latios", + "kyogre", + "groudon", + "rayquaza", + "jirachi", + "deoxys", + "turtwig", + "grotle", + "torterra", + "chimchar", + "monferno", + "infernape", + "piplup", + "prinplup", + "empoleon", + "starly", + "staravia", + "staraptor", + "bidoof", + "bibarel", + "kricketot", + "kricketune", + "shinx", + "luxio", + "luxray", + "budew", + "roserade", + "cranidos", + "rampardos", + "shieldon", + "bastiodon", + "burmy", + "wormadam", + "mothim", + "combee", + "vespiquen", + "pachirisu", + "buizel", + "floatzel", + "cherubi", + "cherrim", + "shellos", + "gastrodon", + "ambipom", + "drifloon", + "drifblim", + "buneary", + "lopunny", + "mismagius", + "honchkrow", + "glameow", + "purugly", + "chingling", + "stunky", + "skuntank", + "bronzor", + "bronzong", + "bonsly", + "mimejr", + "happiny", + "chatot", + "spiritomb", + "gible", + "gabite", + "garchomp", + "munchlax", + "riolu", + "lucario", + "hippopotas", + "hippowdon", + "skorupi", + "drapion", + "croagunk", + "toxicroak", + "carnivine", + "finneon", + "lumineon", + "mantyke", + "snover", + "abomasnow", + "weavile", + "magnezone", + "lickilicky", + "rhyperior", + "tangrowth", + "electivire", + "magmortar", + "togekiss", + "yanmega", + "leafeon", + "glaceon", + "gliscor", + "mamoswine", + "porygon-z", + "gallade", + "probopass", + "dusknoir", + "froslass", + "rotom", + "uxie", + "mesprit", + "azelf", + "dialga", + "palkia", + "heatran", + "regigigas", + "giratina", + "cresselia", + "phione", + "manaphy", + "darkrai", + "shaymin", + "arceus", + "victini", + "snivy", + "servine", + "serperior", + "tepig", + "pignite", + "emboar", + "oshawott", + "dewott", + "samurott", + "patrat", + "watchog", + "lillipup", + "herdier", + "stoutland", + "purrloin", + "liepard", + "pansage", + "simisage", + "pansear", + "simisear", + "panpour", + "simipour", + "munna", + "musharna", + "pidove", + "tranquill", + "unfezant", + "blitzle", + "zebstrika", + "roggenrola", + "boldore", + "gigalith", + "woobat", + "swoobat", + "drilbur", + "excadrill", + "audino", + "timburr", + "gurdurr", + "conkeldurr", + "tympole", + "palpitoad", + "seismitoad", + "throh", + "sawk", + "sewaddle", + "swadloon", + "leavanny", + "venipede", + "whirlipede", + "scolipede", + "cottonee", + "whimsicott", + "petilil", + "lilligant", + "basculin", + "sandile", + "krokorok", + "krookodile", + "darumaka", + "darmanitan", + "maractus", + "dwebble", + "crustle", + "scraggy", + "scrafty", + "sigilyph", + "yamask", + "cofagrigus", + "tirtouga", + "carracosta", + "archen", + "archeops", + "trubbish", + "garbodor", + "zorua", + "zoroark", + "minccino", + "cinccino", + "gothita", + "gothorita", + "gothitelle", + "solosis", + "duosion", + "reuniclus", + "ducklett", + "swanna", + "vanillite", + "vanillish", + "vanilluxe", + "deerling", + "sawsbuck", + "emolga", + "karrablast", + "escavalier", + "foongus", + "amoonguss", + "frillish", + "jellicent", + "alomomola", + "joltik", + "galvantula", + "ferroseed", + "ferrothorn", + "klink", + "klang", + "klinklang", + "tynamo", + "eelektrik", + "eelektross", + "elgyem", + "beheeyem", + "litwick", + "lampent", + "chandelure", + "axew", + "fraxure", + "haxorus", + "cubchoo", + "beartic", + "cryogonal", + "shelmet", + "accelgor", + "stunfisk", + "mienfoo", + "mienshao", + "druddigon", + "golett", + "golurk", + "pawniard", + "bisharp", + "bouffalant", + "rufflet", + "braviary", + "vullaby", + "mandibuzz", + "heatmor", + "durant", + "deino", + "zweilous", + "hydreigon", + "larvesta", + "volcarona", + "cobalion", + "terrakion", + "virizion", + "tornadus", + "thundurus", + "reshiram", + "zekrom", + "landorus", + "kyurem", + "keldeo", + "meloetta", + "genesect", + "chespin", + "quilladin", + "chesnaught", + "fennekin", + "braixen", + "delphox", + "froakie", + "frogadier", + "greninja", + "bunnelby", + "diggersby", + "fletchling", + "fletchinder", + "talonflame", + "scatterbug", + "spewpa", + "vivillon", + "litleo", + "pyroar", + "flabebe", + "floette", + "florges", + "skiddo", + "gogoat", + "pancham", + "pangoro", + "furfrou", + "espurr", + "meowstic", + "honedge", + "doublade", + "aegislash", + "spritzee", + "aromatisse", + "swirlix", + "slurpuff", + "inkay", + "malamar", + "binacle", + "barbaracle", + "skrelp", + "dragalge", + "clauncher", + "clawitzer", + "helioptile", + "heliolisk", + "tyrunt", + "tyrantrum", + "amaura", + "aurorus", + "sylveon", + "hawlucha", + "dedenne", + "carbink", + "goomy", + "sliggoo", + "goodra", + "klefki", + "phantump", + "trevenant", + "pumpkaboo", + "gourgeist", + "bergmite", + "avalugg", + "noibat", + "noivern", + "xerneas", + "yveltal", + "zygarde", + "diancie", + "hoopa", + "volcanion", + "rowlet", + "dartrix", + "decidueye", + "litten", + "torracat", + "incineroar", + "popplio", + "brionne", + "primarina", + "pikipek", + "trumbeak", + "toucannon", + "yungoos", + "gumshoos", + "grubbin", + "charjabug", + "vikavolt", + "crabrawler", + "crabominable", + "oricorio", + "cutiefly", + "ribombee", + "rockruff", + "lycanroc", + "wishiwashi", + "mareanie", + "toxapex", + "mudbray", + "mudsdale", + "dewpider", + "araquanid", + "fomantis", + "lurantis", + "morelull", + "shiinotic", + "salandit", + "salazzle", + "stufful", + "bewear", + "bounsweet", + "steenee", + "tsareena", + "comfey", + "oranguru", + "passimian", + "wimpod", + "golisopod", + "sandygast", + "palossand", + "pyukumuku", + "typenull", + "silvally", + "minior", + "komala", + "turtonator", + "togedemaru", + "mimikyu", + "bruxish", + "drampa", + "dhelmise", + "jangmo-o", + "hakamo-o", + "kommo-o", + "tapukoko", + "tapulele", + "tapubulu", + "tapufini", + "cosmog", + "cosmoem", + "solgaleo", + "lunala", + "nihilego", + "buzzwole", + "pheromosa", + "xurkitree", + "celesteela", + "kartana", + "guzzlord", + "necrozma", + "magearna", + "marshadow", + "poipole", + "naganadel", + "stakataka", + "blacephalon", + "zeraora", + "meltan", + "melmetal", + "grookey", + "thwackey", + "rillaboom", + "scorbunny", + "raboot", + "cinderace", + "sobble", + "drizzile", + "inteleon", + "skwovet", + "greedent", + "rookidee", + "corvisquire", + "corviknight", + "blipbug", + "dottler", + "orbeetle", + "nickit", + "thievul", + "gossifleur", + "eldegoss", + "wooloo", + "dubwool", + "chewtle", + "drednaw", + "yamper", + "boltund", + "rolycoly", + "carkol", + "coalossal", + "applin", + "flapple", + "appletun", + "silicobra", + "sandaconda", + "cramorant", + "arrokuda", + "barraskewda", + "toxel", + "toxtricity", + "sizzlipede", + "centiskorch", + "clobbopus", + "grapploct", + "sinistea", + "polteageist", + "hatenna", + "hattrem", + "hatterene", + "impidimp", + "morgrem", + "grimmsnarl", + "obstagoon", + "perrserker", + "cursola", + "sirfetchd", + "mrrime", + "runerigus", + "milcery", + "alcremie", + "falinks", + "pincurchin", + "snom", + "frosmoth", + "stonjourner", + "eiscue", + "indeedee", + "morpeko", + "cufant", + "copperajah", + "dracozolt", + "arctozolt", + "dracovish", + "arctovish", + "duraludon", + "dreepy", + "drakloak", + "dragapult", + "zacian", + "zamazenta", + "eternatus", + "kubfu", + "urshifu", + "zarude", + "regieleki", + "regidrago", + "glastrier", + "spectrier", + "calyrex", +] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 4efcd0f8d72de..922c347c90620 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -49,907 +49,7 @@ import requests from base_python import AbstractSource, HttpStream, Stream - -POKEMON_LIST = [ - "bulbasaur", - "ivysaur", - "venusaur", - "charmander", - "charmeleon", - "charizard", - "squirtle", - "wartortle", - "blastoise", - "caterpie", - "metapod", - "butterfree", - "weedle", - "kakuna", - "beedrill", - "pidgey", - "pidgeotto", - "pidgeot", - "rattata", - "raticate", - "spearow", - "fearow", - "ekans", - "arbok", - "pikachu", - "raichu", - "sandshrew", - "sandslash", - "nidoranf", - "nidorina", - "nidoqueen", - "nidoranm", - "nidorino", - "nidoking", - "clefairy", - "clefable", - "vulpix", - "ninetales", - "jigglypuff", - "wigglytuff", - "zubat", - "golbat", - "oddish", - "gloom", - "vileplume", - "paras", - "parasect", - "venonat", - "venomoth", - "diglett", - "dugtrio", - "meowth", - "persian", - "psyduck", - "golduck", - "mankey", - "primeape", - "growlithe", - "arcanine", - "poliwag", - "poliwhirl", - "poliwrath", - "abra", - "kadabra", - "alakazam", - "machop", - "machoke", - "machamp", - "bellsprout", - "weepinbell", - "victreebel", - "tentacool", - "tentacruel", - "geodude", - "graveler", - "golem", - "ponyta", - "rapidash", - "slowpoke", - "slowbro", - "magnemite", - "magneton", - "farfetchd", - "doduo", - "dodrio", - "seel", - "dewgong", - "grimer", - "muk", - "shellder", - "cloyster", - "gastly", - "haunter", - "gengar", - "onix", - "drowzee", - "hypno", - "krabby", - "kingler", - "voltorb", - "electrode", - "exeggcute", - "exeggutor", - "cubone", - "marowak", - "hitmonlee", - "hitmonchan", - "lickitung", - "koffing", - "weezing", - "rhyhorn", - "rhydon", - "chansey", - "tangela", - "kangaskhan", - "horsea", - "seadra", - "goldeen", - "seaking", - "staryu", - "starmie", - "mrmime", - "scyther", - "jynx", - "electabuzz", - "magmar", - "pinsir", - "tauros", - "magikarp", - "gyarados", - "lapras", - "ditto", - "eevee", - "vaporeon", - "jolteon", - "flareon", - "porygon", - "omanyte", - "omastar", - "kabuto", - "kabutops", - "aerodactyl", - "snorlax", - "articuno", - "zapdos", - "moltres", - "dratini", - "dragonair", - "dragonite", - "mewtwo", - "mew", - "chikorita", - "bayleef", - "meganium", - "cyndaquil", - "quilava", - "typhlosion", - "totodile", - "croconaw", - "feraligatr", - "sentret", - "furret", - "hoothoot", - "noctowl", - "ledyba", - "ledian", - "spinarak", - "ariados", - "crobat", - "chinchou", - "lanturn", - "pichu", - "cleffa", - "igglybuff", - "togepi", - "togetic", - "natu", - "xatu", - "mareep", - "flaaffy", - "ampharos", - "bellossom", - "marill", - "azumarill", - "sudowoodo", - "politoed", - "hoppip", - "skiploom", - "jumpluff", - "aipom", - "sunkern", - "sunflora", - "yanma", - "wooper", - "quagsire", - "espeon", - "umbreon", - "murkrow", - "slowking", - "misdreavus", - "unown", - "wobbuffet", - "girafarig", - "pineco", - "forretress", - "dunsparce", - "gligar", - "steelix", - "snubbull", - "granbull", - "qwilfish", - "scizor", - "shuckle", - "heracross", - "sneasel", - "teddiursa", - "ursaring", - "slugma", - "magcargo", - "swinub", - "piloswine", - "corsola", - "remoraid", - "octillery", - "delibird", - "mantine", - "skarmory", - "houndour", - "houndoom", - "kingdra", - "phanpy", - "donphan", - "porygon2", - "stantler", - "smeargle", - "tyrogue", - "hitmontop", - "smoochum", - "elekid", - "magby", - "miltank", - "blissey", - "raikou", - "entei", - "suicune", - "larvitar", - "pupitar", - "tyranitar", - "lugia", - "ho-oh", - "celebi", - "treecko", - "grovyle", - "sceptile", - "torchic", - "combusken", - "blaziken", - "mudkip", - "marshtomp", - "swampert", - "poochyena", - "mightyena", - "zigzagoon", - "linoone", - "wurmple", - "silcoon", - "beautifly", - "cascoon", - "dustox", - "lotad", - "lombre", - "ludicolo", - "seedot", - "nuzleaf", - "shiftry", - "taillow", - "swellow", - "wingull", - "pelipper", - "ralts", - "kirlia", - "gardevoir", - "surskit", - "masquerain", - "shroomish", - "breloom", - "slakoth", - "vigoroth", - "slaking", - "nincada", - "ninjask", - "shedinja", - "whismur", - "loudred", - "exploud", - "makuhita", - "hariyama", - "azurill", - "nosepass", - "skitty", - "delcatty", - "sableye", - "mawile", - "aron", - "lairon", - "aggron", - "meditite", - "medicham", - "electrike", - "manectric", - "plusle", - "minun", - "volbeat", - "illumise", - "roselia", - "gulpin", - "swalot", - "carvanha", - "sharpedo", - "wailmer", - "wailord", - "numel", - "camerupt", - "torkoal", - "spoink", - "grumpig", - "spinda", - "trapinch", - "vibrava", - "flygon", - "cacnea", - "cacturne", - "swablu", - "altaria", - "zangoose", - "seviper", - "lunatone", - "solrock", - "barboach", - "whiscash", - "corphish", - "crawdaunt", - "baltoy", - "claydol", - "lileep", - "cradily", - "anorith", - "armaldo", - "feebas", - "milotic", - "castform", - "kecleon", - "shuppet", - "banette", - "duskull", - "dusclops", - "tropius", - "chimecho", - "absol", - "wynaut", - "snorunt", - "glalie", - "spheal", - "sealeo", - "walrein", - "clamperl", - "huntail", - "gorebyss", - "relicanth", - "luvdisc", - "bagon", - "shelgon", - "salamence", - "beldum", - "metang", - "metagross", - "regirock", - "regice", - "registeel", - "latias", - "latios", - "kyogre", - "groudon", - "rayquaza", - "jirachi", - "deoxys", - "turtwig", - "grotle", - "torterra", - "chimchar", - "monferno", - "infernape", - "piplup", - "prinplup", - "empoleon", - "starly", - "staravia", - "staraptor", - "bidoof", - "bibarel", - "kricketot", - "kricketune", - "shinx", - "luxio", - "luxray", - "budew", - "roserade", - "cranidos", - "rampardos", - "shieldon", - "bastiodon", - "burmy", - "wormadam", - "mothim", - "combee", - "vespiquen", - "pachirisu", - "buizel", - "floatzel", - "cherubi", - "cherrim", - "shellos", - "gastrodon", - "ambipom", - "drifloon", - "drifblim", - "buneary", - "lopunny", - "mismagius", - "honchkrow", - "glameow", - "purugly", - "chingling", - "stunky", - "skuntank", - "bronzor", - "bronzong", - "bonsly", - "mimejr", - "happiny", - "chatot", - "spiritomb", - "gible", - "gabite", - "garchomp", - "munchlax", - "riolu", - "lucario", - "hippopotas", - "hippowdon", - "skorupi", - "drapion", - "croagunk", - "toxicroak", - "carnivine", - "finneon", - "lumineon", - "mantyke", - "snover", - "abomasnow", - "weavile", - "magnezone", - "lickilicky", - "rhyperior", - "tangrowth", - "electivire", - "magmortar", - "togekiss", - "yanmega", - "leafeon", - "glaceon", - "gliscor", - "mamoswine", - "porygon-z", - "gallade", - "probopass", - "dusknoir", - "froslass", - "rotom", - "uxie", - "mesprit", - "azelf", - "dialga", - "palkia", - "heatran", - "regigigas", - "giratina", - "cresselia", - "phione", - "manaphy", - "darkrai", - "shaymin", - "arceus", - "victini", - "snivy", - "servine", - "serperior", - "tepig", - "pignite", - "emboar", - "oshawott", - "dewott", - "samurott", - "patrat", - "watchog", - "lillipup", - "herdier", - "stoutland", - "purrloin", - "liepard", - "pansage", - "simisage", - "pansear", - "simisear", - "panpour", - "simipour", - "munna", - "musharna", - "pidove", - "tranquill", - "unfezant", - "blitzle", - "zebstrika", - "roggenrola", - "boldore", - "gigalith", - "woobat", - "swoobat", - "drilbur", - "excadrill", - "audino", - "timburr", - "gurdurr", - "conkeldurr", - "tympole", - "palpitoad", - "seismitoad", - "throh", - "sawk", - "sewaddle", - "swadloon", - "leavanny", - "venipede", - "whirlipede", - "scolipede", - "cottonee", - "whimsicott", - "petilil", - "lilligant", - "basculin", - "sandile", - "krokorok", - "krookodile", - "darumaka", - "darmanitan", - "maractus", - "dwebble", - "crustle", - "scraggy", - "scrafty", - "sigilyph", - "yamask", - "cofagrigus", - "tirtouga", - "carracosta", - "archen", - "archeops", - "trubbish", - "garbodor", - "zorua", - "zoroark", - "minccino", - "cinccino", - "gothita", - "gothorita", - "gothitelle", - "solosis", - "duosion", - "reuniclus", - "ducklett", - "swanna", - "vanillite", - "vanillish", - "vanilluxe", - "deerling", - "sawsbuck", - "emolga", - "karrablast", - "escavalier", - "foongus", - "amoonguss", - "frillish", - "jellicent", - "alomomola", - "joltik", - "galvantula", - "ferroseed", - "ferrothorn", - "klink", - "klang", - "klinklang", - "tynamo", - "eelektrik", - "eelektross", - "elgyem", - "beheeyem", - "litwick", - "lampent", - "chandelure", - "axew", - "fraxure", - "haxorus", - "cubchoo", - "beartic", - "cryogonal", - "shelmet", - "accelgor", - "stunfisk", - "mienfoo", - "mienshao", - "druddigon", - "golett", - "golurk", - "pawniard", - "bisharp", - "bouffalant", - "rufflet", - "braviary", - "vullaby", - "mandibuzz", - "heatmor", - "durant", - "deino", - "zweilous", - "hydreigon", - "larvesta", - "volcarona", - "cobalion", - "terrakion", - "virizion", - "tornadus", - "thundurus", - "reshiram", - "zekrom", - "landorus", - "kyurem", - "keldeo", - "meloetta", - "genesect", - "chespin", - "quilladin", - "chesnaught", - "fennekin", - "braixen", - "delphox", - "froakie", - "frogadier", - "greninja", - "bunnelby", - "diggersby", - "fletchling", - "fletchinder", - "talonflame", - "scatterbug", - "spewpa", - "vivillon", - "litleo", - "pyroar", - "flabebe", - "floette", - "florges", - "skiddo", - "gogoat", - "pancham", - "pangoro", - "furfrou", - "espurr", - "meowstic", - "honedge", - "doublade", - "aegislash", - "spritzee", - "aromatisse", - "swirlix", - "slurpuff", - "inkay", - "malamar", - "binacle", - "barbaracle", - "skrelp", - "dragalge", - "clauncher", - "clawitzer", - "helioptile", - "heliolisk", - "tyrunt", - "tyrantrum", - "amaura", - "aurorus", - "sylveon", - "hawlucha", - "dedenne", - "carbink", - "goomy", - "sliggoo", - "goodra", - "klefki", - "phantump", - "trevenant", - "pumpkaboo", - "gourgeist", - "bergmite", - "avalugg", - "noibat", - "noivern", - "xerneas", - "yveltal", - "zygarde", - "diancie", - "hoopa", - "volcanion", - "rowlet", - "dartrix", - "decidueye", - "litten", - "torracat", - "incineroar", - "popplio", - "brionne", - "primarina", - "pikipek", - "trumbeak", - "toucannon", - "yungoos", - "gumshoos", - "grubbin", - "charjabug", - "vikavolt", - "crabrawler", - "crabominable", - "oricorio", - "cutiefly", - "ribombee", - "rockruff", - "lycanroc", - "wishiwashi", - "mareanie", - "toxapex", - "mudbray", - "mudsdale", - "dewpider", - "araquanid", - "fomantis", - "lurantis", - "morelull", - "shiinotic", - "salandit", - "salazzle", - "stufful", - "bewear", - "bounsweet", - "steenee", - "tsareena", - "comfey", - "oranguru", - "passimian", - "wimpod", - "golisopod", - "sandygast", - "palossand", - "pyukumuku", - "typenull", - "silvally", - "minior", - "komala", - "turtonator", - "togedemaru", - "mimikyu", - "bruxish", - "drampa", - "dhelmise", - "jangmo-o", - "hakamo-o", - "kommo-o", - "tapukoko", - "tapulele", - "tapubulu", - "tapufini", - "cosmog", - "cosmoem", - "solgaleo", - "lunala", - "nihilego", - "buzzwole", - "pheromosa", - "xurkitree", - "celesteela", - "kartana", - "guzzlord", - "necrozma", - "magearna", - "marshadow", - "poipole", - "naganadel", - "stakataka", - "blacephalon", - "zeraora", - "meltan", - "melmetal", - "grookey", - "thwackey", - "rillaboom", - "scorbunny", - "raboot", - "cinderace", - "sobble", - "drizzile", - "inteleon", - "skwovet", - "greedent", - "rookidee", - "corvisquire", - "corviknight", - "blipbug", - "dottler", - "orbeetle", - "nickit", - "thievul", - "gossifleur", - "eldegoss", - "wooloo", - "dubwool", - "chewtle", - "drednaw", - "yamper", - "boltund", - "rolycoly", - "carkol", - "coalossal", - "applin", - "flapple", - "appletun", - "silicobra", - "sandaconda", - "cramorant", - "arrokuda", - "barraskewda", - "toxel", - "toxtricity", - "sizzlipede", - "centiskorch", - "clobbopus", - "grapploct", - "sinistea", - "polteageist", - "hatenna", - "hattrem", - "hatterene", - "impidimp", - "morgrem", - "grimmsnarl", - "obstagoon", - "perrserker", - "cursola", - "sirfetchd", - "mrrime", - "runerigus", - "milcery", - "alcremie", - "falinks", - "pincurchin", - "snom", - "frosmoth", - "stonjourner", - "eiscue", - "indeedee", - "morpeko", - "cufant", - "copperajah", - "dracozolt", - "arctozolt", - "dracovish", - "arctovish", - "duraludon", - "dreepy", - "drakloak", - "dragapult", - "zacian", - "zamazenta", - "eternatus", - "kubfu", - "urshifu", - "zarude", - "regieleki", - "regidrago", - "glastrier", - "spectrier", - "calyrex", -] +from pokemon_list import POKEMON_LIST class PokeapiStream(HttpStream): From fbe16d382ebf727181be54bd81698a5032fee053 Mon Sep 17 00:00:00 2001 From: Abhi Vaidyanatha Date: Fri, 30 Apr 2021 18:52:43 -0700 Subject: [PATCH 5/8] Fix gradle formatting issues --- .../source_pokeapi/pokemon_list.py | 30 ++++++++++++++++++- .../source-pokeapi/source_pokeapi/source.py | 24 --------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py index 970095e4bde9d..19f905e2b52e3 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py @@ -1,3 +1,28 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import pokemon_list + # Store large list of all known Pokemon. POKEMON_LIST = [ "bulbasaur", @@ -898,4 +923,7 @@ "glastrier", "spectrier", "calyrex", -] \ No newline at end of file +] + +# Use the import so flakeCheck doesn't get mad at us. +no_op_list = pokemon_list.POKEMON_LIST diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 922c347c90620..51029d4f473d4 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -21,30 +21,6 @@ # SOFTWARE. -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests From bb16125ecea68ecaf2d75fe9ff39141c711bd735 Mon Sep 17 00:00:00 2001 From: Abhi Vaidyanatha Date: Tue, 4 May 2021 12:39:47 -0700 Subject: [PATCH 6/8] Prepare connector for publishing. --- .../6371b14b-bc68-4236-bfbd-468e8df8e968.json | 7 +++++ .../resources/seed/source_definitions.yaml | 5 +++ docs/SUMMARY.md | 1 + docs/integrations/sources/pokeapi.md | 31 +++++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json create mode 100644 docs/integrations/sources/pokeapi.md diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json new file mode 100644 index 0000000000000..1e82f522e6a5f --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "6371b14b-bc68-4236-bfbd-468e8df8e968", + "name": "PokeAPI", + "dockerRepository": "airbyte/source-pokeapi", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-pokeapi" +} \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5a6b2d368e249..046c48b75ccb1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -264,3 +264,8 @@ dockerRepository: airbyte/source-iterable dockerImageTag: 0.1.0 documentationUrl: https://hub.docker.com/r/airbyte/source-iterable +- sourceDefinitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 + name: PokeAPI + dockerRepository: airbyte/source-pokeapi + dockerImageTag: 0.1.0 + documentationUrl: https://hub.docker.com/r/airbyte/source-pokeapi diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index f6dfbca94cc85..f0b50ecf5243a 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -62,6 +62,7 @@ * [MySQL](integrations/sources/mysql.md) * [Oracle DB](integrations/sources/oracle.md) * [Plaid](integrations/sources/plaid.md) + * [PokéAPI](integrations/sources/pokeapi.md) * [Postgres](integrations/sources/postgres.md) * [Quickbooks](integrations/sources/quickbooks.md) * [Recurly](integrations/sources/recurly.md) diff --git a/docs/integrations/sources/pokeapi.md b/docs/integrations/sources/pokeapi.md new file mode 100644 index 0000000000000..26fd98205bf02 --- /dev/null +++ b/docs/integrations/sources/pokeapi.md @@ -0,0 +1,31 @@ +# PokéAPI + +## Overview + +The PokéAPI source currently supports only Full Refresh syncs. + +This source uses the fully open [PokéAPI](https://pokeapi.co/docs/v2#info) to serve and retrieve information about Pokémon. This connector should be primarily used for educational purposes or for getting a trial source up and running without needing any dependencies. +### Output schema + +Currently, only one output stream is available from this source, which is the Pokémon output stream. This schema is defined [here](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-pokeapi/schemas/pokemon.json). +### Data type mapping + +The PokéAPI uses the same [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions happen as part of this source. + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | No | +| Namespaces | No | + +### Performance considerations + +According to the API's [fair use policy](https://pokeapi.co/docs/v2#fairuse), please make sure to cache resources retrieved from the PokéAPI wherever possible. + +## Dependencies/Requirements + +- As this API is fully open and is not rate-limited, no authentication or rate-limiting is performed, so you can use this connector right out of the box without any further configuration. From 0870f5855260acb334001278b7cfa24003a7ccfe Mon Sep 17 00:00:00 2001 From: Abhi Vaidyanatha Date: Tue, 4 May 2021 17:17:48 -0700 Subject: [PATCH 7/8] Fix some naming bugs --- .../6371b14b-bc68-4236-bfbd-468e8df8e968.json | 2 +- .../connectors/source-pokeapi/sample_files/config.json | 2 +- .../source-pokeapi/sample_files/configured_catalog.json | 2 +- .../source-pokeapi/source_pokeapi/pokemon_list.py | 2 +- .../connectors/source-pokeapi/source_pokeapi/source.py | 9 +++++---- .../connectors/source-pokeapi/source_pokeapi/spec.json | 4 ++-- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json index 1e82f522e6a5f..06efe37dbe4e2 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6371b14b-bc68-4236-bfbd-468e8df8e968.json @@ -4,4 +4,4 @@ "dockerRepository": "airbyte/source-pokeapi", "dockerImageTag": "0.1.0", "documentationUrl": "https://hub.docker.com/r/airbyte/source-pokeapi" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json b/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json index 4111fdb6585a5..392328e8b0276 100644 --- a/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json +++ b/airbyte-integrations/connectors/source-pokeapi/sample_files/config.json @@ -1 +1 @@ -{ "name": "ditto" } +{ "pokemon_name": "ditto" } diff --git a/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json index 6169d1c68cd5c..72ca528d90f73 100644 --- a/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pokeapi/sample_files/configured_catalog.json @@ -7,7 +7,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { - "name": { + "pokemon_name": { "type": "string" } } diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py index 19f905e2b52e3..1e15f42f5eb7a 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py @@ -21,7 +21,7 @@ # SOFTWARE. -import pokemon_list +from . import pokemon_list # Store large list of all known Pokemon. POKEMON_LIST = [ diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 51029d4f473d4..b7c9a2401cfcf 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -25,7 +25,8 @@ import requests from base_python import AbstractSource, HttpStream, Stream -from pokemon_list import POKEMON_LIST + +from . import pokemon_list class PokeapiStream(HttpStream): @@ -70,11 +71,11 @@ def path(self, **kwargs) -> str: # Source class SourcePokeapi(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: - input_pokemon = config["name"] - if input_pokemon not in POKEMON_LIST: + input_pokemon = config["pokemon_name"] + if input_pokemon not in pokemon_list.POKEMON_LIST: return False, f"Input Pokemon {input_pokemon} is invalid. Please check your spelling our input a valid Pokemon." else: return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [Pokemon(name=config["name"])] + return [Pokemon(pokemon_name=config["pokemon_name"])] diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json index a0d624df6d749..d6c21ab78585b 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json @@ -4,10 +4,10 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Pokeapi Spec", "type": "object", - "required": ["name"], + "required": ["pokemon_name"], "additionalProperties": false, "properties": { - "name": { + "pokemon_name": { "type": "string", "description": "Pokemon requested from the API.", "pattern": "^[a-z0-9_\\-]+$", From 77ec8368280a7f2df03d11c88d1b2324c34837e8 Mon Sep 17 00:00:00 2001 From: Abhi Vaidyanatha Date: Tue, 4 May 2021 18:35:46 -0700 Subject: [PATCH 8/8] Update config file path --- airbyte-integrations/connectors/source-pokeapi/build.gradle | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/build.gradle b/airbyte-integrations/connectors/source-pokeapi/build.gradle index 55922454bdbfc..1d52759d1bd6b 100644 --- a/airbyte-integrations/connectors/source-pokeapi/build.gradle +++ b/airbyte-integrations/connectors/source-pokeapi/build.gradle @@ -12,14 +12,12 @@ airbyteStandardSourceTestFile { // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors // All these input paths must live inside this connector's directory (or subdirectories) - // TODO update the spec JSON file specPath = "source_pokeapi/spec.json" // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file // there (in case it contains any credentials) - // TODO update the config file to contain actual credentials - configPath = "secrets/config.json" - // TODO update the sample configured_catalog JSON for use in testing + configPath = "sample_files/config.json" + // Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured // for incremental syncing (e.g. include cursor fields, etc). configuredCatalogPath = "sample_files/configured_catalog.json"