From 7081585861d06545bfeba498af3b270baba15089 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 15 Feb 2018 09:13:27 +1100 Subject: [PATCH 1/8] add a seperate option for premium accounts to use consumer_key and secret values and inherently generate the bearer token on startup --- searchtweets/credentials.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/searchtweets/credentials.py b/searchtweets/credentials.py index f049d8e..38cb0bd 100644 --- a/searchtweets/credentials.py +++ b/searchtweets/credentials.py @@ -11,8 +11,12 @@ import os import logging import yaml +import requests +import base64 from .utils import merge_dicts +OAUTH_ENDPOINT = 'https://api.twitter.com/oauth2/token' + __all__ = ["load_credentials"] logger = logging.getLogger(__name__) @@ -76,8 +80,16 @@ def _parse_credentials(search_creds, account_type): try: if account_type == "premium": - search_args = {"bearer_token": search_creds["bearer_token"], - "endpoint": search_creds["endpoint"]} + if "bearer_token" not in search_creds: + if "consumer_key" in search_creds \ + and "consumer_secret" in search_creds: + search_creds["bearer_token"] = _generate_bearer_token( + search_creds["consumer_key"], + search_creds["consumer_secret"]) + + search_args = { + "bearer_token": search_creds["bearer_token"], + "endpoint": search_creds["endpoint"]} if account_type == "enterprise": search_args = {"username": search_creds["username"], "password": search_creds["password"], @@ -183,3 +195,19 @@ def load_credentials(filename=None, account_type=None, else merge_dicts(env_vars, yaml_vars)) parsed_vars = _parse_credentials(merged_vars, account_type=account_type) return parsed_vars + + +def _generate_bearer_token(consumer_key, consumer_secret): + """ + Return the bearer token for a given pair of consumer key and secret values. + """ + consumer_secret = base64.b64encode(consumer_secret) + auth_value = 'Basic {0} {1}'.format(consumer_key, consumer_secret) + data = 'grant_type=client_credentials' + resp = requests.post(OAUTH_ENDPOINT, + data=data, + headers={'Authorization': auth_value}) + + resp.raise_for_status() + + return resp.json()['access_token'] From bad8ffadbc33d0210f084e22a805583fc92b6395 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 15 Feb 2018 09:17:31 +1100 Subject: [PATCH 2/8] b64 encode expects a byte string --- searchtweets/credentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/searchtweets/credentials.py b/searchtweets/credentials.py index 38cb0bd..22fe935 100644 --- a/searchtweets/credentials.py +++ b/searchtweets/credentials.py @@ -201,7 +201,7 @@ def _generate_bearer_token(consumer_key, consumer_secret): """ Return the bearer token for a given pair of consumer key and secret values. """ - consumer_secret = base64.b64encode(consumer_secret) + consumer_secret = base64.b64encode(consumer_secret.encode()) auth_value = 'Basic {0} {1}'.format(consumer_key, consumer_secret) data = 'grant_type=client_credentials' resp = requests.post(OAUTH_ENDPOINT, From aaaab9fa50183d70dd71f915dec351f8e3e55480 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 15 Feb 2018 09:40:21 +1100 Subject: [PATCH 3/8] after some more testing, better understood the required headers --- searchtweets/credentials.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/searchtweets/credentials.py b/searchtweets/credentials.py index 22fe935..902d96c 100644 --- a/searchtweets/credentials.py +++ b/searchtweets/credentials.py @@ -201,13 +201,19 @@ def _generate_bearer_token(consumer_key, consumer_secret): """ Return the bearer token for a given pair of consumer key and secret values. """ - consumer_secret = base64.b64encode(consumer_secret.encode()) - auth_value = 'Basic {0} {1}'.format(consumer_key, consumer_secret) + auth = base64.b64encode("{0}:{1}".format( + consumer_key, + consumer_secret).encode()).decode() + + headers = { + 'Authorization': 'Basic {0}'.format(auth), + 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'} data = 'grant_type=client_credentials' resp = requests.post(OAUTH_ENDPOINT, data=data, - headers={'Authorization': auth_value}) - - resp.raise_for_status() + headers=headers) + if resp.status_code >= 400: + logger.error(resp.text) + resp.raise_for_status() return resp.json()['access_token'] From 96c681acad224cbac6b03beab1c1746eda4e6cfd Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Sat, 17 Feb 2018 15:56:14 -0700 Subject: [PATCH 4/8] updated docsstrings around credentials allowing consumer key/secrets --- searchtweets/credentials.py | 35 +++++++++++++---------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/searchtweets/credentials.py b/searchtweets/credentials.py index 902d96c..bef0ddb 100644 --- a/searchtweets/credentials.py +++ b/searchtweets/credentials.py @@ -49,7 +49,10 @@ def _load_env_credentials(): "SEARCHTWEETS_USERNAME", "SEARCHTWEETS_PASSWORD", "SEARCHTWEETS_BEARER_TOKEN", - "SEARCHTWEETS_ACCOUNT_TYPE"] + "SEARCHTWEETS_ACCOUNT_TYPE", + "SEARCHTWEETS_CONSUMER_KEY", + "SEARCHTWEETS_CONSUMER_SECRET" + ] renamed = [var.replace("SEARCHTWEETS_", '').lower() for var in vars_] parsed = {r: os.environ.get(var) for (var, r) in zip(vars_, renamed)} @@ -108,7 +111,7 @@ def load_credentials(filename=None, account_type=None, """ Handles credential management. Supports both YAML files and environment variables. A YAML file is preferred for simplicity and configureability. - A YAML credential file should look like this: + A YAML credential file should look something like this: .. code:: yaml @@ -116,27 +119,13 @@ def load_credentials(filename=None, account_type=None, endpoint: username: password: + consumer_key: + consumer_secret: bearer_token: account_type: with the appropriate fields filled out for your account. The top-level key - defaults to ``search_tweets_api`` but can be flexible, e.g.: - - .. code:: yaml - - premium_dev: - account_type: premium - endpoint: - bearer_token: - - enterprise_dev: - account_type: enterprise - endpoint: - username: - password: - - as this method supports a flexible interface for reading the - credential files. You can keep all of your credentials in the same file. + defaults to ``search_tweets_api`` but can be flexible. If a YAML file is not found or is missing keys, this function will check for this information in the environment variables that correspond to @@ -148,9 +137,10 @@ def load_credentials(filename=None, account_type=None, SEARCHTWEETS_PASSWORD SEARCHTWEETS_BEARER_TOKEN SEARCHTWEETS_ACCOUNT_TYPE + ... Again, set the variables that correspond to your account information and - type. + type. See the main documentation for details and more examples. Args: @@ -174,12 +164,12 @@ def load_credentials(filename=None, account_type=None, dict_keys(['bearer_token', 'endpoint']) >>> import os >>> os.environ["SEARCHTWEETS_ENDPOINT"] = "https://endpoint" - >>> os.environ["SEARCHTWEETS_USERNAME"] = "azaleszzz" + >>> os.environ["SEARCHTWEETS_USERNAME"] = "areallybadpassword" >>> os.environ["SEARCHTWEETS_PASSWORD"] = "" >>> load_credentials() {'endpoint': 'https://endpoint', 'password': '', - 'username': 'azaleszzz'} + 'username': 'areallybadpassword'} """ yaml_key = yaml_key if yaml_key is not None else "search_tweets_api" @@ -212,6 +202,7 @@ def _generate_bearer_token(consumer_key, consumer_secret): resp = requests.post(OAUTH_ENDPOINT, data=data, headers=headers) + logger.warning("Grabbing bearer token from OAUTH") if resp.status_code >= 400: logger.error(resp.text) resp.raise_for_status() From a161ba0d229c1aeba01a7a5f908821ed030fdc7c Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 20 Feb 2018 12:01:05 -0700 Subject: [PATCH 5/8] updated docs for credentials --- examples/credential_handling.ipynb | 43 ++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/examples/credential_handling.ipynb b/examples/credential_handling.ipynb index 760282d..9841d41 100644 --- a/examples/credential_handling.ipynb +++ b/examples/credential_handling.ipynb @@ -6,12 +6,17 @@ "source": [ "# Credential Handling\n", "\n", - "The premium and enterprise Search APIs use different authentication methods and we attempt to provide a seamless way to handle authentication for all customers. \n", + "The premium and enterprise Search APIs use different authentication methods and we attempt to provide a seamless way to handle authentication for all customers. We know credentials can be tricking or annoying - please read this in its entirety.\n", + "\n", "\n", "Premium clients will require the `bearer_token` and `endpoint` fields; Enterprise clients require `username`, `password`, and `endpoint`.\n", "If you do not specify the `account_type`, we attempt to discern the account type and declare a warning about this behavior.\n", "\n", - "For premium search products, we are using app-only authentication and the bearer tokens are not delivered with an expiration time. They can be invalidated. Please see [here](https://developer.twitter.com/en/docs/basics/authentication/overview/application-only) for an overview of the premium authentication method.\n", + "For premium search products, we are using app-only authentication and the bearer tokens are not delivered with an expiration time. You can provide either:\n", + "- your application key and secret (the library will handle bearer-token authentication)\n", + "- a bearer token that you get yourself\n", + "\n", + "Many developers might find providing your application key and secret more straightforward and letting this library manage your bearer token generation for you. Please see [here](https://developer.twitter.com/en/docs/basics/authentication/overview/application-only) for an overview of the premium authentication method.\n", "\n", "We support both YAML-file based methods and environment variables for storing credentials, and provide flexible handling with sensible defaults.\n", "\n", @@ -25,7 +30,8 @@ "search_tweets_api:\n", " account_type: premium\n", " endpoint: \n", - " bearer_token: \n", + " consumer_key: \n", + " consumer_secret: \n", "```\n", "\n", "For enterprise customers, the simplest credential file should look like this:\n", @@ -37,14 +43,18 @@ " endpoint: \n", " username: \n", " password: \n", - "```\n", - "\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "By default, this library expects this file at `\"~/.twitter_keys.yaml\"`, but you can pass the relevant location as needed, either with the ``--credential-file`` flag for the command-line app or as demonstrated below in a Python program.\n", "\n", "Both above examples require no special command-line arguments or in-program arguments. The credential parsing methods, unless otherwise specified, will look for a YAML key called `search_tweets_api`.\n", "\n", "\n", - "\n", "For developers who have multiple endpoints and/or search products, you can keep all credentials in the same file and specify specific keys to use. `--credential-file-key` specifies this behavior in the command line app. An example:\n", "\n", "\n", @@ -53,7 +63,10 @@ "search_tweets_30_day_dev:\n", " account_type: premium\n", " endpoint: \n", - " bearer_token: \n", + " consumer_key: \n", + " consumer_secret: \n", + " (optional) bearer_token: \n", + " \n", " \n", "search_tweets_30_day_prod:\n", " account_type: premium\n", @@ -87,6 +100,8 @@ "export SEARCHTWEETS_PASSWORD=\n", "export SEARCHTWEETS_BEARER_TOKEN=\n", "export SEARCHTWEETS_ACCOUNT_TYPE=\n", + "export SEARCHTWEETS_CONSUMER_KEY=\n", + "export SEARCHTWEETS_CONSUMER_SECRET=\n", "```\n", "\n", "The `load_credentials` function will attempt to find these variables if it cannot load fields from the YAML file, and it will **overwrite any credentials from the YAML file that are present as environment variables** if they have been parsed. This behavior can be changed by setting the `load_credentials` parameter `env_overwrite` to `False`.\n", @@ -97,7 +112,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 10, "metadata": { "collapsed": true }, @@ -108,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -119,7 +134,7 @@ " 'username': ''}" ] }, - "execution_count": 2, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -132,7 +147,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -142,7 +157,7 @@ " 'endpoint': 'https://api.twitter.com/1.1/tweets/search/30day/dev.json'}" ] }, - "execution_count": 3, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -164,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -183,7 +198,7 @@ " 'username': ''}" ] }, - "execution_count": 5, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } From f1a13da541f91936bcef7f065ca9c8b8f2df9f51 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 20 Feb 2018 12:01:43 -0700 Subject: [PATCH 6/8] simplifying bearer token retrieval --- searchtweets/credentials.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/searchtweets/credentials.py b/searchtweets/credentials.py index bef0ddb..4d5f021 100644 --- a/searchtweets/credentials.py +++ b/searchtweets/credentials.py @@ -191,17 +191,10 @@ def _generate_bearer_token(consumer_key, consumer_secret): """ Return the bearer token for a given pair of consumer key and secret values. """ - auth = base64.b64encode("{0}:{1}".format( - consumer_key, - consumer_secret).encode()).decode() - - headers = { - 'Authorization': 'Basic {0}'.format(auth), - 'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'} - data = 'grant_type=client_credentials' + data = [('grant_type', 'client_credentials')] resp = requests.post(OAUTH_ENDPOINT, data=data, - headers=headers) + auth=(consumer_key, consumer_secret)) logger.warning("Grabbing bearer token from OAUTH") if resp.status_code >= 400: logger.error(resp.text) From 9a229c0def2ecf769ad569a82dfd34690b1cdb3d Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 20 Feb 2018 12:02:26 -0700 Subject: [PATCH 7/8] updated base readme with new docs --- README.rst | 23 ++++++++++++++++++----- examples/credential_handling.rst | 23 ++++++++++++++++++----- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 1ee875a..7aec329 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,8 @@ Credential Handling The premium and enterprise Search APIs use different authentication methods and we attempt to provide a seamless way to handle -authentication for all customers. +authentication for all customers. We know credentials can be tricking or +annoying - please read this in its entirety. Premium clients will require the ``bearer_token`` and ``endpoint`` fields; Enterprise clients require ``username``, ``password``, and @@ -57,8 +58,14 @@ fields; Enterprise clients require ``username``, ``password``, and discern the account type and declare a warning about this behavior. For premium search products, we are using app-only authentication and -the bearer tokens are not delivered with an expiration time. They can be -invalidated. Please see +the bearer tokens are not delivered with an expiration time. You can +provide either: - your application key and secret (the library will +handle bearer-token authentication) - a bearer token that you get +yourself + +Many developers might find providing your application key and secret +more straightforward and letting this library manage your bearer token +generation for you. Please see `here `__ for an overview of the premium authentication method. @@ -77,7 +84,8 @@ this: search_tweets_api: account_type: premium endpoint: - bearer_token: + consumer_key: + consumer_secret: For enterprise customers, the simplest credential file should look like this: @@ -109,7 +117,10 @@ line app. An example: search_tweets_30_day_dev: account_type: premium endpoint: - bearer_token: + consumer_key: + consumer_secret: + (optional) bearer_token: + search_tweets_30_day_prod: account_type: premium @@ -139,6 +150,8 @@ can set the appropriate variables for your product of the following: export SEARCHTWEETS_PASSWORD= export SEARCHTWEETS_BEARER_TOKEN= export SEARCHTWEETS_ACCOUNT_TYPE= + export SEARCHTWEETS_CONSUMER_KEY= + export SEARCHTWEETS_CONSUMER_SECRET= The ``load_credentials`` function will attempt to find these variables if it cannot load fields from the YAML file, and it will **overwrite any diff --git a/examples/credential_handling.rst b/examples/credential_handling.rst index 946c1e1..76130ac 100644 --- a/examples/credential_handling.rst +++ b/examples/credential_handling.rst @@ -4,7 +4,8 @@ Credential Handling The premium and enterprise Search APIs use different authentication methods and we attempt to provide a seamless way to handle -authentication for all customers. +authentication for all customers. We know credentials can be tricking or +annoying - please read this in its entirety. Premium clients will require the ``bearer_token`` and ``endpoint`` fields; Enterprise clients require ``username``, ``password``, and @@ -12,8 +13,14 @@ fields; Enterprise clients require ``username``, ``password``, and discern the account type and declare a warning about this behavior. For premium search products, we are using app-only authentication and -the bearer tokens are not delivered with an expiration time. They can be -invalidated. Please see +the bearer tokens are not delivered with an expiration time. You can +provide either: - your application key and secret (the library will +handle bearer-token authentication) - a bearer token that you get +yourself + +Many developers might find providing your application key and secret +more straightforward and letting this library manage your bearer token +generation for you. Please see `here `__ for an overview of the premium authentication method. @@ -33,7 +40,8 @@ this: search_tweets_api: account_type: premium endpoint: - bearer_token: + consumer_key: + consumer_secret: For enterprise customers, the simplest credential file should look like this: @@ -67,7 +75,10 @@ line app. An example: search_tweets_30_day_dev: account_type: premium endpoint: - bearer_token: + consumer_key: + consumer_secret: + (optional) bearer_token: + search_tweets_30_day_prod: account_type: premium @@ -98,6 +109,8 @@ can set the appropriate variables for your product of the following: export SEARCHTWEETS_PASSWORD= export SEARCHTWEETS_BEARER_TOKEN= export SEARCHTWEETS_ACCOUNT_TYPE= + export SEARCHTWEETS_CONSUMER_KEY= + export SEARCHTWEETS_CONSUMER_SECRET= The ``load_credentials`` function will attempt to find these variables if it cannot load fields from the YAML file, and it will **overwrite any From 0763f647ca5fc1584b8203710f6009230600e9ca Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 20 Feb 2018 12:05:08 -0700 Subject: [PATCH 8/8] update version --- docs/source/conf.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 6757b0c..3bd2c1d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -66,9 +66,9 @@ # built documents. # # The short X.Y version. -version = '1.2' +version = '1.3' # The full version, including alpha/beta/rc tags. -release = '1.2.1' +release = '1.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index d53aa4f..0094a91 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ long_description=open('README.rst', 'r').read(), author_email='agonzales@twitter.com', license='MIT', - version='1.2.1', + version='1.3.0', install_requires=["requests", "tweet_parser", "pyyaml"], packages=find_packages(), scripts=["tools/search_tweets.py"],