From 8e1bb98858f2356845ea9cb77ca8839fc79cce65 Mon Sep 17 00:00:00 2001 From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com> Date: Thu, 7 May 2020 01:52:26 +0900 Subject: [PATCH 1/6] Fix feedback bug and add features. --- src/main/python/mlsearch/api_requester.py | 27 +++- src/main/python/mlsearch/config.py | 38 ++++-- src/main/scripts/mlsearch | 146 ++++++++++++++-------- 3 files changed, 143 insertions(+), 68 deletions(-) diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py index 30082d2..ca44f5d 100644 --- a/src/main/python/mlsearch/api_requester.py +++ b/src/main/python/mlsearch/api_requester.py @@ -44,6 +44,15 @@ def __init__(self, source, query, init_idx, count, y_next_page_token=None): "y_next_page_token": None, } + @property + def youtube_query_order(self): + return self._config.YOUTUBE_ORDER + + @youtube_query_order.setter + def youtube_query_order(self, youtube_order): + if youtube_order: + self._config.YOUTUBE_ORDER = youtube_order + @property def github_acc_token(self): return self._config.GITHUB_ACC_TOKEN @@ -217,6 +226,12 @@ def _fetch_paperwithcode(self) -> [Protocol]: def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: """Fetch the Youtube Repository""" results = [] + input_query = str(self.params["query"]).lower().strip() + user_query = input_query + + if not self._config.YOUTUBE_FIX_KEYWORD.strip() in user_query: + user_query = input_query + self._config.YOUTUBE_QUERY_FILTER + youtube = googleapiclient.discovery.build( self._config.YOUTUBE_SERVICE_NAME, self._config.YOUTUBE_API_VERSION, @@ -226,7 +241,7 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: part=self._config.YOUTUBE_PART, maxResults=self.params["count"], order=self._config.YOUTUBE_ORDER, - q=self.params["query"], + q=user_query, safeSearch=self._config.YOUTUBE_SAFESEARCH, pageToken=y_next_page_token, ) @@ -234,6 +249,10 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: if "items" in response and len(response["items"]) > 0: for item in response["items"]: + # Skip if the video id is null + if not item.get("id", dict({"videoId": None})).get("videoId", None): + continue + data = { "video_id": self._unescape( item.get("id", dict({"videoId": None})).get("videoId", None) @@ -282,6 +301,7 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: ) > 0 ) + self.data["y_query_order"] = self._config.YOUTUBE_ORDER self.data["response_code"] = 200 def fetch_data(self) -> json: @@ -295,6 +315,11 @@ def fetch_data(self) -> json: self._fetch_github() if self.params.get("source", "") == "youtube": + if not self._config.YOUTUBE_ORDER in self._config.VALID_YOUTUBE_ORDER: + self.data["response_code"] = 400 + self.data["content"] = "Invalid Youtube Query Order." + return self.data + self._fetch_youtube(self.params.get("y_next_page_token", None)) # TODO: Implement the function for Coursera. However, this function diff --git a/src/main/python/mlsearch/config.py b/src/main/python/mlsearch/config.py index c53574f..1bd805d 100644 --- a/src/main/python/mlsearch/config.py +++ b/src/main/python/mlsearch/config.py @@ -1,24 +1,38 @@ import os + class Config(object): """Class for API Request configuration.""" # Paper with code configuration - PWC_USER_NAME = os.environ.get('PWC_USER_NAME') or '' - PWC_PASSWORD = os.environ.get('PWC_PASSWORD') or '' - PWC_URL = os.environ.get('PWC_URL') or "https://paperswithcode.com/api/v0/search/?q=" + PWC_USER_NAME = os.environ.get("PWC_USER_NAME") or "" + PWC_PASSWORD = os.environ.get("PWC_PASSWORD") or "" + PWC_URL = ( + os.environ.get("PWC_URL") or "https://paperswithcode.com/api/v0/search/?q=" + ) # Github configuration - GITHUB_ACC_TOKEN = os.environ.get('GITHUB_ACC_TOKEN') or None - GITHUB_URL = os.environ.get('GITHUB_URL') or "in:readme+in:description" + GITHUB_ACC_TOKEN = os.environ.get("GITHUB_ACC_TOKEN") or None + GITHUB_URL = os.environ.get("GITHUB_URL") or "in:readme+in:description" # AIP Source - VALID_API_SOURCE = ['paperwithcode', 'github', 'coursera', 'youtube'] + VALID_API_SOURCE = ["paperwithcode", "github", "coursera", "youtube"] # Youtube configuration - YOUTUBE_SERVICE_NAME = os.environ.get('YOUTUBE_SERVICE_NAME') or "youtube" - YOUTUBE_API_VERSION = os.environ.get('YOUTUBE_API_VERSION') or "v3" - YOUTUBE_DEVELOPER_KEY = os.environ.get('YOUTUBE_DEVELOPER_KEY') or None - YOUTUBE_ORDER = os.environ.get('YOUTUBE_ORDER') or "relevance" - YOUTUBE_SAFESEARCH = os.environ.get('YOUTUBE_SAFESEARCH') or "strict" - YOUTUBE_PART = os.environ.get('YOUTUBE_PART') or "snippet" \ No newline at end of file + YOUTUBE_SERVICE_NAME = os.environ.get("YOUTUBE_SERVICE_NAME") or "youtube" + YOUTUBE_API_VERSION = os.environ.get("YOUTUBE_API_VERSION") or "v3" + YOUTUBE_DEVELOPER_KEY = os.environ.get("YOUTUBE_DEVELOPER_KEY") or None + YOUTUBE_ORDER = os.environ.get("YOUTUBE_ORDER") or "relevance" + YOUTUBE_SAFESEARCH = os.environ.get("YOUTUBE_SAFESEARCH") or "strict" + YOUTUBE_PART = os.environ.get("YOUTUBE_PART") or "snippet" + YOUTUBE_FIX_KEYWORD = "machine learning" + YOUTUBE_QUERY_FILTER = " " + YOUTUBE_FIX_KEYWORD + " -news" + VALID_YOUTUBE_ORDER = [ + "date", + "rating", + "relevance", + "title", + # "videoCount", # This is for channel only + "viewCount", + ] + diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch index 0d1b98e..878f055 100644 --- a/src/main/scripts/mlsearch +++ b/src/main/scripts/mlsearch @@ -6,67 +6,101 @@ import os import json # For debugging purpose -if 'mlsearch' not in sys.modules: - sys.path.append(os.path.join(os.getcwd(), 'src/main/python')) +if "mlsearch" not in sys.modules: + sys.path.append(os.path.join(os.getcwd(), "src/main/python")) from mlsearch.api_requester import APIRequest from mlsearch import helper as hp ap = argparse.ArgumentParser() -ap.add_argument('-q', '--query', required=True, help="Keyword for searching.") -ap.add_argument('-i', '--init_idx', required=True, help="Initial index for pagination.") -ap.add_argument('-c', '--count', required=True, help="Total number of results to be fetched.") -ap.add_argument('-s', '--source', required=True, help="Source API to be looking for.") -ap.add_argument('-ck', '--cookies', required=True, help="Cookies of current user.") -ap.add_argument('-tm', '--timestamp', required=True, help="Timestamp of requesting API.") -ap.add_argument('-pu', '--pwc_user', required=False, help="Paper with code repository user name.") -ap.add_argument('-pp', '--pwc_password', required=False, help="Paper with code repository password.") -ap.add_argument('-gt', '--github_acc_token', required=False, help="Github access token.") -ap.add_argument('-yk', '--youtube_dev_key', required=False, help="Youtube developer key.") -ap.add_argument('-yntp', '--y_next_page_token', required=False, help="Next page token for Youtube API.") +ap.add_argument("-q", "--query", required=True, help="Keyword for searching.") +ap.add_argument("-i", "--init_idx", required=True, help="Initial index for pagination.") +ap.add_argument( + "-c", "--count", required=True, help="Total number of results to be fetched." +) +ap.add_argument("-s", "--source", required=True, help="Source API to be looking for.") +ap.add_argument("-ck", "--cookies", required=True, help="Cookies of current user.") +ap.add_argument( + "-tm", "--timestamp", required=True, help="Timestamp of requesting API." +) +ap.add_argument( + "-pu", "--pwc_user", required=False, help="Paper with code repository user name." +) +ap.add_argument( + "-pp", "--pwc_password", required=False, help="Paper with code repository password." +) +ap.add_argument( + "-gt", "--github_acc_token", required=False, help="Github access token." +) +ap.add_argument( + "-yk", "--youtube_dev_key", required=False, help="Youtube developer key." +) +ap.add_argument( + "-yntp", + "--y_next_page_token", + required=False, + help="Next page token for Youtube API.", +) +ap.add_argument( + "-yo", "--youtube_query_order", required=False, help="Youtube Query Order." +) args = vars(ap.parse_args()) + def main(event): headers = { - 'Access-Control-Allow-Origin': '*', - 'X-Requested-With': '*', - 'Access-Control-Allow-Headers': 'Content-Type,X-Amz-Date,Authorization,X-Api-Key,x-requested-with', - 'Access-Control-Allow-Methods': 'OPTIONS,POST,GET' - } + "Access-Control-Allow-Origin": "*", + "X-Requested-With": "*", + "Access-Control-Allow-Headers": "Content-Type,X-Amz-Date,Authorization,X-Api-Key,x-requested-with", + "Access-Control-Allow-Methods": "OPTIONS,POST,GET", + } try: param_names = [ - 'query', 'init_idx', - 'count', 'source', - 'cookies', 'timestamp', - 'y_next_page_token'] - response_msg = hp.response('success', 200) + "query", + "init_idx", + "count", + "source", + "cookies", + "timestamp", + "y_next_page_token", + ] + response_msg = hp.response("success", 200) if hp.is_valid_parameters(event, param_names): params = hp.parse_parameters(event) if params.values(): api_request = APIRequest( - params['source'], - params['query'], - params['init_idx'], - params['count'], - params['y_next_page_token']) - if 'pwc_user'in event and 'pwc_password' in event: - api_request.pwc_auth_info = (event['pwc_user'], event['pwc_password']) - if 'github_acc_token' in event: - api_request.github_acc_token = event['github_acc_token'] - if 'youtube_developer_key' in event: - api_request.youtube_developer_key = event['youtube_developer_key'] + params["source"], + params["query"], + params["init_idx"], + params["count"], + params["y_next_page_token"], + ) + if "pwc_user" in event and "pwc_password" in event: + api_request.pwc_auth_info = ( + event["pwc_user"], + event["pwc_password"], + ) + if "github_acc_token" in event: + api_request.github_acc_token = event["github_acc_token"] + if "youtube_developer_key" in event: + api_request.youtube_developer_key = event["youtube_developer_key"] + if "youtube_query_order" in event: + api_request.youtube_query_order = event["youtube_query_order"] data = api_request.fetch_data() response_msg = hp.response( - message=data.get('content',''), - status_code=data.get('response_code'), + message=data.get("content", ""), + status_code=data.get("response_code"), headers=headers, optional_attributes={ - 'has_next_page': data.get('has_next_page', False), - 'y_next_page_token': data.get('y_next_page_token', None)}) + "has_next_page": data.get("has_next_page", False), + "y_next_page_token": data.get("y_next_page_token", None), + "y_query_order": data.get("y_query_order", None) + }, + ) return response_msg - response_msg = hp.response('Invalid parameters.', 400) + response_msg = hp.response("Invalid parameters.", 400) return response_msg except (ValueError, TypeError) as ex: @@ -77,26 +111,28 @@ def main(event): response_msg = hp.response(str(ex), 500) return response_msg + if __name__ == "__main__": event = { - 'query': args['query'], - 'init_idx': args['init_idx'], - 'count': args['count'], - 'source': args['source'], - 'cookies': args['cookies'], - 'timestamp': args['timestamp'], - 'y_next_page_token': args['y_next_page_token'] + "query": args["query"], + "init_idx": args["init_idx"], + "count": args["count"], + "source": args["source"], + "cookies": args["cookies"], + "timestamp": args["timestamp"], + "y_next_page_token": args["y_next_page_token"], } - if args['pwc_user']: - event['pwc_user'] = args['pwc_user'] - if args['pwc_password']: - event['pwc_password'] = args['pwc_password'] - if args['github_acc_token']: - event['github_acc_token'] = args['github_acc_token'] - if args['youtube_dev_key']: - event['youtube_developer_key'] = args['youtube_dev_key'] - + if args["pwc_user"]: + event["pwc_user"] = args["pwc_user"] + if args["pwc_password"]: + event["pwc_password"] = args["pwc_password"] + if args["github_acc_token"]: + event["github_acc_token"] = args["github_acc_token"] + if args["youtube_dev_key"]: + event["youtube_developer_key"] = args["youtube_dev_key"] + if args["youtube_query_order"]: + event["youtube_query_order"] = args["youtube_query_order"] result = main(event) pp = pprint.PrettyPrinter(indent=2) pp.pprint(result) From 46bf6f6e39f5dbfd62e8bec4e30949bcf24f3a4f Mon Sep 17 00:00:00 2001 From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com> Date: Thu, 7 May 2020 01:59:50 +0900 Subject: [PATCH 2/6] Fix ambiguous youtube query order keyword. --- src/main/scripts/mlsearch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch index 878f055..527b615 100644 --- a/src/main/scripts/mlsearch +++ b/src/main/scripts/mlsearch @@ -42,7 +42,7 @@ ap.add_argument( help="Next page token for Youtube API.", ) ap.add_argument( - "-yo", "--youtube_query_order", required=False, help="Youtube Query Order." + "-yo", "--y_query_order", required=False, help="Youtube Query Order." ) args = vars(ap.parse_args()) @@ -84,8 +84,8 @@ def main(event): api_request.github_acc_token = event["github_acc_token"] if "youtube_developer_key" in event: api_request.youtube_developer_key = event["youtube_developer_key"] - if "youtube_query_order" in event: - api_request.youtube_query_order = event["youtube_query_order"] + if "y_query_order" in event: + api_request.youtube_query_order = event["y_query_order"] data = api_request.fetch_data() response_msg = hp.response( message=data.get("content", ""), @@ -131,8 +131,8 @@ if __name__ == "__main__": event["github_acc_token"] = args["github_acc_token"] if args["youtube_dev_key"]: event["youtube_developer_key"] = args["youtube_dev_key"] - if args["youtube_query_order"]: - event["youtube_query_order"] = args["youtube_query_order"] + if args["y_query_order"]: + event["y_query_order"] = args["y_query_order"] result = main(event) pp = pprint.PrettyPrinter(indent=2) pp.pprint(result) From 9abf08fda1d98ca010b159548e6ff6b31a74c04e Mon Sep 17 00:00:00 2001 From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com> Date: Thu, 7 May 2020 02:19:02 +0900 Subject: [PATCH 3/6] Add usage to README file. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c425d65..da9ec19 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ Optional Parameters: Youtube developer key. -ynpt NEXT_PAGE_TOKEN, --y_next_page_token NEXT_PAGE_TOKEN Next page token for Youtube API. + -yo Y_QUERY_ORDER, --y_query_order Y_QUERY_ORDER + Youtube Query Order. ```
From 569a8dc855857901be513b75ff32c19898e168ce Mon Sep 17 00:00:00 2001 From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com> Date: Sun, 17 May 2020 03:39:33 +0900 Subject: [PATCH 4/6] Fix YouTube response result and Exceptions. --- src/main/python/mlsearch/api_requester.py | 65 +++++++++++++++++------ src/main/python/mlsearch/config.py | 10 +++- src/main/scripts/mlsearch | 10 ++-- 3 files changed, 63 insertions(+), 22 deletions(-) diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py index ca44f5d..9becbe0 100644 --- a/src/main/python/mlsearch/api_requester.py +++ b/src/main/python/mlsearch/api_requester.py @@ -2,10 +2,14 @@ from mlsearch.protocol import Protocol from github import Github from requests.auth import HTTPBasicAuth +from github.GithubException import BadCredentialsException +from github.GithubException import RateLimitExceededException +from googleapiclient.errors import HttpError import googleapiclient.discovery import json import requests import html +import random # import scholarly @@ -68,8 +72,12 @@ def youtube_developer_key(self): @youtube_developer_key.setter def youtube_developer_key(self, developer_key): - if developer_key: + if isinstance(developer_key, list): self._config.YOUTUBE_DEVELOPER_KEY = developer_key + elif isinstance(developer_key, str) and "," in developer_key: + self._config.YOUTUBE_DEVELOPER_KEY = developer_key.strip().split(",") + elif developer_key and isinstance(developer_key, str): + self._config.YOUTUBE_DEVELOPER_KEY.append(developer_key) @property def pwc_auth_info(self): @@ -220,8 +228,13 @@ def _fetch_paperwithcode(self) -> [Protocol]: results.append(Protocol(data)) self.data["content"] = [proto.to_JSON() for proto in results] - - self.data["response_code"] = query_result.status_code + else: + print(str(query_result.status_code), query_result.content) + self.data["response_code"] = query_result.status_code + self.data["content"] = ( + "There is an error in fetching data from PWC server." + f" {json.loads(query_result.content).get('error')}" + ) def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: """Fetch the Youtube Repository""" @@ -232,18 +245,23 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: if not self._config.YOUTUBE_FIX_KEYWORD.strip() in user_query: user_query = input_query + self._config.YOUTUBE_QUERY_FILTER + sampled_dev_key = None + if len(self._config.YOUTUBE_DEVELOPER_KEY) > 0: + sampled_dev_key = random.choice(self._config.YOUTUBE_DEVELOPER_KEY) youtube = googleapiclient.discovery.build( self._config.YOUTUBE_SERVICE_NAME, self._config.YOUTUBE_API_VERSION, - developerKey=self._config.YOUTUBE_DEVELOPER_KEY, + developerKey=sampled_dev_key, ) + request = youtube.search().list( part=self._config.YOUTUBE_PART, maxResults=self.params["count"], order=self._config.YOUTUBE_ORDER, q=user_query, safeSearch=self._config.YOUTUBE_SAFESEARCH, - pageToken=y_next_page_token, + # Disabled the next page token due to limitation of api access. + # pageToken=y_next_page_token, ) response = request.execute() @@ -293,14 +311,15 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: "source": self.params.get("source", ""), } results.append(Protocol(data)) - self.data["y_next_page_token"] = response.get("nextPageToken", None) + # self.data["y_next_page_token"] = response.get("nextPageToken", None) self.data["content"] = [proto.to_JSON() for proto in results] - self.data["has_next_page"] = ( - response.get("pageInfo", dict({"totalResults": 0})).get( - "totalResults", 0 - ) - > 0 - ) + # self.data["has_next_page"] = ( + # response.get("pageInfo", dict({"totalResults": 0})).get( + # "totalResults", 0 + # ) + # > 0 + # ) + self.data["has_next_page"] = False self.data["y_query_order"] = self._config.YOUTUBE_ORDER self.data["response_code"] = 200 @@ -312,15 +331,28 @@ def fetch_data(self) -> json: self._fetch_paperwithcode() if self.params.get("source", "") == "github": - self._fetch_github() + try: + self._fetch_github() + except BadCredentialsException: + self.data["response_code"] = 400 + self.data["content"] = "Invalid Github developer key." + except RateLimitExceededException: + self.data["response_code"] = 503 + self.data["content"] = "Access rate limitation reached." if self.params.get("source", "") == "youtube": if not self._config.YOUTUBE_ORDER in self._config.VALID_YOUTUBE_ORDER: self.data["response_code"] = 400 self.data["content"] = "Invalid Youtube Query Order." return self.data - - self._fetch_youtube(self.params.get("y_next_page_token", None)) + try: + self._fetch_youtube(self.params.get("y_next_page_token", None)) + except HttpError as ex: + print(str(ex)) + self.data["response_code"] = 400 + self.data[ + "content" + ] = "Seems there is an authentication error with Youtube server." # TODO: Implement the function for Coursera. However, this function # may be handled by the backend server. @@ -328,7 +360,8 @@ def fetch_data(self) -> json: pass except Exception as ex: + print(str(ex)) + self.data["content"] = "Oops... Something has gone wrong in server." self.data["response_code"] = 500 - self.data["content"] = str(ex) return self.data diff --git a/src/main/python/mlsearch/config.py b/src/main/python/mlsearch/config.py index 1bd805d..729e4ec 100644 --- a/src/main/python/mlsearch/config.py +++ b/src/main/python/mlsearch/config.py @@ -21,7 +21,15 @@ class Config(object): # Youtube configuration YOUTUBE_SERVICE_NAME = os.environ.get("YOUTUBE_SERVICE_NAME") or "youtube" YOUTUBE_API_VERSION = os.environ.get("YOUTUBE_API_VERSION") or "v3" - YOUTUBE_DEVELOPER_KEY = os.environ.get("YOUTUBE_DEVELOPER_KEY") or None + # Parsing Youtube Keys + YOUTUBE_DEVELOPER_KEY = list() + developer_key = os.environ.get("YOUTUBE_DEVELOPER_KEY") + if isinstance(developer_key, list): + YOUTUBE_DEVELOPER_KEY = developer_key + elif isinstance(developer_key, str) and "," in developer_key: + YOUTUBE_DEVELOPER_KEY = developer_key.strip().split(",") + elif developer_key and isinstance(developer_key, str): + YOUTUBE_DEVELOPER_KEY.append(developer_key) YOUTUBE_ORDER = os.environ.get("YOUTUBE_ORDER") or "relevance" YOUTUBE_SAFESEARCH = os.environ.get("YOUTUBE_SAFESEARCH") or "strict" YOUTUBE_PART = os.environ.get("YOUTUBE_PART") or "snippet" diff --git a/src/main/scripts/mlsearch b/src/main/scripts/mlsearch index 527b615..7ae2448 100644 --- a/src/main/scripts/mlsearch +++ b/src/main/scripts/mlsearch @@ -33,7 +33,7 @@ ap.add_argument( "-gt", "--github_acc_token", required=False, help="Github access token." ) ap.add_argument( - "-yk", "--youtube_dev_key", required=False, help="Youtube developer key." + "-yk", "--y_dev_key", required=False, help="Youtube developer key." ) ap.add_argument( "-yntp", @@ -82,8 +82,8 @@ def main(event): ) if "github_acc_token" in event: api_request.github_acc_token = event["github_acc_token"] - if "youtube_developer_key" in event: - api_request.youtube_developer_key = event["youtube_developer_key"] + if "y_dev_key" in event: + api_request.youtube_developer_key = event["y_dev_key"] if "y_query_order" in event: api_request.youtube_query_order = event["y_query_order"] data = api_request.fetch_data() @@ -129,8 +129,8 @@ if __name__ == "__main__": event["pwc_password"] = args["pwc_password"] if args["github_acc_token"]: event["github_acc_token"] = args["github_acc_token"] - if args["youtube_dev_key"]: - event["youtube_developer_key"] = args["youtube_dev_key"] + if args["y_dev_key"]: + event["y_dev_key"] = args["y_dev_key"] if args["y_query_order"]: event["y_query_order"] = args["y_query_order"] result = main(event) From 9c52cf512b65d908db1097485b405a8c0e375e56 Mon Sep 17 00:00:00 2001 From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com> Date: Sun, 24 May 2020 15:16:27 +0900 Subject: [PATCH 5/6] Check Auth Error on YouTube before happening. --- src/main/python/mlsearch/api_requester.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py index 9becbe0..f07413d 100644 --- a/src/main/python/mlsearch/api_requester.py +++ b/src/main/python/mlsearch/api_requester.py @@ -10,9 +10,11 @@ import requests import html import random +import collections # import scholarly +ErrorType = collections.namedtuple("ErrorType", "reason status") class APIRequest: """For handling the different Valid API requests.""" @@ -248,6 +250,13 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: sampled_dev_key = None if len(self._config.YOUTUBE_DEVELOPER_KEY) > 0: sampled_dev_key = random.choice(self._config.YOUTUBE_DEVELOPER_KEY) + + if not sampled_dev_key: + auth_error = ErrorType( + reason="Empty YouTube Developer Key.", status="400" + ) + raise HttpError(auth_error, str.encode("YouTube Developer Key Required.")) + youtube = googleapiclient.discovery.build( self._config.YOUTUBE_SERVICE_NAME, self._config.YOUTUBE_API_VERSION, From e0270f49da2dab6b54adac94b63c0f952046d919 Mon Sep 17 00:00:00 2001 From: saihtaungkham <31495612+saihtaungkham@users.noreply.github.com> Date: Mon, 25 May 2020 10:47:11 +0900 Subject: [PATCH 6/6] Fix YouTube response result and Exceptions. --- src/main/python/mlsearch/api_requester.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/python/mlsearch/api_requester.py b/src/main/python/mlsearch/api_requester.py index f07413d..a1a9a8b 100644 --- a/src/main/python/mlsearch/api_requester.py +++ b/src/main/python/mlsearch/api_requester.py @@ -248,15 +248,14 @@ def _fetch_youtube(self, y_next_page_token=None) -> [Protocol]: user_query = input_query + self._config.YOUTUBE_QUERY_FILTER sampled_dev_key = None - if len(self._config.YOUTUBE_DEVELOPER_KEY) > 0: - sampled_dev_key = random.choice(self._config.YOUTUBE_DEVELOPER_KEY) - - if not sampled_dev_key: + if not len(self._config.YOUTUBE_DEVELOPER_KEY) > 0: auth_error = ErrorType( reason="Empty YouTube Developer Key.", status="400" ) raise HttpError(auth_error, str.encode("YouTube Developer Key Required.")) + sampled_dev_key = random.choice(self._config.YOUTUBE_DEVELOPER_KEY) + youtube = googleapiclient.discovery.build( self._config.YOUTUBE_SERVICE_NAME, self._config.YOUTUBE_API_VERSION,