From 2f54069f7eb6dc197226765cae02720276b51d2e Mon Sep 17 00:00:00 2001 From: Jabbey92 <32350542+Jabbey92@users.noreply.github.com> Date: Tue, 23 Apr 2019 06:24:55 -0400 Subject: [PATCH 1/3] socket.SIGALRM does not support windows Attempting to revert to not having to check if `get_keys` is done. --- coc/http.py | 42 +++++++----------------------------------- 1 file changed, 7 insertions(+), 35 deletions(-) diff --git a/coc/http.py b/coc/http.py index ed49c8c8..fe92a7e4 100644 --- a/coc/http.py +++ b/coc/http.py @@ -30,44 +30,22 @@ import logging import aiohttp import asyncio -import signal import sys -import functools +import time +from functools import wraps +from threading import Thread from urllib.parse import urlencode from itertools import cycle from datetime import datetime from collections import deque -from .errors import HTTPException, Maitenance, NotFound, InvalidArgument, Forbidden, InvalidCredentials +from .errors import HTTPException, Maitenance, NotFound, InvalidArgument, Forbidden log = logging.getLogger(__name__) KEY_MINIMUM, KEY_MAXIMUM = 1, 10 -def timeout(seconds, error_message='Client timed out.'): - def decorated(func): - def _handle_timeout(signum, frame): - raise TimeoutError(error_message) - - def wrapper(*args, **kwargs): - if sys.platform == 'win32': - # TODO: Fix for windows - # for now just return function and ignore the problem - return func(*args, **kwargs) - - signal.signal(signal.SIGALRM, _handle_timeout) - signal.alarm(seconds) - try: - result = func(*args, **kwargs) - finally: - signal.alarm(0) - return result - - return functools.wraps(func)(wrapper) - return decorated - - async def json_or_text(response): try: ret = await response.json() @@ -149,7 +127,8 @@ def __init__(self, client, loop, email, password, self.__lock = asyncio.Semaphore(per_second) self.__throttle = Throttler(per_second, loop=self.loop) - asyncio.ensure_future(self.get_keys()) + loop.run_until_complete(loop.create_task(self.get_keys())) + async def get_keys(self): self.__session = aiohttp.ClientSession(loop=self.loop) @@ -183,18 +162,13 @@ async def get_keys(self): async def close(self): if self.__session: await self.__session.close() - - @timeout(60, "Client timed out while attempting to establish a connection to the Developer Portal") - async def ensure_logged_in(self): - while not hasattr(self, 'keys'): - await asyncio.sleep(0.1) async def request(self, route, **kwargs): method = route.method url = route.url if 'headers' not in kwargs: - await self.ensure_logged_in() + #self.ensure_logged_in() key = next(self.keys) headers = { @@ -366,8 +340,6 @@ async def login_to_site(self, email, password): response_dict = await sess.json() log.debug('%s has received %s', 'https://developer.clashofclans.com/api/login', response_dict) - if sess.status == 403: - raise InvalidCredentials(sess, response_dict) session = sess.cookies.get('session').value From a19a95be853e0dc8ef28b3b8d91d35b596c71ac0 Mon Sep 17 00:00:00 2001 From: Jabbey92 <32350542+Jabbey92@users.noreply.github.com> Date: Tue, 23 Apr 2019 06:38:38 -0400 Subject: [PATCH 2/3] remove unnecessary lines --- coc/http.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/coc/http.py b/coc/http.py index fe92a7e4..cdb92774 100644 --- a/coc/http.py +++ b/coc/http.py @@ -31,7 +31,6 @@ import aiohttp import asyncio import sys -import time from functools import wraps from threading import Thread @@ -168,7 +167,6 @@ async def request(self, route, **kwargs): url = route.url if 'headers' not in kwargs: - #self.ensure_logged_in() key = next(self.keys) headers = { From d7485a8278ddeb36bc6e0614506c161ed4aeeddd Mon Sep 17 00:00:00 2001 From: Jabbey92 <32350542+Jabbey92@users.noreply.github.com> Date: Tue, 23 Apr 2019 07:43:18 -0400 Subject: [PATCH 3/3] Create a separate loop for `get_keys` function Instead of using the current loop to grab the keys and ensuring future, create a seperate one, this way there are no conflicts with the current running loop while ensuring the client logs in and loads the keys before releasing itself. --- coc/http.py | 123 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 51 deletions(-) diff --git a/coc/http.py b/coc/http.py index cdb92774..6e3b4476 100644 --- a/coc/http.py +++ b/coc/http.py @@ -31,7 +31,9 @@ import aiohttp import asyncio import sys +import time +from contextlib import contextmanager from functools import wraps from threading import Thread from urllib.parse import urlencode @@ -109,7 +111,6 @@ def __init__(self, method, path, kwargs: dict=None, api_page=False): else: self.url = url - class HTTPClient: def __init__(self, client, loop, email, password, key_names, key_count, throttle_limit): @@ -126,10 +127,11 @@ def __init__(self, client, loop, email, password, self.__lock = asyncio.Semaphore(per_second) self.__throttle = Throttler(per_second, loop=self.loop) - loop.run_until_complete(loop.create_task(self.get_keys())) + __key_loop__ = asyncio.new_event_loop() + __key_loop__.run_until_complete(self.get_keys(__key_loop__)) - async def get_keys(self): + async def get_keys(self, loop): self.__session = aiohttp.ClientSession(loop=self.loop) key_count = self.key_count @@ -167,6 +169,7 @@ async def request(self, route, **kwargs): url = route.url if 'headers' not in kwargs: + #self.ensure_logged_in() key = next(self.keys) headers = { @@ -179,45 +182,61 @@ async def request(self, route, **kwargs): kwargs['headers']['Content-Type'] = 'application/json' async with self.__lock: - async with self.__throttle, self.__session.request(method, url, **kwargs) as r: - log.debug('%s (%s) has returned %s', url, method, r.status) - data = await json_or_text(r) - log.debug(data) - - if 200 <= r.status < 300: - log.debug('%s has received %s', url, data) - return data - - if r.status == 400: - raise InvalidArgument(r, data) - - if r.status == 403: - if data.get('reason') in ['accessDenied.invalidIp']: - if 'key' in locals(): - await self.reset_key(key) - log.info('Reset Clash of Clans key') - return await self.request(route, **kwargs) - - raise Forbidden(r, data) - - if r.status == 404: - raise NotFound(r, data) - if r.status == 429: - log.error('We have been rate-limited by the API. ' - 'Reconsider the number of requests you are allowing per second.') - raise HTTPException(r, data) - - if r.status == 503: - raise Maitenance(r, data) - else: - raise HTTPException(r, data) + with self.get_correct_loop_session() as session: + async with self.__throttle, session.request(method, url, **kwargs) as r: + log.debug('%s (%s) has returned %s', url, method, r.status) + data = await json_or_text(r) + log.debug(data) + + if 200 <= r.status < 300: + log.debug('%s has received %s', url, data) + return data + + if r.status == 400: + raise InvalidArgument(r, data) + + if r.status == 403: + if data.get('reason') in ['accessDenied.invalidIp']: + if 'key' in locals(): + await self.reset_key(key) + log.info('Reset Clash of Clans key') + return await self.request(route, **kwargs) + + raise Forbidden(r, data) + + if r.status == 404: + raise NotFound(r, data) + if r.status == 429: + log.error('We have been rate-limited by the API. ' + 'Reconsider the number of requests you are allowing per second.') + raise HTTPException(r, data) + + if r.status == 503: + raise Maitenance(r, data) + else: + raise HTTPException(r, data) + + @contextmanager + def get_correct_loop_session(self): + loop = asyncio.get_event_loop() + try: + session = None + if loop != self.__session: + session = aiohttp.ClientSession(loop=loop) + yield session + else: + yield self.__session + finally: + if session: + asyncio.ensure_future(session.close()) async def get_ip(self): - async with self.__session.request('GET', 'http://ip.42.pl/short') as r: - log.debug('%s (%s) has returned %s', 'http://ip.42.pl/short', 'GET', r.status) - ip = await r.text() - log.debug('%s has received %s', 'http://ip.42.pl/short', ip) - return ip + with self.get_correct_loop_session() as session: + async with session.request('GET', 'http://ip.42.pl/short') as r: + log.debug('%s (%s) has returned %s', 'http://ip.42.pl/short', 'GET', r.status) + ip = await r.text() + log.debug('%s has received %s', 'http://ip.42.pl/short', ip) + return ip @staticmethod def create_cookies(response_dict, session): @@ -333,13 +352,14 @@ async def login_to_site(self, email, password): headers = { 'content-type': 'application/json' } - async with self.__session.post('https://developer.clashofclans.com/api/login', - json=login_data, headers=headers) as sess: - response_dict = await sess.json() - log.debug('%s has received %s', 'https://developer.clashofclans.com/api/login', - response_dict) + with self.get_correct_loop_session() as session: + async with session.post('https://developer.clashofclans.com/api/login', + json=login_data, headers=headers) as sess: + response_dict = await sess.json() + log.debug('%s has received %s', 'https://developer.clashofclans.com/api/login', + response_dict) - session = sess.cookies.get('session').value + session = sess.cookies.get('session').value return response_dict, session @@ -348,11 +368,12 @@ async def find_site_keys(self, cookies): "cookie": cookies, "content-type": "application/json" } - async with self.__session.post('https://developer.clashofclans.com/api/apikey/list', - data=json.dumps({}), headers=headers) as sess: - existing_keys_dict = await sess.json() - log.debug('%s has received %s', 'https://developer.clashofclans.com/api/apikey/list', - existing_keys_dict) + with self.get_correct_loop_session() as session: + async with session.post('https://developer.clashofclans.com/api/apikey/list', + data=json.dumps({}), headers=headers) as sess: + existing_keys_dict = await sess.json() + log.debug('%s has received %s', 'https://developer.clashofclans.com/api/apikey/list', + existing_keys_dict) return existing_keys_dict