diff --git a/metaappscriptsdk/__init__.py b/metaappscriptsdk/__init__.py index 8dfbf52..aff389e 100644 --- a/metaappscriptsdk/__init__.py +++ b/metaappscriptsdk/__init__.py @@ -13,6 +13,7 @@ from metaappscriptsdk.logger.logger import Logger from metaappscriptsdk.schedule.Schedule import Schedule from metaappscriptsdk.services import get_api_call_headers, process_meta_api_error_code +from metaappscriptsdk.services.ApiProxyService import ApiProxyService from metaappscriptsdk.services.DbQueryService import DbQueryService from metaappscriptsdk.services.DbService import DbService from metaappscriptsdk.services.ExportService import ExportService @@ -33,11 +34,15 @@ class MetaApp(object): build_num = None starter_api_url = None meta_url = None + api_proxy_url = None starter = starter_api log = Logger() schedule = Schedule() worker = None - user_agent = None + + # Будет поставляться в конец UserAgent + user_agent_postfix = "" + developer_settings = None # Пользователь, из под которого пройдет авторизация после того, @@ -52,6 +57,7 @@ class MetaApp(object): UserManagementService = None StarterService = None MailService = None + ApiProxyService = None __default_headers = set() __db_list = {} @@ -59,6 +65,7 @@ class MetaApp(object): def __init__(self, service_id=None, debug=None, starter_api_url="http://STUB_URL", meta_url="https://meta.realweb.ru", + api_proxy_url="http://apiproxy.apis.kb.1ad.ru", include_worker=None ): if debug is None: @@ -69,20 +76,20 @@ def __init__(self, service_id=None, debug=None, if debug == 'false': debug = False self.debug = debug + self.api_proxy_url = api_proxy_url deprecated_logs = [] if service_id: deprecated_logs.append(u"Параметр service_id скоро будет удален из MetaApp") - service_ns = os.environ.get('SERVICE_NAMESPACE', "appscript") # для ns в логах + service_ns = os.environ.get('SERVICE_NAMESPACE', "appscript") # для ns в логах service_id = os.environ.get('SERVICE_ID', "local_debug_serivce") - self.build_num = os.environ.get('BUILD_NUM') + self.build_num = os.environ.get('BUILD_NUM', '0') self.service_id = service_id create_logger(service_id=service_id, service_ns=service_ns, build_num=self.build_num, debug=self.debug) self.__read_developer_settings() - self.user_agent = self.__build_user_agent() for deprecated_log_msg in deprecated_logs: self.log.warning("#" * 15) @@ -109,6 +116,7 @@ def __init__(self, service_id=None, debug=None, self.MailService = MailService(self, self.__default_headers) self.DbService = DbService(self, self.__default_headers) self.UserManagementService = UserManagementService(self, self.__default_headers) + self.ApiProxyService = ApiProxyService(self, self.__default_headers) if include_worker: if not debug: @@ -145,9 +153,9 @@ def db(self, db_alias, shard_key=None): self.__db_list[db_key] = DbQueryService(self, self.__default_headers, {"db_alias": db_alias, "dbAlias": db_alias, "shard_find_key": shard_key, "shardKey": shard_key}) return self.__db_list[db_key] - def __build_user_agent(self): - v = sys.version_info - return self.service_id + " | Python " + str(v.major) + "." + str(v.minor) + "." + str(v.micro) + " META SDK os:" + os.name + @property + def user_agent(self): + return self.service_id + " | b" + self.build_num + (' | ' + self.user_agent_postfix if self.user_agent_postfix else "") def __read_developer_settings(self): """ @@ -201,7 +209,7 @@ def api_call(self, service, method, data, options): "timeout": (60, 1800) } - for try_idx in range(8): + for try_idx in range(20): try: # Пока непонятно почему частенько получаем ошибку: # Failed to establish a new connection: [Errno 110] Connection timed out',)) @@ -260,7 +268,7 @@ def native_api_call(self, service, method, data, options, multipart_form=False, request['data'] = json.dumps(data) request['headers'] = _headers - for try_idx in range(8): + for try_idx in range(20): try: # Пока непонятно почему частенько получаем ошибку: # Failed to establish a new connection: [Errno 110] Connection timed out',)) diff --git a/metaappscriptsdk/examples/apiproxy/ga.py b/metaappscriptsdk/examples/apiproxy/ga.py new file mode 100644 index 0000000..2d532fd --- /dev/null +++ b/metaappscriptsdk/examples/apiproxy/ga.py @@ -0,0 +1,36 @@ +from metaappscriptsdk import MetaApp + +META = MetaApp() + + +refresh_token ="XXXXX" +ga_property_id = 'XXXX' + +resp = META.ApiProxyService.call_proxy("google_analytics", { + "refresh_token": refresh_token, + "version": "v4", + "method": "reports.batchGet", + "method_params": { + "body": { + # "useResourceQuotas": True, # Включение привелений GA360 + "reportRequests": [ + { + "viewId": ga_property_id, + "dateRanges": [ + { + "startDate": "2015-06-15", + "endDate": "2015-06-30" + }], + "metrics": [ + { + "expression": "ga:sessions" + }], + "dimensions": [ + { + "name": "ga:browser" + }] + }] + } + } +}, "native_call", False, []) +print(u"resp = %s" % str(resp.text)) diff --git a/metaappscriptsdk/exceptions.py b/metaappscriptsdk/exceptions.py index d3365f6..d518f30 100644 --- a/metaappscriptsdk/exceptions.py +++ b/metaappscriptsdk/exceptions.py @@ -48,3 +48,18 @@ def __str__(self): class UnexpectedResponseError(Exception): pass + + +class RetryHttpRequestError(Exception): + def __init__(self, err_details): + self.err_details = err_details + + +class ApiProxyError(Exception): + def __init__(self, err_details): + self.err_details = err_details + + +class EndOfTriesError(Exception): + def __init__(self, err_details=None): + self.err_details = err_details diff --git a/metaappscriptsdk/info.py b/metaappscriptsdk/info.py index 3b9b3e2..d2bdaf7 100644 --- a/metaappscriptsdk/info.py +++ b/metaappscriptsdk/info.py @@ -1,4 +1,4 @@ -__version__ = '0.5.37' +__version__ = '0.6.0' __package_name__ = 'metaappscriptsdk' if __name__ == "__main__": diff --git a/metaappscriptsdk/services/ApiProxyService.py b/metaappscriptsdk/services/ApiProxyService.py new file mode 100644 index 0000000..4435719 --- /dev/null +++ b/metaappscriptsdk/services/ApiProxyService.py @@ -0,0 +1,75 @@ +import json + +import requests +import time + +from metaappscriptsdk.exceptions import RetryHttpRequestError, EndOfTriesError, UnexpectedError, ApiProxyError + + +class ApiProxyService: + def __init__(self, app, default_headers): + """ + :type app: metaappscriptsdk.MetaApp + """ + self.__app = app + self.__default_headers = default_headers + self.__options = {} + self.__data_get_cache = {} + self.__data_get_flatten_cache = {} + + def call_proxy(self, engine, payload, method, analyze_json_error_param, retry_request_substr_variants, stream=False): + """ + :param engine: Система + :param payload: Данные для запроса + :param method: string Может содержать native_call | tsv | json_newline + :param analyze_json_error_param: Нужно ли производить анализ параметра error d jndtnt ghjrcb + :param retry_request_substr_variants: Список подстрок, при наличии которых в ответе будет происходить перезапрос + :param stream: + :return: + """ + log_ctx = {"engine": engine, "method": payload.get('method'), "method_params": payload.get('method_params')} + self.__app.log.info("Call api proxy", log_ctx) + body = { + "engine": engine, + "payload": payload + } + for try_idx in range(20): + try: + # 1h таймаут, так как бывают большие долгие данные, а лимит хоть какой-то нужен + body_str = json.dumps(body) + resp = requests.post(self.__app.api_proxy_url + "/" + method, body_str, timeout=3600, stream=stream, headers={ + "User-Agent": self.__app.user_agent + }) + + self.check_err(resp, analyze_json_error_param=analyze_json_error_param, retry_request_substr_variants=retry_request_substr_variants) + return resp + except RetryHttpRequestError as e: + self.__app.log.warning("Sleep retry query: " + str(e.err_details) if e.err_details else "", log_ctx) + time.sleep(20) + raise EndOfTriesError("Api of api proxy tries request") + + def check_err(self, resp, analyze_json_error_param=False, retry_request_substr_variants=None): + """ + :type retry_request_substr_variants: list Список вхождений строк, при налиции которых в ошидке апи будет произведен повторный запрос к апи + """ + if retry_request_substr_variants is None: + retry_request_substr_variants = [] + + # РКН блокировки вызывают ошибку SSL + retry_request_substr_variants.append("TLSV1_ALERT_ACCESS_DENIED") + + if resp.status_code >= 400: + rtext = resp.text + for v_ in retry_request_substr_variants: + if v_ in rtext: + raise RetryHttpRequestError(rtext) + raise UnexpectedError("HTTP request failed: {} {}".format(resp.status_code, rtext)) + if analyze_json_error_param: + data_ = resp.json() + if 'error' in data_ and data_.get('error'): + full_err_ = json.dumps(data_.get('error')) + for v_ in retry_request_substr_variants: + if v_ in full_err_: + raise RetryHttpRequestError(full_err_) + raise ApiProxyError(full_err_) + return resp diff --git a/metaappscriptsdk/services/__init__.py b/metaappscriptsdk/services/__init__.py index 727ce1d..0d6425e 100644 --- a/metaappscriptsdk/services/__init__.py +++ b/metaappscriptsdk/services/__init__.py @@ -1,4 +1,3 @@ -# coding=utf-8 from metaappscriptsdk.exceptions import AuthError, ServerError, RequestError, UnexpectedError, NoContentError