[+] ApiProxyService

metaappscriptsdk/__init__.py

@@ -13,6 +13,7 @@
 from metaappscriptsdk.logger.logger import Logger
 from metaappscriptsdk.schedule.Schedule import Schedule
 from metaappscriptsdk.services import get_api_call_headers, process_meta_api_error_code
+from metaappscriptsdk.services.ApiProxyService import ApiProxyService
 from metaappscriptsdk.services.DbQueryService import DbQueryService
 from metaappscriptsdk.services.DbService import DbService
 from metaappscriptsdk.services.ExportService import ExportService
@@ -33,11 +34,15 @@ class MetaApp(object):
     build_num = None
     starter_api_url = None
     meta_url = None
+    api_proxy_url = None
     starter = starter_api
     log = Logger()
     schedule = Schedule()
     worker = None
-    user_agent = None
+
+    # Будет поставляться в конец UserAgent
+    user_agent_postfix = ""
+
     developer_settings = None
 
     # Пользователь, из под которого пройдет авторизация после того,
@@ -52,13 +57,15 @@ class MetaApp(object):
     UserManagementService = None
     StarterService = None
     MailService = None
+    ApiProxyService = None
 
     __default_headers = set()
     __db_list = {}
 
     def __init__(self, service_id=None, debug=None,
                  starter_api_url="http://STUB_URL",
                  meta_url="https://meta.realweb.ru",
+                 api_proxy_url="http://apiproxy.apis.kb.1ad.ru",
                  include_worker=None
                  ):
         if debug is None:
@@ -69,20 +76,20 @@ def __init__(self, service_id=None, debug=None,
             if debug == 'false':
                 debug = False
         self.debug = debug
+        self.api_proxy_url = api_proxy_url
 
         deprecated_logs = []
 
         if service_id:
             deprecated_logs.append(u"Параметр service_id скоро будет удален из MetaApp")
 
-        service_ns = os.environ.get('SERVICE_NAMESPACE', "appscript") # для ns в логах
+        service_ns = os.environ.get('SERVICE_NAMESPACE', "appscript")  # для ns в логах
         service_id = os.environ.get('SERVICE_ID', "local_debug_serivce")
-        self.build_num = os.environ.get('BUILD_NUM')
+        self.build_num = os.environ.get('BUILD_NUM', '0')
         self.service_id = service_id
         create_logger(service_id=service_id, service_ns=service_ns, build_num=self.build_num, debug=self.debug)
 
         self.__read_developer_settings()
-        self.user_agent = self.__build_user_agent()
 
         for deprecated_log_msg in deprecated_logs:
             self.log.warning("#" * 15)
@@ -109,6 +116,7 @@ def __init__(self, service_id=None, debug=None,
         self.MailService = MailService(self, self.__default_headers)
         self.DbService = DbService(self, self.__default_headers)
         self.UserManagementService = UserManagementService(self, self.__default_headers)
+        self.ApiProxyService = ApiProxyService(self, self.__default_headers)
 
         if include_worker:
             if not debug:
@@ -145,9 +153,9 @@ def db(self, db_alias, shard_key=None):
             self.__db_list[db_key] = DbQueryService(self, self.__default_headers, {"db_alias": db_alias, "dbAlias": db_alias, "shard_find_key": shard_key, "shardKey": shard_key})
         return self.__db_list[db_key]
 
-    def __build_user_agent(self):
-        v = sys.version_info
-        return self.service_id + " | Python " + str(v.major) + "." + str(v.minor) + "." + str(v.micro) + " META SDK os:" + os.name
+    @property
+    def user_agent(self):
+        return self.service_id + " | b" + self.build_num + (' | ' + self.user_agent_postfix if self.user_agent_postfix else "")
 
     def __read_developer_settings(self):
         """
@@ -201,7 +209,7 @@ def api_call(self, service, method, data, options):
             "timeout": (60, 1800)
         }
 
-        for try_idx in range(8):
+        for try_idx in range(20):
             try:
                 # Пока непонятно почему частенько получаем ошибку:
                 # Failed to establish a new connection: [Errno 110] Connection timed out',))
@@ -260,7 +268,7 @@ def native_api_call(self, service, method, data, options, multipart_form=False,
             request['data'] = json.dumps(data)
         request['headers'] = _headers
 
-        for try_idx in range(8):
+        for try_idx in range(20):
             try:
                 # Пока непонятно почему частенько получаем ошибку:
                 # Failed to establish a new connection: [Errno 110] Connection timed out',))

metaappscriptsdk/examples/apiproxy/ga.py

@@ -0,0 +1,36 @@
+from metaappscriptsdk import MetaApp
+
+META = MetaApp()
+
+
+refresh_token ="XXXXX"
+ga_property_id = 'XXXX'
+
+resp = META.ApiProxyService.call_proxy("google_analytics", {
+    "refresh_token": refresh_token,
+    "version": "v4",
+    "method": "reports.batchGet",
+    "method_params": {
+        "body": {
+            # "useResourceQuotas": True, # Включение привелений GA360
+            "reportRequests": [
+                {
+                    "viewId": ga_property_id,
+                    "dateRanges": [
+                        {
+                            "startDate": "2015-06-15",
+                            "endDate": "2015-06-30"
+                        }],
+                    "metrics": [
+                        {
+                            "expression": "ga:sessions"
+                        }],
+                    "dimensions": [
+                        {
+                            "name": "ga:browser"
+                        }]
+                }]
+        }
+    }
+}, "native_call", False, [])
+print(u"resp = %s" % str(resp.text))

metaappscriptsdk/exceptions.py

@@ -48,3 +48,18 @@ def __str__(self):
 
 class UnexpectedResponseError(Exception):
     pass
+
+
+class RetryHttpRequestError(Exception):
+    def __init__(self, err_details):
+        self.err_details = err_details
+
+
+class ApiProxyError(Exception):
+    def __init__(self, err_details):
+        self.err_details = err_details
+
+
+class EndOfTriesError(Exception):
+    def __init__(self, err_details=None):
+        self.err_details = err_details

metaappscriptsdk/info.py

@@ -1,4 +1,4 @@
-__version__ = '0.5.37'
+__version__ = '0.6.0'
 __package_name__ = 'metaappscriptsdk'
 
 if __name__ == "__main__":

metaappscriptsdk/services/ApiProxyService.py

@@ -0,0 +1,75 @@
+import json
+
+import requests
+import time
+
+from metaappscriptsdk.exceptions import RetryHttpRequestError, EndOfTriesError, UnexpectedError, ApiProxyError
+
+
+class ApiProxyService:
+    def __init__(self, app, default_headers):
+        """
+        :type app: metaappscriptsdk.MetaApp
+        """
+        self.__app = app
+        self.__default_headers = default_headers
+        self.__options = {}
+        self.__data_get_cache = {}
+        self.__data_get_flatten_cache = {}
+
+    def call_proxy(self, engine, payload, method, analyze_json_error_param, retry_request_substr_variants, stream=False):
+        """
+        :param engine: Система
+        :param payload: Данные для запроса
+        :param method: string Может содержать native_call | tsv | json_newline
+        :param analyze_json_error_param: Нужно ли производить анализ параметра error d jndtnt ghjrcb
+        :param retry_request_substr_variants: Список подстрок, при наличии которых в ответе будет происходить перезапрос
+        :param stream:
+        :return:
+        """
+        log_ctx = {"engine": engine, "method": payload.get('method'), "method_params": payload.get('method_params')}
+        self.__app.log.info("Call api proxy", log_ctx)
+        body = {
+            "engine": engine,
+            "payload": payload
+        }
+        for try_idx in range(20):
+            try:
+                # 1h таймаут, так как бывают большие долгие данные, а лимит хоть какой-то нужен
+                body_str = json.dumps(body)
+                resp = requests.post(self.__app.api_proxy_url + "/" + method, body_str, timeout=3600, stream=stream, headers={
+                    "User-Agent": self.__app.user_agent
+                })
+
+                self.check_err(resp, analyze_json_error_param=analyze_json_error_param, retry_request_substr_variants=retry_request_substr_variants)
+                return resp
+            except RetryHttpRequestError as e:
+                self.__app.log.warning("Sleep retry query: " + str(e.err_details) if e.err_details else "", log_ctx)
+                time.sleep(20)
+        raise EndOfTriesError("Api of api proxy tries request")
+
+    def check_err(self, resp, analyze_json_error_param=False, retry_request_substr_variants=None):
+        """
+        :type retry_request_substr_variants: list Список вхождений строк, при налиции которых в ошидке апи будет произведен повторный запрос к апи
+        """
+        if retry_request_substr_variants is None:
+            retry_request_substr_variants = []
+
+        # РКН блокировки вызывают ошибку SSL
+        retry_request_substr_variants.append("TLSV1_ALERT_ACCESS_DENIED")
+
+        if resp.status_code >= 400:
+            rtext = resp.text
+            for v_ in retry_request_substr_variants:
+                if v_ in rtext:
+                    raise RetryHttpRequestError(rtext)
+            raise UnexpectedError("HTTP request failed: {} {}".format(resp.status_code, rtext))
+        if analyze_json_error_param:
+            data_ = resp.json()
+            if 'error' in data_ and data_.get('error'):
+                full_err_ = json.dumps(data_.get('error'))
+                for v_ in retry_request_substr_variants:
+                    if v_ in full_err_:
+                        raise RetryHttpRequestError(full_err_)
+                raise ApiProxyError(full_err_)
+        return resp

metaappscriptsdk/services/__init__.py

@@ -1,4 +1,3 @@
-# coding=utf-8
 from metaappscriptsdk.exceptions import AuthError, ServerError, RequestError, UnexpectedError, NoContentError