From ebe709ef6c03ab45f409cb7b5a23ff9c6f992269 Mon Sep 17 00:00:00 2001 From: Peter Hu Date: Thu, 11 May 2023 15:07:15 -0700 Subject: [PATCH] remove ci_common_utils (#6542) --- .../ci_common_utils/__init__.py | 7 -- .../ci_common_utils/google_api.py | 88 ------------------- .../ci_common_utils/ci_common_utils/logger.py | 81 ----------------- tools/ci_common_utils/setup.py | 24 ----- tools/ci_common_utils/tests/__init__.py | 0 tools/ci_common_utils/tests/test_logger.py | 45 ---------- 6 files changed, 245 deletions(-) delete mode 100644 tools/ci_common_utils/ci_common_utils/__init__.py delete mode 100644 tools/ci_common_utils/ci_common_utils/google_api.py delete mode 100644 tools/ci_common_utils/ci_common_utils/logger.py delete mode 100644 tools/ci_common_utils/setup.py delete mode 100644 tools/ci_common_utils/tests/__init__.py delete mode 100644 tools/ci_common_utils/tests/test_logger.py diff --git a/tools/ci_common_utils/ci_common_utils/__init__.py b/tools/ci_common_utils/ci_common_utils/__init__.py deleted file mode 100644 index 0078efdb784..00000000000 --- a/tools/ci_common_utils/ci_common_utils/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from .google_api import GoogleApi -from .logger import Logger - -__all__ = ( - "Logger", - "GoogleApi", -) diff --git a/tools/ci_common_utils/ci_common_utils/google_api.py b/tools/ci_common_utils/ci_common_utils/google_api.py deleted file mode 100644 index 2ce4ef2a0f7..00000000000 --- a/tools/ci_common_utils/ci_common_utils/google_api.py +++ /dev/null @@ -1,88 +0,0 @@ -import time -from dataclasses import dataclass -from typing import Mapping, Any, List, ClassVar - -import jwt -import requests - -from .logger import Logger - -TOKEN_TTL = 3600 - - -@dataclass -class GoogleApi: - """ - Simple Google API client - """ - logger: ClassVar[Logger] = Logger() - - config: Mapping[str, Any] - scopes: List[str] - _access_token: str = None - - def get(self, url: str, params: Mapping = None) -> Mapping[str, Any]: - """Sends a GET request""" - token = self.get_access_token() - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - "X-Goog-User-Project": self.project_id - } - # Making a get request - response = requests.get(url, headers=headers, params=params) - response.raise_for_status() - return response.json() - - def post(self, url: str, json: Mapping = None, params: Mapping = None) -> Mapping[str, Any]: - """Sends a POST request""" - token = self.get_access_token() - - headers = { - "Authorization": f"Bearer {token}", - "X-Goog-User-Project": self.project_id - } - # Making a get request - response = requests.post(url, headers=headers, json=json, params=params) - try: - response.raise_for_status() - except Exception: - self.logger.error(f"error body: {response.text}") - raise - return response.json() - - @property - def token_uri(self): - return self.config["token_uri"] - - @property - def project_id(self): - return self.config["project_id"] - - def __generate_jwt(self) -> str: - """Generates JWT token by a service account json file and scopes""" - now = int(time.time()) - claim = { - "iat": now, - "iss": self.config["client_email"], - "scope": ",".join(self.scopes), - "aud": self.token_uri, - "exp": now + TOKEN_TTL, - } - return jwt.encode(claim, self.config["private_key"].encode(), algorithm="RS256") - - def get_access_token(self) -> str: - """Generates an access token by a service account json file and scopes""" - - if self._access_token is None: - self._access_token = self.__get_access_token() - - return self._access_token - - def __get_access_token(self) -> str: - jwt = self.__generate_jwt() - resp = requests.post(self.token_uri, data={ - "assertion": jwt, - "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", - }) - return resp.json()["access_token"] diff --git a/tools/ci_common_utils/ci_common_utils/logger.py b/tools/ci_common_utils/ci_common_utils/logger.py deleted file mode 100644 index 00ec0a7c716..00000000000 --- a/tools/ci_common_utils/ci_common_utils/logger.py +++ /dev/null @@ -1,81 +0,0 @@ -import datetime as dt -import inspect -import logging -import logging.handlers -import sys -from typing import Callable - - -class MyFormatter(logging.Formatter): - """Custom formatter for logging - """ - converter = dt.datetime.fromtimestamp - - def formatTime(self, record, datefmt=None): - """! @brief redefinition of format of log - """ - ct = self.converter(record.created) - if datefmt: - s = ct.strftime(datefmt) - else: - t = ct.strftime("%Y-%m-%d %H:%M:%S") - s = "%s,%03d" % (t, record.msecs) - return s - - -class Logger: - """Simple logger with a pretty log header - the method error returns the value 1 - the method critical terminates a script work - """ - - def __init__(self): - formatter = MyFormatter( - fmt='[%(asctime)s] - %(levelname)-6s - %(message)s', - datefmt='%d/%m/%Y %H:%M:%S.%f') - - logger_name = __name__ - stack_items = inspect.stack() - for i in range(len(stack_items)): - if stack_items[i].filename.endswith("ci_common_utils/logger.py"): - logger_name = ".".join(stack_items[i + 1].filename.split("/")[-3:])[:-3] - - self._logger = logging.getLogger(logger_name) - self._logger.setLevel(logging.DEBUG) - self._logger.propagate = False - - handler = logging.StreamHandler() - handler.setLevel(logging.DEBUG) - handler.setFormatter(formatter) - self._logger.addHandler(handler) - - @classmethod - def __prepare_log_line(cls, func_name: str, func: Callable) -> Callable: - def wrapper(*args): - prefix = "" - stack_items = inspect.stack() - for i in range(len(stack_items)): - if stack_items[i].filename.endswith("ci_common_utils/logger.py"): - filepath = stack_items[i + 1].filename - line_number = stack_items[i + 1].lineno - - # show last 3 path items only - filepath = "/".join(filepath.split("/")[-3:]) - prefix = f"[{filepath}:{line_number}]" - break - if prefix: - args = list(args) - args[0] = f"{prefix} # {args[0]}" - func(*args) - if func_name == "critical": - sys.exit(1) - elif func_name == "error": - return 1 - return 0 - - return wrapper - - def __getattr__(self, function_name: str): - if not hasattr(self._logger, function_name): - return super().__getattr__(function_name) - return self.__prepare_log_line(function_name, getattr(self._logger, function_name), ) diff --git a/tools/ci_common_utils/setup.py b/tools/ci_common_utils/setup.py deleted file mode 100644 index 2c5aa9692ec..00000000000 --- a/tools/ci_common_utils/setup.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["cryptography", "requests", "pyjwt~=2.3.0"] - -TEST_REQUIREMENTS = ["requests-mock"] - -setup( - version="0.0.0", - name="ci_common_utils", - description="Suite of all often used classes and common functions", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - python_requires=">=3.9", - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/tools/ci_common_utils/tests/__init__.py b/tools/ci_common_utils/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/ci_common_utils/tests/test_logger.py b/tools/ci_common_utils/tests/test_logger.py deleted file mode 100644 index e45464550ed..00000000000 --- a/tools/ci_common_utils/tests/test_logger.py +++ /dev/null @@ -1,45 +0,0 @@ -import re -from datetime import datetime, timedelta - -import pytest - -from ci_common_utils import Logger - -LOG_RE = re.compile( - r'^\[(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\.\d{6})\] -' - r'\s+(\w+)\s+- \[.*tests/test_logger.py:(\d+)\] # (.+)') -LOGGER = Logger() -TEST_MESSAGE = 'sbhY=)9\'v-}LT=)jjF66(XrZh=]>7Xp"?/zCz,=eu8K47u8' - - -def check_output(msg: str, expected_line_number: int, expected_log_level: str): - m = LOG_RE.match(msg) - assert m is not None, f"incorrect message format, pattern: {LOG_RE.pattern}" - date_time, log_level, line_number, msg = m.groups() - - assert int(line_number) == expected_line_number - assert expected_log_level == log_level - assert expected_log_level == log_level - dt = datetime.strptime(date_time, '%d/%m/%Y %H:%M:%S.%f') - now = datetime.now() - delta = timedelta(seconds=1) - assert now - delta < dt < now - - -@pytest.mark.parametrize('log_func,expected_log_level,expected_code', ( - (LOGGER.debug, 'DEBUG', 0), - (LOGGER.warning, 'WARNING', 0), - (LOGGER.info, 'INFO', 0), - (LOGGER.error, 'ERROR', 1) -)) -def test_log_message(capfd, log_func, expected_log_level, expected_code): - assert log_func(TEST_MESSAGE) == expected_code - _, err = capfd.readouterr() - check_output(err, 36, expected_log_level) - - -def test_critical_message(capfd): - with pytest.raises(SystemExit) as (err): - LOGGER.critical(TEST_MESSAGE) - _, err = capfd.readouterr() - check_output(err, 43, 'CRITICAL')