diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..82d79bc --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,6 @@ +## Describe your changes + +## Checklist before requesting a review +- [ ] I have performed a self-review of my code +- [ ] If it is a core feature, I have added thorough tests. + diff --git a/README.md b/README.md index 9b25dd2..2801b5d 100644 --- a/README.md +++ b/README.md @@ -1 +1,65 @@ -# plugin-github-inven-collector \ No newline at end of file +# plugin-github-inven-collector + +## Introduction +**plugin-spaceone-inven-collector** is an Inventory Collector Plugin for SpaceONE, designed to collect resources from Github. +This plugin has been developed with a new plugin framework. + +## Setup and Run +Since the current spaceone-inventory package is in a pre-release state, follow the steps below to configure the package. + +### 1. Virtual Environment Setup +Set up a virtual environment using the venv library. +```bash +python3 -m venv venv +``` +```bash +source venv/bin/activate +``` + +### 2. Package Installation +Install the package with the following commands in the created virtual environment. + +```bash +pip3 install -r pkg/pip_requirements.txt +pip3 install --pre spaceone-inventory +``` + +### 3. Interpreter Configuration +If you are using PyCharm, configure the virtual environment as the interpreter. +![img.png](docs/interpreter_settings.png) +![img.png](docs/settings_source_directory.png) +![img.png](docs/run_settings.png) + +After following the above steps, run the Plugin Server. +![img.png](docs/run_plugin_server.png) + +## Local Environment Testing +After running the Plugin Server, **perform tests for each method** with the following commands. + +### Check available API methods: + +```bash +spacectl api-resources +``` + +#### Collector.init +``` +spacectl exec init inventory.Collector -f test/init.yml +``` + + +#### Collector.verify + +```bash +spacectl exec verify inventory.Collector -f test/verify.yml +``` + +#### Collector.collect + +```bash +spacectl exec collect inventory.Collector -f test/collect.yml +``` + +#### Note +Metadata will be defined as a dictionary and will be converted to YAML. +The spaceone-inventory package is in a pre-release state, so the `--pre` option must be added when using pip install. \ No newline at end of file diff --git a/docs/interpreter_settings.png b/docs/interpreter_settings.png new file mode 100644 index 0000000..52187d2 Binary files /dev/null and b/docs/interpreter_settings.png differ diff --git a/docs/run_plugin_server.png b/docs/run_plugin_server.png new file mode 100644 index 0000000..f830b65 Binary files /dev/null and b/docs/run_plugin_server.png differ diff --git a/docs/run_settings.png b/docs/run_settings.png new file mode 100644 index 0000000..8dae677 Binary files /dev/null and b/docs/run_settings.png differ diff --git a/docs/settings_source_directory.png b/docs/settings_source_directory.png new file mode 100644 index 0000000..fdbf8b2 Binary files /dev/null and b/docs/settings_source_directory.png differ diff --git a/pkg/pip_requirements.txt b/pkg/pip_requirements.txt index fc9497f..304ee56 100644 --- a/pkg/pip_requirements.txt +++ b/pkg/pip_requirements.txt @@ -1,2 +1 @@ -spaceone-core spaceone-api \ No newline at end of file diff --git a/src/plugin/connector/__init__.py b/src/plugin/connector/__init__.py new file mode 100644 index 0000000..2e9fcce --- /dev/null +++ b/src/plugin/connector/__init__.py @@ -0,0 +1,61 @@ +import logging +import requests + +from spaceone.core.connector import BaseConnector + +__all__ = ["RequestConnector"] + +_LOGGER = logging.getLogger(__name__) + + +class RequestConnector(BaseConnector): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def send_request( + self, url, headers, body=dict, method="GET", page=None, per_page=30 + ): + try: + if page: + for response in self._pagination(url, headers, body, per_page, page): + yield response + else: + response = requests.get(url, headers=headers).json() + yield response + except Exception as e: + _LOGGER.error(f"Request Error: {e}") + raise e + + @staticmethod + def _pagination(url, headers, body, per_page, page): + responses = [] + while True: + if url in "&": + url = f"{url}&per_page={per_page}&page={page}" + else: + url = f"{url}?per_page={per_page}&page={page}" + response = requests.get(url, headers=headers) + response_json = response.json() + if not response_json: + break + page += 1 + responses.extend(response_json) + return responses + + @staticmethod + def make_header(secret_data): + github_token = secret_data.get("github_token") + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {github_token}", + "X-GitHub-Api-Version": "2022-11-28" + } + return headers + + @staticmethod + def make_header_dockerhub(secret_data): + dockerhub_token = secret_data.get("dockerhub_token") + headers = { + "Authorization": f"Bearer {dockerhub_token}", + } + return headers diff --git a/src/plugin/connector/dockerhub_connector.py b/src/plugin/connector/dockerhub_connector.py new file mode 100644 index 0000000..b0ca0f9 --- /dev/null +++ b/src/plugin/connector/dockerhub_connector.py @@ -0,0 +1,27 @@ +import abc +import requests +import logging + +from plugin.connector import RequestConnector + +_LOGGER = logging.getLogger(__name__) + + +class DockerhubConnector(RequestConnector): + + def list_tags(self, namespace, repository, secret_data): + url = f'https://hub.docker.com/v2/namespaces/{namespace}/repositories/{repository}/tags' + headers = self.make_header_dockerhub(secret_data) + + try: + response = self.send_request(url, headers=headers) + _response = list(response) + if _response and _response[0].get('results'): + results = _response[0].get('results') + + return results + else: + return [] + except Exception as e: + _LOGGER.error(f"Request Error: {e}") + raise e diff --git a/src/plugin/connector/pypi/__init__.py b/src/plugin/connector/pypi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/connector/repository_connector.py b/src/plugin/connector/repository_connector.py new file mode 100644 index 0000000..b86e5d1 --- /dev/null +++ b/src/plugin/connector/repository_connector.py @@ -0,0 +1,40 @@ +import logging + +from plugin.connector import RequestConnector + +_LOGGER = logging.getLogger(__name__) + + +class RepositoryConnector(RequestConnector): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def list_organization_repos(self, secret_data) -> list: + headers = self.make_header(secret_data) + url = f"https://api.github.com/orgs/{secret_data.get('organization_name')}/repos" + try: + response = self.send_request(url, headers=headers, page=1) + return response + except Exception as e: + _LOGGER.error(f"Request Error: {e}") + raise e + + def list_repo_tags(self, repo_name, secret_data, page=1) -> list: + headers = self.make_header(secret_data) + url = f"https://api.github.com/repos/{secret_data.get('organization_name')}/{repo_name}/tags" + try: + response = self.send_request(url, headers=headers, page=page, per_page=100) + return response + except Exception as e: + _LOGGER.error(f"Request Error: {e}") + raise e + + def list_repo_topics(self, repo_name, secret_data): + headers = self.make_header(secret_data) + url = f"https://api.github.com/repos/{secret_data.get('organization_name')}/{repo_name}/topics" + try: + response = self.send_request(url, headers=headers) + return response + except Exception as e: + _LOGGER.error(f"Request Error: {e}") + raise e diff --git a/src/plugin/main.py b/src/plugin/main.py index 47466e8..1dbc465 100644 --- a/src/plugin/main.py +++ b/src/plugin/main.py @@ -1,66 +1,42 @@ +import logging + from spaceone.inventory.plugin.collector.lib.server import CollectorPluginServer +from plugin.manager.repository_manager import RepositoryManager +_LOGGER = logging.getLogger('cloudforet') app = CollectorPluginServer() @app.route('Collector.init') def collector_init(params: dict) -> dict: - """ init plugin by options - - Args: - params (CollectorInitRequest): { - 'options': 'dict', # Required - 'domain_id': 'str' - } - - Returns: - PluginResponse: { - 'metadata': 'dict' - } - """ - pass - - -@app.route('Collector.verify') -def collector_verify(params: dict) -> None: - """ Verifying collector plugin - - Args: - params (CollectorVerifyRequest): { - 'options': 'dict', # Required - 'secret_data': 'dict', # Required - 'schema': 'str', - 'domain_id': 'str' - } - - Returns: - None - """ - pass + return {'metadata': {'options_schema': _create_options_schema()}} @app.route('Collector.collect') def collector_collect(params: dict) -> dict: - """ Collect external data - - Args: - params (CollectorCollectRequest): { - 'options': 'dict', # Required - 'secret_data': 'dict', # Required - 'schema': 'str', - 'domain_id': 'str' - } - - Returns: - Generator[ResourceResponse, None, None] - { - 'state': 'SUCCESS | FAILURE', - 'resource_type': 'inventory.CloudService | inventory.CloudServiceType | inventory.Region', - 'resource_data': 'dict', - 'match_keys': 'list', - 'error_message': 'str' - 'metadata': 'dict' + options = params['options'] + secret_data = params['secret_data'] + schema = params.get('schema') + + repository_manager = RepositoryManager() + return repository_manager.collect_resources(options, secret_data, schema) + + +def _create_options_schema(): + return { + 'required': ['items'], + 'order': ['items'], + 'type': 'object', + 'properties': { + 'items': { + 'title': 'Item filter', + 'type': 'array', + 'items': { + 'enum': [ + 'fields' + ] + } + } } - """ - pass + } diff --git a/src/plugin/manager/__init__.py b/src/plugin/manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/manager/repository_manager.py b/src/plugin/manager/repository_manager.py new file mode 100644 index 0000000..0a00f7a --- /dev/null +++ b/src/plugin/manager/repository_manager.py @@ -0,0 +1,131 @@ +import logging +import re +from packaging import version +from spaceone.inventory.plugin.collector.lib import * +from plugin.connector.repository_connector import RepositoryConnector +from plugin.connector.dockerhub_connector import DockerhubConnector + +_LOGGER = logging.getLogger(__name__) + + +class RepositoryManager: + def __init__(self, **kwargs): + self.connector: RepositoryConnector = RepositoryConnector(**kwargs) + + self.cloud_service_group = 'Repository' + self.cloud_service_type = 'Repository' + self.provider = 'github_samuel_v3' + self.metadata_path = 'plugin/metadata/repository/repository.yaml' + + def collect_resources(self, options, secret_data, schema): + try: + yield from self.collect_cloud_service_type(options, secret_data, schema) + yield from self.collect_cloud_service(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + + def collect_cloud_service_type(self, options, secret_data, schema): + cloud_service_type = make_cloud_service_type( + name=self.cloud_service_type, + group=self.cloud_service_group, + provider=self.provider, + metadata_path=self.metadata_path, + is_primary=True, + is_major=True + ) + + yield make_response( + cloud_service_type=cloud_service_type, + match_keys=[['name', 'reference.resource_id', 'account', 'provider']], + resource_type='inventory.CloudServiceType' + ) + + def collect_cloud_service(self, options, secret_data, schema): + repository_connector = RepositoryConnector() + repo_items = repository_connector.list_organization_repos(secret_data) + for item in repo_items: + repo_name = item.get('name') + item['github_tag'] = self.get_latest_tag(repo_name, secret_data) + item['dev_dockerhub_tag'] = self.get_latest_dockerhub_tag( + secret_data['dev_dockerhub_name'], repo_name, secret_data + ) + item['prod_dockerhub_tag'] = self.get_latest_dockerhub_tag( + secret_data['prod_dockerhub_name'], repo_name, secret_data + ) + item['pypi_tag'] = self.get_latest_pypi_tag(item['name'], secret_data) + item['type'] = self.get_repo_type_by_topics(item['topics']) + item['topics'] = item['topics'] + cloud_service = make_cloud_service( + name=repo_name, + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + data=item, + ) + yield make_response( + cloud_service=cloud_service, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + ) + + def get_latest_tag(self, repo_name, secret_data) -> str: + all_tag_items = [] + page = 1 + repository_connector = RepositoryConnector() + + while True: + tag_items = repository_connector.list_repo_tags(repo_name, secret_data, page) + tags = list(tag_items) + if not tags: + break + + all_tag_items.extend(tags) + page += 1 + + if not all_tag_items: + return '' + + versions = [{'version': version.parse(self.extract_version(item['name'])), 'name': item['name']} for item in + all_tag_items if + self.extract_version(item['name'])] + latest_version = max(versions, key=lambda x: x['version'])['name'] + return str(latest_version) + + @staticmethod + def get_latest_dockerhub_tag(namespace, repo_name, secret_data) -> str: + dockerhub_connector = DockerhubConnector() + tag_items = dockerhub_connector.list_tags(namespace, repo_name, secret_data) + + tag_items.sort(key=lambda x: x['last_updated'], reverse=True) + + if not tag_items: + return '' + + return tag_items[0]['name'] + + @staticmethod + def get_latest_pypi_tag(repo_name, secret_data) -> str: + return '1.0.0' + + @staticmethod + def get_repo_type_by_topics(topics: list): + if 'plugin' in topics: + return 'plugin' + elif 'core' in topics: + return 'core' + elif 'doc' in topics: + return 'doc' + elif 'spacectl' in topics: + return 'tools' + else: + return 'common' + + @staticmethod + def extract_version(tag: str): + version_pattern = re.compile(r'\d+\.\d+(?:\.\d+)?') + match = version_pattern.search(tag) + return match.group() if match else None diff --git a/src/plugin/metadata/__init__.py b/src/plugin/metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/metadata/repository/__init__.py b/src/plugin/metadata/repository/__init__.py new file mode 100644 index 0000000..f13d50b --- /dev/null +++ b/src/plugin/metadata/repository/__init__.py @@ -0,0 +1,36 @@ +# +# Copyright 2020 The SpaceONE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from setuptools import setup, find_packages + +with open("VERSION", "r") as f: + VERSION = f.read().strip() + f.close() + +setup( + name="plugin-github-inven-collector", + version=VERSION, + description="Collector plugin for Github", + long_description="", + url="https://www.spaceone.dev/", + author="SpaceONE Study Group", + author_email="admin@spaceone.dev", + license="Apache License 2.0", + packages=find_packages(), + install_requires=["spaceone-core", "spaceone-api", "spaceone-inventory"], + package_data={"plugin": ["metadata/repository/*.yaml"]}, + zip_safe=False, +) diff --git a/src/plugin/metadata/repository/repository.yaml b/src/plugin/metadata/repository/repository.yaml new file mode 100644 index 0000000..69835a0 --- /dev/null +++ b/src/plugin/metadata/repository/repository.yaml @@ -0,0 +1,18 @@ +--- +search: + fields: + - Name: data.name + - Type: data.type + - Topics: data.topics + +table: + sort: + key: data.name + desc: true + fields: + - Type: data.type + - GitHub Tag: data.github_tag + - Dev DockerHub Tag: data.dev_dockerhub_tag + - Prod DockerHub Tag: data.prod_dockerhub_tag + - Pypi Tag: data.pypi_tag + - Topics: data.topics \ No newline at end of file diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..3e068c9 --- /dev/null +++ b/src/setup.py @@ -0,0 +1,36 @@ +# +# Copyright 2020 The SpaceONE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from setuptools import setup, find_packages + +with open("VERSION", "r") as f: + VERSION = f.read().strip() + f.close() + +setup( + name="plugin-github-inven-collector", + version=VERSION, + description="Forest Extension Practice", + long_description="", + url="https://www.spaceone.dev/", + author="MEGAZONE SpaceONE Team", + author_email="admin@spaceone.dev", + license="Apache License 2.0", + packages=find_packages(), + install_requires=["spaceone-core", "spaceone-api", "spaceone-inventory"], + package_data={"plugin": ["metadata/repository/*.yaml"]}, + zip_safe=False, +)