diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..82d79bc --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,6 @@ +## Describe your changes + +## Checklist before requesting a review +- [ ] I have performed a self-review of my code +- [ ] If it is a core feature, I have added thorough tests. + diff --git a/README.md b/README.md index 9b25dd2..2801b5d 100644 --- a/README.md +++ b/README.md @@ -1 +1,65 @@ -# plugin-github-inven-collector \ No newline at end of file +# plugin-github-inven-collector + +## Introduction +**plugin-spaceone-inven-collector** is an Inventory Collector Plugin for SpaceONE, designed to collect resources from Github. +This plugin has been developed with a new plugin framework. + +## Setup and Run +Since the current spaceone-inventory package is in a pre-release state, follow the steps below to configure the package. + +### 1. Virtual Environment Setup +Set up a virtual environment using the venv library. +```bash +python3 -m venv venv +``` +```bash +source venv/bin/activate +``` + +### 2. Package Installation +Install the package with the following commands in the created virtual environment. + +```bash +pip3 install -r pkg/pip_requirements.txt +pip3 install --pre spaceone-inventory +``` + +### 3. Interpreter Configuration +If you are using PyCharm, configure the virtual environment as the interpreter. +![img.png](docs/interpreter_settings.png) +![img.png](docs/settings_source_directory.png) +![img.png](docs/run_settings.png) + +After following the above steps, run the Plugin Server. +![img.png](docs/run_plugin_server.png) + +## Local Environment Testing +After running the Plugin Server, **perform tests for each method** with the following commands. + +### Check available API methods: + +```bash +spacectl api-resources +``` + +#### Collector.init +``` +spacectl exec init inventory.Collector -f test/init.yml +``` + + +#### Collector.verify + +```bash +spacectl exec verify inventory.Collector -f test/verify.yml +``` + +#### Collector.collect + +```bash +spacectl exec collect inventory.Collector -f test/collect.yml +``` + +#### Note +Metadata will be defined as a dictionary and will be converted to YAML. +The spaceone-inventory package is in a pre-release state, so the `--pre` option must be added when using pip install. \ No newline at end of file diff --git a/docs/interpreter_settings.png b/docs/interpreter_settings.png new file mode 100644 index 0000000..52187d2 Binary files /dev/null and b/docs/interpreter_settings.png differ diff --git a/docs/run_plugin_server.png b/docs/run_plugin_server.png new file mode 100644 index 0000000..f830b65 Binary files /dev/null and b/docs/run_plugin_server.png differ diff --git a/docs/run_settings.png b/docs/run_settings.png new file mode 100644 index 0000000..8dae677 Binary files /dev/null and b/docs/run_settings.png differ diff --git a/docs/settings_source_directory.png b/docs/settings_source_directory.png new file mode 100644 index 0000000..fdbf8b2 Binary files /dev/null and b/docs/settings_source_directory.png differ diff --git a/src/VERSION b/src/VERSION new file mode 100644 index 0000000..0ec25f7 --- /dev/null +++ b/src/VERSION @@ -0,0 +1 @@ +v1.0.0 diff --git a/src/plugin/connector/__init__.py b/src/plugin/connector/__init__.py new file mode 100644 index 0000000..493cbdd --- /dev/null +++ b/src/plugin/connector/__init__.py @@ -0,0 +1,74 @@ +import logging +from spaceone.core.connector import BaseConnector +import requests + +__all__ = ["GitHubConnector"] + +_LOGGER = logging.getLogger(__name__) + +class GitHubConnector(BaseConnector): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @staticmethod + def send_request(url, headers, params=None, method="GET", page=None, per_page=30): + try: + if page: + for response in GitHubConnector._pagination(url, headers, params, per_page, page): + yield response + else: + response = GitHubConnector._make_request(url, headers, params, method) + yield response + except Exception as e: + _LOGGER.error(f"Request Error: {e}") + raise e + + @staticmethod + def _make_request(url, headers, params=None, method="GET"): + response = None + try: + if method == "GET": + response = requests.get(url, headers=headers, params=params) + + response.raise_for_status() + response_json = response.json() + + return response_json + except requests.exceptions.HTTPError as errh: + _LOGGER.error(f"HTTP Error: {errh}") + except requests.exceptions.ConnectionError as errc: + _LOGGER.error(f"Error Connecting: {errc}") + except requests.exceptions.Timeout as errt: + _LOGGER.error(f"Timeout Error: {errt}") + except requests.exceptions.RequestException as err: + _LOGGER.error(f"Request Error: {err}") + + if response and not response.content: + _LOGGER.warning(f"Non-JSON response received: {response.content}") + + return None + + @staticmethod + def _pagination(url, headers, params, per_page, page): + responses = [] + while True: + paginated_url = f"{url}{'&' if '&' in url else '?'}per_page={per_page}&page={page}" + response_json = GitHubConnector._make_request(paginated_url, headers, params) + + if not response_json: + break + + page += 1 + responses.extend(response_json) + + return responses + + @staticmethod + def make_header(secret_data): + github_token = secret_data.get("github_token") + return { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {github_token}", + "X-GitHub-Api-Version": "2022-11-28", + } diff --git a/src/plugin/connector/repo_connector.py b/src/plugin/connector/repo_connector.py new file mode 100644 index 0000000..500833f --- /dev/null +++ b/src/plugin/connector/repo_connector.py @@ -0,0 +1,20 @@ +from . import GitHubConnector + + +class RepoConnector(GitHubConnector): + + def get_repositories(self, secret_data): + org_name = secret_data.get("org_name") + url = f"https://api.github.com/orgs/{org_name}/repos" + headers = self.make_header(secret_data) + return self.send_request(url, headers, page=1000) + + def get_repo_issues(self, secret_data, repo_owner, repo_name): + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/issues" + headers = self.make_header(secret_data) + return self.send_request(url, headers, page=1000) + + def get_repo_pulls(self, secret_data, repo_owner, repo_name): + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/pulls" + headers = self.make_header(secret_data) + return self.send_request(url, headers, page=1000) diff --git a/src/plugin/main.py b/src/plugin/main.py index 47466e8..923dcbc 100644 --- a/src/plugin/main.py +++ b/src/plugin/main.py @@ -1,66 +1,39 @@ from spaceone.inventory.plugin.collector.lib.server import CollectorPluginServer +from plugin.manager.repo_manager import RepoManager app = CollectorPluginServer() @app.route('Collector.init') def collector_init(params: dict) -> dict: - """ init plugin by options - - Args: - params (CollectorInitRequest): { - 'options': 'dict', # Required - 'domain_id': 'str' - } - - Returns: - PluginResponse: { - 'metadata': 'dict' - } - """ - pass - - -@app.route('Collector.verify') -def collector_verify(params: dict) -> None: - """ Verifying collector plugin - - Args: - params (CollectorVerifyRequest): { - 'options': 'dict', # Required - 'secret_data': 'dict', # Required - 'schema': 'str', - 'domain_id': 'str' - } - - Returns: - None - """ - pass + return {'metadata': {'options_schema': _create_options_schema()}} @app.route('Collector.collect') def collector_collect(params: dict) -> dict: - """ Collect external data - - Args: - params (CollectorCollectRequest): { - 'options': 'dict', # Required - 'secret_data': 'dict', # Required - 'schema': 'str', - 'domain_id': 'str' - } - - Returns: - Generator[ResourceResponse, None, None] - { - 'state': 'SUCCESS | FAILURE', - 'resource_type': 'inventory.CloudService | inventory.CloudServiceType | inventory.Region', - 'resource_data': 'dict', - 'match_keys': 'list', - 'error_message': 'str' - 'metadata': 'dict' + options = params['options'] + secret_data = params['secret_data'] + schema = params.get('schema') + + repository_manager = RepoManager() + return repository_manager.collect_resources(options, secret_data, schema) + + +def _create_options_schema(): + return { + 'required': ['items'], + 'order': ['items'], + 'type': 'object', + 'properties': { + 'items': { + 'title': 'Item filter', + 'type': 'array', + 'items': { + 'enum': [ + 'fields' + ] + } + } } - """ - pass + } \ No newline at end of file diff --git a/src/plugin/manager/__init__.py b/src/plugin/manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/manager/repo_manager.py b/src/plugin/manager/repo_manager.py new file mode 100644 index 0000000..79c3688 --- /dev/null +++ b/src/plugin/manager/repo_manager.py @@ -0,0 +1,105 @@ +import logging + +from spaceone.inventory.plugin.collector.lib import * +from ..connector.repo_connector import RepoConnector + + +_LOGGER = logging.getLogger("cloudforet") + + +class RepoManager: + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.cloud_service_group = "Repository" + self.cloud_service_type = "Issue" + self.provider = "Github" + self.metadata_path = "plugin/metadata/repository/issues.yaml" + + def collect_resources(self, options, secret_data, schema): + try: + yield from self.collect_cloud_service_type(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + resource_type="inventory.CloudServiceType", + ) + + try: + yield from self.collect_cloud_service(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + + def collect_cloud_service_type(self, options, secret_data, schema): + cloud_service_type = make_cloud_service_type( + name=self.cloud_service_type, + group=self.cloud_service_group, + provider=self.provider, + metadata_path=self.metadata_path, + ) + + yield make_response( + cloud_service_type=cloud_service_type, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + resource_type="inventory.CloudServiceType", + ) + + def collect_cloud_service(self, options, secret_data, schema): + repo_connector = RepoConnector() + repos = repo_connector.get_repositories(secret_data) + + all_issues = [] + + _LOGGER.debug(f"Repos: {repos}") + + for repo in repos: + _LOGGER.debug(f"Processing repo: {repo}") + + owner = repo.get("owner", {}) + repo_name = repo.get("name") + + if repo_name and owner: + repo_owner = owner.get("login") + + _LOGGER.debug(f"Processing issues for repo: {repo_name}") + + issues = repo_connector.get_repo_issues(secret_data, repo_owner, repo_name) + pulls = repo_connector.get_repo_pulls(secret_data, repo_owner, repo_name) + + for issue in issues: + issue["repo_name"] = repo_name + issue["pulls"] = pulls + + all_issues.extend(issues) + else: + _LOGGER.warning("Missing required attributes in repo: {repo}") + + all_issues.sort(key=lambda x: x.get("created_at", ""), reverse=True) + + for issue in all_issues: + labels = issue.get("labels", []) + issue_labels = {label.get("name"): label for label in labels} + issue["labels"] = issue_labels + + cloud_service = make_cloud_service( + name=self.cloud_service_type, + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + data=issue, + ) + + yield make_response( + cloud_service=cloud_service, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + ) + diff --git a/src/plugin/metadata/__init__.py b/src/plugin/metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/metadata/repository/issues.yaml b/src/plugin/metadata/repository/issues.yaml new file mode 100644 index 0000000..df939eb --- /dev/null +++ b/src/plugin/metadata/repository/issues.yaml @@ -0,0 +1,49 @@ +--- +search: + fields: + - Title: data.title + - Description: data.description + - Number: data.number + - Repository: data.repo_name + - Created by: data.user.login + +table: + sort: + key: data.updated_at + desc: true + fields: + - Title: data.title + - State: data.state + type: badge + outline_color: green.500 + shape: ROUND + - Number: data.number + - URL: data.html_url + - Created by: data.user.login + - Label: data.lables + type: dict + - Comments: data.comments + - Repository: data.repo_name + - Repository_url: data.repository_url + - Created at: data.created_at + type: datetime + - Updated at: data.updated_at + type: datetime + +tabs.0: + name: Issue Details + type: html + fields: + - Issue: data.body + +tab.1: + name: Pulls + type: table + fields: + - Title: data.pulls.title + - Url: data.pulls.html_url + - User: data.pulls.user.login + - Created at: data.pulls.created_at + type: datetime + - Updated at: data.pulls.updated_at + type: datetime diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..f13d50b --- /dev/null +++ b/src/setup.py @@ -0,0 +1,36 @@ +# +# Copyright 2020 The SpaceONE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from setuptools import setup, find_packages + +with open("VERSION", "r") as f: + VERSION = f.read().strip() + f.close() + +setup( + name="plugin-github-inven-collector", + version=VERSION, + description="Collector plugin for Github", + long_description="", + url="https://www.spaceone.dev/", + author="SpaceONE Study Group", + author_email="admin@spaceone.dev", + license="Apache License 2.0", + packages=find_packages(), + install_requires=["spaceone-core", "spaceone-api", "spaceone-inventory"], + package_data={"plugin": ["metadata/repository/*.yaml"]}, + zip_safe=False, +)