Skip to content

Commit

Permalink
dev: github importer (#3205)
Browse files Browse the repository at this point in the history 
* dev: initiate github import

* dev: github importer all issues import

* dev: github comments and links for the imported issues

* dev: update controller to use logger and spread the resultData in getAllEntities

* dev: removed console log

* dev: update code structure and sync functions

* dev: updated retry logic when exception

* dev: add imported data as well

* dev: update logger and repo fetch

* dev: update jira integration to new structure

* dev: update migrations

* dev: update the reason field
  • Loading branch information
@pablohashescobar
pablohashescobar committed Dec 20, 2023
1 parent 1cc18a0 commit 7010448
Show file tree
Hide file tree
Showing 15 changed files with 3,989 additions and 348 deletions.
243 changes: 140 additions & 103 deletions apiserver/plane/app/views/importer.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
# Python imports
import uuid
import json
import requests

# Third party imports
from rest_framework import status
from rest_framework.response import Response

# Django imports
from django.db.models import Max, Q

from django.conf import settings
# Module imports
from plane.app.views import BaseAPIView
from plane.db.models import (
Expand All @@ -34,19 +36,15 @@
IssueFlatSerializer,
ModuleSerializer,
)
from plane.utils.integrations.github import get_github_repo_details
from plane.utils.importers.jira import jira_project_issue_summary
from plane.bgtasks.importer_task import service_importer
from plane.utils.html_processor import strip_tags
from plane.app.permissions import WorkSpaceAdminPermission

from plane.bgtasks.importer_task import service_importer

class ServiceIssueImportSummaryEndpoint(BaseAPIView):
def get(self, request, slug, service):
if service == "github":
owner = request.GET.get("owner", False)
repo = request.GET.get("repo", False)

if not owner or not repo:
return Response(
{"error": "Owner and repo are required"},
Expand All @@ -57,30 +55,44 @@ def get(self, request, slug, service):
integration__provider="github", workspace__slug=slug
)

access_tokens_url = workspace_integration.metadata.get(
"access_tokens_url", False
)
installtion_id = workspace_integration.config.get("installation_id", False)

if not access_tokens_url:
# Check for the installation id
if not installtion_id:
return Response(
{
"error": "There was an error during the installation of the GitHub app. To resolve this issue, we recommend reinstalling the GitHub app."
},
status=status.HTTP_400_BAD_REQUEST,
)

issue_count, labels, collaborators = get_github_repo_details(
access_tokens_url, owner, repo
)
# Request segway for the required information
if settings.SEGWAY_BASE_URL:
headers = {
"Content-Type": "application/json",
"x-api-key": settings.SEGWAY_KEY,
}
data = {
"owner": owner,
"repo": repo,
"installationId": installtion_id,
}
res = requests.post(
f"{settings.SEGWAY_BASE_URL}/api/github",
data=json.dumps(data),
headers=headers,
)
if "error" in res.json():
return Response(res.json(), status=status.HTTP_400_BAD_REQUEST)
else:
return Response(
res.json(),
status=status.HTTP_200_OK,
)
return Response(
{
"issue_count": issue_count,
"labels": labels,
"collaborators": collaborators,
},
status=status.HTTP_200_OK,
{"error": "Inetgration service is not available please try later"},
status=status.HTTP_400_BAD_REQUEST,
)

if service == "jira":
# Check for all the keys
params = {
Expand All @@ -101,16 +113,35 @@ def get(self, request, slug, service):
email = request.GET.get("email", "")
cloud_hostname = request.GET.get("cloud_hostname", "")

response = jira_project_issue_summary(
email, api_token, project_key, cloud_hostname
)
if "error" in response:
return Response(response, status=status.HTTP_400_BAD_REQUEST)
else:
return Response(
response,
status=status.HTTP_200_OK,
if settings.SEGWAY_BASE_URL:
headers = {
"Content-Type": "application/json",
"x-api-key": settings.SEGWAY_KEY,
}
data = {
"project_key": project_key,
"api_token": api_token,
"email": email,
"cloud_hostname": cloud_hostname,
}
res = requests.post(
f"{settings.SEGWAY_BASE_URL}/api/jira",
data=json.dumps(data),
headers=headers,
)

if "error" in res.json():
return Response(res.json(), status=status.HTTP_400_BAD_REQUEST)
else:
return Response(
res.json(),
status=status.HTTP_200_OK,
)

return Response(
{"error": "Inetgration service is not available please try later"},
status=status.HTTP_400_BAD_REQUEST,
)
return Response(
{"error": "Service not supported yet"},
status=status.HTTP_400_BAD_REQUEST,
Expand All @@ -123,97 +154,103 @@ class ImportServiceEndpoint(BaseAPIView):
]

def post(self, request, slug, service):
if service not in ["github", "jira"]:
return Response(
{"error": "Servivce not supported yet"},
status=status.HTTP_400_BAD_REQUEST,
)

if service == "github":
workspace_integration = WorkspaceIntegration.objects.get(
integration__provider="github", workspace__slug=slug
)

installation_id = workspace_integration.config.get("installation_id", False)

project_id = request.data.get("project_id", False)

if not project_id:
return Response(
{"error": "Project ID is required"},
status=status.HTTP_400_BAD_REQUEST,
)

workspace = Workspace.objects.get(slug=slug)

if service == "github":
data = request.data.get("data", False)
metadata = request.data.get("metadata", False)
config = request.data.get("config", False)
if not data or not metadata or not config:
return Response(
{"error": "Data, config and metadata are required"},
status=status.HTTP_400_BAD_REQUEST,
)
# Validate the data
data = request.data.get("data", False)
metadata = request.data.get("metadata", False)
config = request.data.get("config", False)
if not data or not metadata or not config:
return Response(
{"error": "Data, config and metadata are required"},
status=status.HTTP_400_BAD_REQUEST,
)

api_token = APIToken.objects.filter(
user=request.user, workspace=workspace
).first()
if api_token is None:
api_token = APIToken.objects.create(
user=request.user,
label="Importer",
workspace=workspace,
)
# Update config
if config and service == "github":
config.update({"installation_id": installation_id})

importer = Importer.objects.create(
service=service,
project_id=project_id,
status="queued",
initiated_by=request.user,
data=data,
metadata=metadata,
token=api_token,
config=config,
created_by=request.user,
updated_by=request.user,
# Get the api token -- # derecated
api_token = APIToken.objects.filter(
user=request.user, workspace=workspace
).first()
if api_token is None:
api_token = APIToken.objects.create(
user=request.user,
label="Importer",
workspace=workspace,
)

service_importer.delay(service, importer.id)
serializer = ImporterSerializer(importer)
return Response(serializer.data, status=status.HTTP_201_CREATED)

if service == "jira":
data = request.data.get("data", False)
metadata = request.data.get("metadata", False)
config = request.data.get("config", False)
if not data or not metadata:
return Response(
{"error": "Data, config and metadata are required"},
status=status.HTTP_400_BAD_REQUEST,
)
api_token = APIToken.objects.filter(
user=request.user, workspace=workspace
).first()
if api_token is None:
api_token = APIToken.objects.create(
user=request.user,
label="Importer",
workspace=workspace,
)
# Create an import
importer = Importer.objects.create(
service=service,
project_id=project_id,
status="queued",
initiated_by=request.user,
data=data,
metadata=metadata,
token=api_token,
config=config,
created_by=request.user,
updated_by=request.user,
)

importer = Importer.objects.create(
service=service,
project_id=project_id,
status="queued",
initiated_by=request.user,
data=data,
metadata=metadata,
token=api_token,
config=config,
created_by=request.user,
updated_by=request.user,
# Push it to segway
if settings.SEGWAY_BASE_URL:
headers = {
"Content-Type": "application/json",
"x-api-key": settings.SEGWAY_KEY,
}
data = {
"metadata": metadata,
"data": data,
"config": config,
"workspace_id": str(workspace.id),
"project_id": str(project_id),
"created_by": str(request.user.id),
"importer_id": str(importer.id),
}
res = requests.post(
f"{settings.SEGWAY_BASE_URL}/api/github/import",
data=json.dumps(data),
headers=headers,
)

service_importer.apply_async(
args=[],
kwargs={"service": service, "importer_id": importer.id},
routing_key="internal",
)
serializer = ImporterSerializer(importer)
return Response(serializer.data, status=status.HTTP_201_CREATED)
if "error" in res.json():
importer.status = "failed"
importer.reason = str(res.json())
importer.save()
else:
importer.status = "processing"
importer.save(update_fields=["status"])
else:
importer.status = "failed"
importer.reason = "Segway base url is not present"
importer.save(update_fields=["status", "reason"])

return Response(
{"error": "Servivce not supported yet"},
status=status.HTTP_400_BAD_REQUEST,
)
# return the response
serializer = ImporterSerializer(importer)
return Response(serializer.data, status=status.HTTP_201_CREATED)

def get(self, request, slug):
imports = (
Expand Down
Loading

0 comments on commit 7010448

Please sign in to comment.