Auto Draft PR

.eslintrc

@@ -33,7 +33,7 @@
         "pipedream/props-description": "warn",
         "pipedream/source-name": "warn",
         "pipedream/source-description": "warn",
-        "pipedream/no-ts-version": "error"
+        "pipedream/no-ts-version": "warn"
       }
     },
     {

packages/component_code_gen/README.md

@@ -75,12 +75,20 @@ poetry run python main.py --type action --app slack --instructions instructions.
 ### GitHub Issues
 
 The command below will parse through GitHub issue description and generate code for the list of components.
-All the code will be written to a folder named `output`. Be sure to add `BROWSERLESS_API_KEY` to `.env` to parse API docs.
+Be sure to add `BROWSERLESS_API_KEY` to `.env` to parse API docs.
 
 ```
 poetry run python main.py --issue issue_number
 ```
 
+#### Draft PRs
+
+The default behaviour is to automatically create a branch and submit a Draft PR. The command above is equivalent to:
+
+```
+poetry run python main.py --issue issue_number --skip_pr=False
+```
+
 #### Output Dir
 
 The default `output_dir` is where Pipedream components live in the repo: `pipedream/components`. The generated components

packages/component_code_gen/code_gen/generate.py

@@ -1,5 +1,4 @@
 import os
-import time
 from selenium import webdriver
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
@@ -96,11 +95,10 @@ def parse_urls(driver, urls):
                 EC.presence_of_element_located((By.TAG_NAME, 'body'))
             )
             body = " ".join(element.text.split())
-            content = {
+            contents.append({
                 "url": url,
                 "content": body,
-            }
-            contents.append(content)
+            })
         except Exception as e:
             print(f"Error scraping {url}: {e}")
 

packages/component_code_gen/code_gen/generate_for_github_issue.py

@@ -1,14 +1,19 @@
 from collections import OrderedDict
 import os
-import json
+import git
 import requests
+import subprocess
 import markdown_to_json
 import config.logging_config as logging_config
 from code_gen.generate import main
 
 logger = logging_config.getLogger(__name__)
 
 
+def run_command(command):
+    subprocess.run(command, shell=True)
+
+
 def extract_prompts(markdown_dict):
     prompts = []
     for key, value in markdown_dict.items():
@@ -20,13 +25,20 @@ def extract_prompts(markdown_dict):
 
 
 def get_all_docs_urls(markdown_dict):
-    urls = []
-    for key, value in markdown_dict.items():
-        if key == "urls":
-            urls.extend(value)
-        elif isinstance(value, (dict, OrderedDict)):
-            urls.extend(get_all_docs_urls(value))
-    return urls
+    def extract_urls(markdown_dict):
+        urls = []
+        for key, value in markdown_dict.items():
+            if key == "urls":
+                urls.extend(value)
+            elif isinstance(value, (dict, OrderedDict)):
+                urls.extend(extract_urls(value))
+        return urls
+    all_urls = extract_urls(markdown_dict)
+    return unique_urls(all_urls)
+
+
+def unique_urls(urls):
+    return list(set([url.split("#")[0] for url in urls]))
 
 
 def generate_app_file_prompt(requirements, app_file_content):
@@ -45,7 +57,31 @@ def generate_app_file_prompt(requirements, app_file_content):
 {requirements}"""
 
 
-def generate(issue_number, output_dir, verbose=False, tries=3):
+def generate(issue_number, output_dir, generate_pr=True, clean=False, verbose=False, tries=3):
+    repo_path = os.path.abspath(os.path.join("..", ".."))
+    output_dir = os.path.abspath(output_dir)
+
+    if generate_pr:
+        output_dir = os.path.join(repo_path, "components")
+        repo = git.Repo(repo_path)
+
+        if not clean and repo.index.diff(None):
+            logger.warn(
+                "Your git stage is not clean. Please stash/commit your changes or use --clean to discard them")
+            return
+
+        branch_name = f"issue-{issue_number}"
+        run_command("git fetch origin")
+
+        if any(reference.name == branch_name for reference in repo.references):
+            # branch name already exists
+            run_command(f"git checkout {branch_name}")
+        else:
+            # create new branch
+            run_command(f"git checkout -b {branch_name}")
+
+        run_command("git reset --hard origin/master")
+
     # parse github issue description
     md = requests.get(
         f"https://api.github.com/repos/PipedreamHQ/pipedream/issues/{issue_number}").json()["body"].lower()
@@ -54,7 +90,8 @@ def generate(issue_number, output_dir, verbose=False, tries=3):
     global_urls = []
     requirements = []
 
-    file_path = f"{output_dir}/{app}/{app}.app.mjs"
+    app_base_path = os.path.join(output_dir, app)
+    file_path = os.path.join(app_base_path, f"{app}.app.mjs")
 
     # If the directory at file_path doesn't exist, create it
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
@@ -121,7 +158,7 @@ def generate(issue_number, output_dir, verbose=False, tries=3):
                 "type": component_type,
                 "key": component_key,
                 "instructions": f"The component key is {app}-{component_key}. {instructions}",
-                "urls": global_urls + urls,
+                "urls": unique_urls(global_urls + urls),
             })
 
     for component in requirements:
@@ -137,3 +174,16 @@ def generate(issue_number, output_dir, verbose=False, tries=3):
         os.makedirs(os.path.dirname(file_path), exist_ok=True)
         with open(file_path, 'w') as f:
             f.write(result)
+
+    if generate_pr:
+        # XXX: add deps to package.json
+        # XXX: remove ts stuff and .gitignore
+        # GitPython requires to manually check .gitignore paths when adding files to index
+        # so this is easier
+        run_command(f"npx eslint {app_base_path} --fix")
+        run_command(["npx pnpm i"])
+        run_command(f"git add -f {app_base_path} {repo_path}/pnpm-lock.yaml")
+        run_command(f"git commit --no-verify -m '{app} init'")
+        run_command(f"git push -f --set-upstream origin {branch_name}")
+        run_command(
+            f"gh pr create --draft --title 'New Components - {app}' --body 'Resolves #{issue_number}.'")

packages/component_code_gen/main.py

@@ -8,6 +8,10 @@
     parser = argparse.ArgumentParser()
     parser.add_argument('--issue', help='The issue number on github',
                         type=int, required=False)
+    parser.add_argument('--skip-pr', dest='skip_pr', help='Generate a PR on github',
+                        required=False, default=False, action='store_true')
+    parser.add_argument('--clean', dest='clean', help='Clean git stage',
+                        required=False, default=False, action='store_true')
     parser.add_argument('--type', help='Which kind of code you want to generate?',
                         choices=available_templates.keys())
     parser.add_argument('--app', help='The app_name_slug')
@@ -26,8 +30,8 @@
     args = parser.parse_args()
 
     if args.issue:
-        generate(args.issue, output_dir=args.output_dir,
-                 verbose=args.verbose, tries=args.tries)
+        generate(args.issue, output_dir=args.output_dir, generate_pr=not args.skip_pr,
+                 clean=args.clean, verbose=args.verbose, tries=args.tries)
     else:
         if not args.type:
             raise argparse.ArgumentTypeError("--type is required")

packages/component_code_gen/pyproject.toml

@@ -13,6 +13,7 @@ langchain = "^0.0.312"
 supabase = "^1.0.3"
 selenium = "^4.12.0"
 markdown-to-json = "^2.1.0"
+gh = "^0.0.4"
 
 
 [build-system]