Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updates to pyflyte init templating #1665

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions flytekit/clis/sdk_in_container/helpers.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
import os
import shutil
from dataclasses import replace
from typing import Optional

import rich_click as click
from git import Repo

from flytekit.clis.sdk_in_container.constants import CTX_CONFIG_FILE
from flytekit.configuration import Config, ImageConfig, get_config_file
Expand Down Expand Up @@ -63,3 +66,34 @@ def patch_image_config(config_file: Optional[str], image_config: ImageConfig) ->
if addl.name not in additional_image_names:
new_additional_images.append(addl)
return replace(image_config, default_image=new_default, images=new_additional_images)


def clone_and_copy_repo_dir(git_url, branch, src_dir, dest_dir):
"""
Clones a git repository, checks out to a specific branch, and copies a specified directory into a local directory.

Parameters:
- github_url: The URL of the GitHub repository.
- branch: The branch of the repository.
- src_dir: The directory in the repository to copy.
- dest_dir: The local directory to copy into.
"""
# Clone the repo
repo = Repo.clone_from(git_url, "temp_repo")

# Checkout to the branch
repo.git.checkout(branch)
Comment on lines +82 to +85
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can pass the branch name directly in the call to Repo.clone_from.


# Define the source directory path
src_path = os.path.join("temp_repo", src_dir)

# Check if source directory exists
if not os.path.exists(src_path):
print(f"Template named {src_dir} does not exist in the repository.")
return

# Copy the files from the source directory to the destination directory
shutil.copytree(src_path, dest_dir)

# Remove the temporary cloned repo
shutil.rmtree("temp_repo")
Comment on lines +95 to +99
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about we use a TemporaryDirectory to hold the directory? This way we don't need to do this book-keeping and it also allows for this command to be run multiple times.

31 changes: 11 additions & 20 deletions flytekit/clis/sdk_in_container/init.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import rich_click as click
from cookiecutter.main import cookiecutter

from flytekit.clis.sdk_in_container.helpers import clone_and_copy_repo_dir


@click.command("init")
Expand All @@ -8,29 +9,19 @@
default="simple-example",
help="cookiecutter template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git",
)
@click.option(
"--repository-url",
default="https://github.com/flyteorg/flytekit-python-template.git",
help="template repository url pointing to a git repository containing flytekit templates.",
)
@click.option("--repository-branch", default="main", help="template repository branch to be used.")
@click.argument("project-name")
def init(template, project_name):
def init(template, repository_url, repository_branch, project_name):
"""
Create flyte-ready projects.
"""
config = {
"project_name": project_name,
"app": "flyte",
"workflow": "my_wf",
}
cookiecutter(
"https://github.com/flyteorg/flytekit-python-template.git",
checkout="main",
no_input=True,
# We do not want to clobber existing files/directories.
overwrite_if_exists=False,
extra_context=config,
# By specifying directory we can have multiple templates in the same repository,
# as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html.
# The idea is to extend the number of templates, each in their own subdirectory, for example
# a tensorflow-based example.
directory=template,
)

clone_and_copy_repo_dir(repository_url, repository_branch, template, project_name)

click.echo(
f"Visit the {project_name} directory and follow the next steps in the Getting started guide (https://docs.flyte.org/en/latest/getting_started.html) to proceed."
Expand Down
1 change: 0 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@
"docstring-parser>=0.9.0",
"diskcache>=5.2.1",
"cloudpickle>=2.0.0",
"cookiecutter>=1.7.3",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❤️

"numpy",
"gitpython",
"kubernetes>=12.0.1",
Expand Down