Removes cookiecutter dependency (#2360)

* Removes cookiecutter dependency Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> * Updates requirements Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> * Raise error with directory already exists Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> * Lint and regenerate dev-requirements Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> --------- Signed-off-by: Thomas J. Fan <thomasjpfan@gmail.com> Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> Co-authored-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> Signed-off-by: Jan Fiedler <jan@union.ai>
flyteorg · Jul 25, 2024 · 89a4507 · 89a4507
1 parent 6982ea8
commit 89a4507
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 52 deletions.
diff --git a/dev-requirements.in b/dev-requirements.in
@@ -20,6 +20,7 @@ IPython
 keyrings.alt
 setuptools_scm
 pytest-icdiff
+jinja2
 
 # Tensorflow is not available for python 3.12 yet: https://github.com/tensorflow/tensorflow/issues/62003
 tensorflow; python_version<'3.12'

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -24,8 +24,6 @@ aioitertools==0.11.0
     # via aiobotocore
 aiosignal==1.3.1
     # via aiohttp
-arrow==1.3.0
-    # via cookiecutter
 asttokens==2.4.1
     # via stack-data
 astunparse==1.6.3
@@ -47,8 +45,6 @@ azure-identity==1.15.0
     # via adlfs
 azure-storage-blob==12.19.0
     # via adlfs
-binaryornot==0.4.4
-    # via cookiecutter
 botocore==1.31.17
     # via aiobotocore
 cachetools==5.3.2
@@ -63,21 +59,16 @@ cffi==1.16.0
     #   cryptography
 cfgv==3.4.0
     # via pre-commit
-chardet==5.2.0
-    # via binaryornot
 charset-normalizer==3.3.2
     # via requests
 click==8.1.7
     # via
-    #   cookiecutter
     #   flytekit
     #   rich-click
 cloudpickle==3.0.0
     # via flytekit
 codespell==2.2.6
     # via -r dev-requirements.in
-cookiecutter==2.5.0
-    # via flytekit
 coverage[toml]==7.4.1
     # via
     #   -r dev-requirements.in
@@ -114,8 +105,10 @@ filelock==3.13.1
     #   virtualenv
 flatbuffers==23.5.26
     # via tensorflow-macos
-flyteidl==1.10.6
-    # via flytekit
+flyteidl @ git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl
+    # via
+    #   -r dev-requirements.in
+    #   flytekit
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -176,6 +169,7 @@ googleapis-common-protos==1.62.0
     #   flytekit
     #   google-api-core
     #   grpcio-status
+    #   protoc-gen-openapiv2
 grpcio==1.60.1
     # via
     #   flytekit
@@ -191,6 +185,8 @@ h5py==3.10.0
     # via tensorflow-macos
 hypothesis==6.98.2
     # via -r dev-requirements.in
+icdiff==2.0.7
+    # via pytest-icdiff
 identify==2.5.33
     # via pre-commit
 idna==3.6
@@ -206,7 +202,9 @@ iniconfig==2.0.0
 ipython==8.21.0
     # via -r dev-requirements.in
 isodate==0.6.1
-    # via azure-storage-blob
+    # via
+    #   azure-storage-blob
+    #   flytekit
 jaraco-classes==3.3.0
     # via
     #   keyring
@@ -215,7 +213,7 @@ jedi==0.19.1
     # via ipython
 jinja2==3.1.3
     # via
-    #   cookiecutter
+    #   -r dev-requirements.in
     #   torch
 jmespath==1.0.1
     # via botocore
@@ -233,13 +231,15 @@ keyring==24.3.0
 keyrings-alt==5.0.0
     # via -r dev-requirements.in
 kubernetes==29.0.0
-    # via flytekit
+    # via -r dev-requirements.in
 libclang==16.0.6
     # via tensorflow-macos
 markdown==3.5.2
     # via tensorboard
 markdown-it-py==3.0.0
-    # via rich
+    # via
+    #   flytekit
+    #   rich
 markupsafe==2.1.5
     # via
     #   jinja2
@@ -307,7 +307,7 @@ oauthlib==3.2.2
     #   kubernetes
     #   requests-oauthlib
 opt-einsum==3.3.0
-    # via tensorflow
+    # via tensorflow-macos
 orjson==3.9.12
     # via -r dev-requirements.in
 packaging==23.2
@@ -333,6 +333,8 @@ pluggy==1.4.0
     # via pytest
 portalocker==2.8.2
     # via msal-extensions
+pprintpp==0.4.0
+    # via pytest-icdiff
 pre-commit==3.6.0
     # via -r dev-requirements.in
 prometheus-client==0.19.0
@@ -350,10 +352,10 @@ protobuf==4.23.4
     #   googleapis-common-protos
     #   grpcio-status
     #   proto-plus
-    #   protoc-gen-swagger
+    #   protoc-gen-openapiv2
     #   tensorboard
     #   tensorflow-macos
-protoc-gen-swagger==0.1.0
+protoc-gen-openapiv2==0.0.1
     # via flyteidl
 ptyprocess==0.7.0
     # via pexpect
@@ -373,24 +375,26 @@ pyflakes==3.2.0
     # via autoflake
 pygments==2.17.2
     # via
+    #   flytekit
     #   ipython
     #   rich
 pyjwt[crypto]==2.8.0
-    # via
-    #   msal
-    #   pyjwt
+    # via msal
 pytest==7.4.4
     # via
     #   -r dev-requirements.in
     #   pytest-asyncio
     #   pytest-cov
+    #   pytest-icdiff
     #   pytest-mock
     #   pytest-timeout
     #   pytest-xdist
 pytest-asyncio==0.23.4
     # via -r dev-requirements.in
 pytest-cov==4.1.0
     # via -r dev-requirements.in
+pytest-icdiff==0.9
+    # via -r dev-requirements.in
 pytest-mock==3.12.0
     # via -r dev-requirements.in
 pytest-timeout==2.2.0
@@ -399,7 +403,6 @@ pytest-xdist==3.5.0
     # via -r dev-requirements.in
 python-dateutil==2.8.2
     # via
-    #   arrow
     #   botocore
     #   croniter
     #   google-cloud-bigquery
@@ -409,8 +412,6 @@ python-json-logger==2.0.7
     # via flytekit
 python-magic==0.4.27 ; platform_system == "Darwin" or platform_system == "Linux"
     # via -r dev-requirements.in
-python-slugify==8.0.3
-    # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
 pytz==2024.1
@@ -419,15 +420,13 @@ pytz==2024.1
     #   pandas
 pyyaml==6.0.1
     # via
-    #   cookiecutter
     #   flytekit
     #   kubernetes
     #   pre-commit
 requests==2.31.0
     # via
     #   azure-core
     #   azure-datalake-store
-    #   cookiecutter
     #   docker
     #   flytekit
     #   gcsfs
@@ -444,7 +443,6 @@ requests-oauthlib==1.3.1
     #   kubernetes
 rich==13.7.0
     # via
-    #   cookiecutter
     #   flytekit
     #   rich-click
 rich-click==1.7.3
@@ -492,8 +490,6 @@ tensorflow-macos==2.15.0
     # via tensorflow
 termcolor==2.4.0
     # via tensorflow-macos
-text-unidecode==1.3
-    # via python-slugify
 threadpoolctl==3.2.0
     # via scikit-learn
 torch==2.2.0 ; python_version < "3.12"
@@ -504,12 +500,12 @@ traitlets==5.14.1
     #   matplotlib-inline
 types-croniter==2.0.0.20240106
     # via -r dev-requirements.in
+types-decorator==5.1.8.20240310
+    # via -r dev-requirements.in
 types-mock==5.1.0.20240106
     # via -r dev-requirements.in
 types-protobuf==4.24.0.20240129
     # via -r dev-requirements.in
-types-python-dateutil==2.8.19.20240106
-    # via arrow
 types-requests==2.31.0.6
     # via -r dev-requirements.in
 types-urllib3==1.26.25.14

diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py
@@ -1,36 +1,61 @@
+import os
+import re
+from io import BytesIO
+from zipfile import ZipFile
+
+import requests
 import rich_click as click
-from cookiecutter.main import cookiecutter
 
 
 @click.command("init")
 @click.option(
     "--template",
     default="basic-template-imagespec",
-    help="cookiecutter template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git",
+    help="template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git",
 )
 @click.argument("project-name")
 def init(template, project_name):
     """
     Create flyte-ready projects.
     """
-    config = {
-        "project_name": project_name,
-        "app": "flyte",
-        "workflow": "my_wf",
-    }
-    cookiecutter(
-        "https://github.com/flyteorg/flytekit-python-template.git",
-        checkout="main",
-        no_input=True,
-        # We do not want to clobber existing files/directories.
-        overwrite_if_exists=False,
-        extra_context=config,
-        # By specifying directory we can have multiple templates in the same repository,
-        # as described in https://cookiecutter.readthedocs.io/en/1.7.2/advanced/directories.html.
-        # The idea is to extend the number of templates, each in their own subdirectory, for example
-        # a tensorflow-based example.
-        directory=template,
-    )
+    if os.path.exists(project_name):
+        raise click.ClickException(f"{project_name} directory already exists")
+
+    template_zip_url = "https://github.com/flyteorg/flytekit-python-template/archive/refs/heads/main.zip"
+
+    response = requests.get(template_zip_url)
+
+    if response.status_code != 200:
+        raise click.ClickException("Unable to download template from github.com/flyteorg/flytekit-python-template")
+
+    zip_content = BytesIO(response.content)
+    zip_root_name = "flytekit-python-template-main"
+    project_name_template = "{{cookiecutter.project_name}}"
+    prefix = os.path.join(zip_root_name, template, project_name_template, "")
+    prefix_len = len(prefix)
+
+    # We use a regex here to be more compatible with cookiecutter templating
+    project_template_regex = re.compile(rb"\{\{ ?cookiecutter\.project_name ?\}\}")
+
+    project_name_bytes = project_name.encode("utf-8")
+
+    with ZipFile(zip_content, "r") as zip_file:
+        template_members = [m for m in zip_file.namelist() if m.startswith(prefix)]
+
+        for member in template_members:
+            dest = os.path.join(project_name, member[prefix_len:])
+
+            # member is a directory
+            if dest.endswith(os.sep):
+                if not os.path.exists(dest):
+                    os.mkdir(dest)
+                continue
+
+            # member is a file
+            with zip_file.open(member) as zip_member, open(dest, "wb") as dest_file:
+                zip_contents = zip_member.read()
+                processed_contents = project_template_regex.sub(project_name_bytes, zip_contents)
+                dest_file.write(processed_contents)
 
     click.echo(
         f"Visit the {project_name} directory and follow the next steps in the Getting started guide (https://docs.flyte.org/en/latest/getting_started_with_workflow_development/index.html) to proceed."

diff --git a/pyproject.toml b/pyproject.toml
@@ -15,7 +15,6 @@ dependencies = [
     "adlfs>=2023.3.0",
     "click>=6.6,<9.0",
     "cloudpickle>=2.0.0",
-    "cookiecutter>=1.7.3",
     "croniter>=0.3.20,<4.0.0",
     "dataclasses-json>=0.5.2,<0.5.12", # TODO: remove upper-bound after fixing change in contract
     "diskcache>=5.2.1",