From e656ef4cbaee1a03a2dceb3c9a3617e1c7d89b10 Mon Sep 17 00:00:00 2001 From: danielperezz Date: Sun, 28 Sep 2025 12:43:16 +0300 Subject: [PATCH 1/4] create a CLI for generating item.yaml and organize the CLI directory --- cli/README.md | 66 +++++++++++++++++++++++ cli/cli.py | 7 +-- cli/common/generate_item_yaml.py | 53 +++++++++++++++++++ cli/common/item_yaml.py | 54 ------------------- cli/functions/new_function_item.py | 67 ------------------------ cli/utils/function_item_template.yaml.j2 | 22 ++++++++ cli/utils/item_template.yaml | 21 -------- cli/utils/module_item_template.yaml.j2 | 16 ++++++ 8 files changed, 159 insertions(+), 147 deletions(-) create mode 100644 cli/README.md create mode 100644 cli/common/generate_item_yaml.py delete mode 100644 cli/common/item_yaml.py delete mode 100644 cli/functions/new_function_item.py create mode 100644 cli/utils/function_item_template.yaml.j2 delete mode 100644 cli/utils/item_template.yaml create mode 100644 cli/utils/module_item_template.yaml.j2 diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 000000000..4a3cd3bfc --- /dev/null +++ b/cli/README.md @@ -0,0 +1,66 @@ +## Available Commands +(Explore more advanced options in the code, this is basic usage demonstration) + +### generate-item-yaml +Generate an `item.yaml` file (basic draft) in the appropriate directory from a Jinja2 template + +Usage: + `python -m cli.cli generate-item-yaml TYPE NAME` + +Example: + `python -m cli.cli generate-item-yaml function aggregate` + +--- + +### item-to-function +Creates a `function.yaml` file based on a provided `item.yaml` file. + +Usage: + `python -m cli.cli item-to-function --item-path PATH` + +Example: + `python -m cli.cli item-to-function --item-path functions/src/aggregate` + +--- + +### function-to-item +Creates a `item.yaml` file based on a provided `function.yaml` file. + +Usage: + `python -m cli.cli function-to-item PATH` + +Example: + `python -m cli.cli function-to-item --path functions/src/aggregate` + +--- + +### run-tests +Run assets test suite. + +Usage: + `python -m cli.cli run-tests -r PATH -s TYPE -fn NAME` + +Example: + `python -m cli.cli run-tests -r functions/src/aggregate -s py -fn aggregate` + +--- + +### build-marketplace +Build and push (create a PR) the updated marketplace/ directory (e.g: marketplace/functions) + +Usage: + `python -m cli.cli build-marketplace -s SOURCE-DIR -sn TYPE -m MARKETPLACE-DIR -c CHANNEL -v -f` + +Example: + `python -m cli.cli build-marketplace -s ./functions/src -sn functions -m marketplace -c master -v -f` + +--- + +### update-readme +Regenerate the `README.md` files in each of the asset directories (functions/modules). + +Usage: + `python -m cli.cli update-readme --asset TYPE` + +Example: + `python -m cli.cli update-readme --asset functions --asset modules` \ No newline at end of file diff --git a/cli/cli.py b/cli/cli.py index 8fee9891a..e8e6922fe 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -17,22 +17,19 @@ from cli.functions.function_to_item import function_to_item_cli from cli.functions.item_to_function import item_to_function_cli from cli.marketplace.build import build_marketplace_cli -from cli.functions.new_function_item import new_item as new_function_item from cli.common.test_suite import test_suite -from cli.common.item_yaml import update_functions_yaml from cli.common.update_readme import update_readme +from cli.common.generate_item_yaml import generate_item_yaml @click.group() def cli(): pass - -cli.add_command(new_function_item) +cli.add_command(generate_item_yaml, name="generate-item-yaml") cli.add_command(item_to_function_cli, name="item-to-function") cli.add_command(function_to_item_cli, name="function-to-item") cli.add_command(test_suite, name="run-tests") cli.add_command(build_marketplace_cli, name="build-marketplace") -cli.add_command(update_functions_yaml, name="update-functions-yaml") cli.add_command(update_readme, name="update-readme") if __name__ == "__main__": diff --git a/cli/common/generate_item_yaml.py b/cli/common/generate_item_yaml.py new file mode 100644 index 000000000..a1042ee9f --- /dev/null +++ b/cli/common/generate_item_yaml.py @@ -0,0 +1,53 @@ +import sys +from pathlib import Path +from datetime import datetime +import click +from jinja2 import Environment, FileSystemLoader + +TEMPLATES = { + "function": "cli/utils/function_item_template.yaml.j2", + "module": "cli/utils/module_item_template.yaml.j2", +} + + +@click.command() +@click.argument("type", type=click.Choice(TEMPLATES.keys())) +@click.argument("name") +def generate_item_yaml(type, name): + """ + Generate an item.yaml file from a template. + + TYPE: one of {function, module} + NAME: name of the function/module (also directory name) + """ + # Construct the target path + path = Path(f"{type}s/src/{name}").resolve() + output_file = path / "item.yaml" + + if output_file.exists(): + click.echo(f"Error: {output_file} already exists.", err=True) + sys.exit(1) + + if not path.exists(): + click.echo(f"Error: {path} does not exist.", err=True) + sys.exit(1) + + # Render parameters + params = { + "example": f"{name}.ipynb", + "generationDate": datetime.utcnow().strftime("%Y-%m-%d"), + "name": name, + "filename": f"{name}.py", + } + + # Load and render template + env = Environment(loader=FileSystemLoader(".")) + template = env.get_template(TEMPLATES[type]) + rendered = template.render(params) + + output_file.write_text(rendered) + click.echo(f"Created {output_file}") + + +if __name__ == "__main__": + generate_item_yaml() \ No newline at end of file diff --git a/cli/common/item_yaml.py b/cli/common/item_yaml.py deleted file mode 100644 index a14ea48c2..000000000 --- a/cli/common/item_yaml.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2019 Iguazio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import click -from cli.utils.path_iterator import PathIterator -from cli.utils.helpers import is_item_dir -import yaml -import datetime - - -@click.command() -@click.option("-r", "--root-directory", default=".", help="Path to root directory") -@click.option("-v", "--version", help="update version number in function item yaml") -@click.option("-mv", "--mlrun-version", help="update mlrun version in function item.yaml") -@click.option("-p", "--platform-version", help="update platform version in function item.yaml") -@click.option("-d", "--date-time", help="update date-time in function item.yaml") -def update_functions_yaml(root_directory: str, - version: str, - mlrun_version: str, - platform_version: str, - date_time: str): - if not root_directory: - click.echo("-r/--root-directory is required") - exit(1) - - item_iterator = PathIterator(root=root_directory, rule=is_item_dir, as_path=True) - for inner_dir in item_iterator: - item_yaml = "item.yaml" - if (inner_dir / item_yaml).exists(): - path = str(inner_dir)+"/"+item_yaml - stream = open(path, 'r') - data = yaml.load(stream=stream, Loader=yaml.FullLoader) - if version: - data['version'] = version - if mlrun_version: - data['mlrunVersion'] = mlrun_version - if platform_version: - data['platformVersion'] = platform_version - if date_time: - data['generationDate'] = datetime.datetime.now().strftime('%Y-%m-%d:%H-%M') - print(data) - with open(path, 'w') as yaml_file: - yaml_file.write(yaml.dump(data, default_flow_style=False)) diff --git a/cli/functions/new_function_item.py b/cli/functions/new_function_item.py deleted file mode 100644 index 70eb30d55..000000000 --- a/cli/functions/new_function_item.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2019 Iguazio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from datetime import datetime -from pathlib import Path - -import click - - -@click.command() -@click.option( - "-p", "--path", help="Path to directory in which a new item.yaml will be created" -) -@click.option("-o", "--override", is_flag=True, help="Override if already exists") -def new_item(path: str, override: bool): - path = Path(path) / "item.yaml" - - if not path.parent.exists(): - path.parent.mkdir(parents=True) - elif path.exists() and not override: - click.echo( - f"{path / 'item.yaml'} already exists, set [-o, --override] to override" - ) - exit(1) - - with open(path, "w") as f: - f.write( - f""" -apiVersion: v1 -categories: [] # List of category names -description: '' # Short description -doc: '' # Path to README.md if exists -example: '' # Path to examole notebook -generationDate: {str(datetime.utcnow())} -icon: '' # Path to icon file -labels: {{}} # Key values label pairs -maintainers: [] # List of maintainers -mlrunVersion: '' # Function’s MLRun version requirement, should follow python’s versioning schema -name: '' # Function name -platformVersion: '' # Function’s Iguazio version requirement, should follow python’s versioning schema -spec: - filename: '' # Implementation file - handler: '' # Handler function name - image: '' # Base image name - kind: '' # Function kind - requirements: [] # List of Pythonic library requirements - customFields: {{}} # Custom spec fields - env: [] # Spec environment params -url: '' -version: 0.0.1 # Function version, should follow standard semantic versioning schema -""" - ) - - -if __name__ == "__main__": - new_item() diff --git a/cli/utils/function_item_template.yaml.j2 b/cli/utils/function_item_template.yaml.j2 new file mode 100644 index 000000000..da35ef819 --- /dev/null +++ b/cli/utils/function_item_template.yaml.j2 @@ -0,0 +1,22 @@ +apiVersion: v1 +categories: [] {# List of category names #} +description: '' {# Short description #} +doc: '' {# Path to README.md if exists #} +example: {{ example|default('') }} {# Path to example notebook #} +generationDate: {{ generationDate|default('') }} {# Automatically generated ISO8086 datetime #} +hidden: false {# Hide function from the UI #} +icon: '' {# Path to icon file #} +labels: {# Key values label pairs #} + author: Iguazio +maintainers: [] {# List of maintainers #} +mlrunVersion: '' {# Function’s MLRun version requirement, should follow python’s versioning schema #} +name: {{ name|default('') }} {# Function name #} +platformVersion: '' {# Function’s Iguazio version requirement, should follow python’s versioning schema #} +spec: + filename: {{ filename|default('') }} {# Implementation file #} + handler: '' {# Handler function name #} + image: mlrun/mlrun {# Base image name #} + kind: '' {# Function kind #} + requirements: [] {# List of Pythonic library requirements #} +url: '' +version: 1.0.0 {# Function version, should follow standard semantic versioning schema #} \ No newline at end of file diff --git a/cli/utils/item_template.yaml b/cli/utils/item_template.yaml deleted file mode 100644 index b1d38d334..000000000 --- a/cli/utils/item_template.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -categories: [] # List of category names -description: '' # Short description -doc: '' # Path to README.md if exists -example: '' # Path to examole notebook -generationDate: '' # Automatically generated ISO8086 datetime -hidden: false # Hide function from the UI -icon: '' # Path to icon file -labels: {} # Key values label pairs -maintainers: [] # List of maintainers -mlrunVersion: '' # Function’s MLRun version requirement, should follow python’s versioning schema -name: '' # Function name -platformVersion: '' # Function’s Iguazio version requirement, should follow python’s versioning schema -spec: - filename: '' # Implementation file - handler: '' # Handler function name - image: '' # Base image name - kind: '' # Function kind - requirements: [] # List of Pythonic library requirements -url: '' # ??? -version: '' # Function version, should follow standard semantic versioning schema \ No newline at end of file diff --git a/cli/utils/module_item_template.yaml.j2 b/cli/utils/module_item_template.yaml.j2 new file mode 100644 index 000000000..e04408fbb --- /dev/null +++ b/cli/utils/module_item_template.yaml.j2 @@ -0,0 +1,16 @@ +apiVersion: v1 +categories: [] {# List of category names #} +description: '' {# Short description #} +example: {{ example|default('') }} {# Path to example notebook #} +generationDate: {{ generationDate|default('') }} {# Automatically generated ISO8086 datetime #} +hidden: false {# Hide function from the UI #} +labels: + author: Iguazio +mlrunVersion: '' {# Function’s MLRun version requirement, should follow python’s versioning schema #} +name: {{ name|default('') }} {# Function name #} +spec: + filename: {{ filename|default('') }} {# Implementation file #} + image: mlrun/mlrun {# Base image name #} + kind: '' {# Function kind #} + requirements: [] {# List of Pythonic library requirements #} +version: 1.0.0 {# Function version, should follow standard semantic versioning schema #} \ No newline at end of file From 36ff01698a8e38c6b0e1d672bef9b9242d7480ad Mon Sep 17 00:00:00 2001 From: danielperezz Date: Sun, 28 Sep 2025 12:47:37 +0300 Subject: [PATCH 2/4] modify comments to module --- cli/utils/module_item_template.yaml.j2 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/utils/module_item_template.yaml.j2 b/cli/utils/module_item_template.yaml.j2 index e04408fbb..539cd6f0a 100644 --- a/cli/utils/module_item_template.yaml.j2 +++ b/cli/utils/module_item_template.yaml.j2 @@ -3,14 +3,14 @@ categories: [] {# List of category names #} description: '' {# Short description #} example: {{ example|default('') }} {# Path to example notebook #} generationDate: {{ generationDate|default('') }} {# Automatically generated ISO8086 datetime #} -hidden: false {# Hide function from the UI #} +hidden: false {# Hide Module from the UI #} labels: author: Iguazio -mlrunVersion: '' {# Function’s MLRun version requirement, should follow python’s versioning schema #} -name: {{ name|default('') }} {# Function name #} +mlrunVersion: '' {# Module’s MLRun version requirement, should follow python’s versioning schema #} +name: {{ name|default('') }} {# Module name #} spec: filename: {{ filename|default('') }} {# Implementation file #} image: mlrun/mlrun {# Base image name #} - kind: '' {# Function kind #} + kind: '' {# Module kind #} requirements: [] {# List of Pythonic library requirements #} -version: 1.0.0 {# Function version, should follow standard semantic versioning schema #} \ No newline at end of file +version: 1.0.0 {# Module version, should follow standard semantic versioning schema #} \ No newline at end of file From 7ed10a1f84b4a082211cececb713ec0725d07f23 Mon Sep 17 00:00:00 2001 From: danielperezz Date: Tue, 30 Sep 2025 11:05:34 +0300 Subject: [PATCH 3/4] PR fixes --- cli/common/generate_item_yaml.py | 12 +++++++----- requirements.txt | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cli/common/generate_item_yaml.py b/cli/common/generate_item_yaml.py index a1042ee9f..ee319a2d6 100644 --- a/cli/common/generate_item_yaml.py +++ b/cli/common/generate_item_yaml.py @@ -11,20 +11,22 @@ @click.command() -@click.argument("type", type=click.Choice(TEMPLATES.keys())) +@click.argument("type", type=click.Choice(list(TEMPLATES.keys()))) @click.argument("name") -def generate_item_yaml(type, name): +@click.option("--overwrite", is_flag=True, help="Replace existing file instead of raising an error.") +def generate_item_yaml(type, name, overwrite: bool = False): """ Generate an item.yaml file from a template. - TYPE: one of {function, module} - NAME: name of the function/module (also directory name) +type: one of the supported types (currently only `function` or `module`) +name: the function/module name (also used as the directory name) +overwrite: whether to overwrite existing item.yaml file """ # Construct the target path path = Path(f"{type}s/src/{name}").resolve() output_file = path / "item.yaml" - if output_file.exists(): + if not overwrite and output_file.exists(): click.echo(f"Error: {output_file} already exists.", err=True) sys.exit(1) diff --git a/requirements.txt b/requirements.txt index e58ca8e98..c393fd552 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ wheel bs4 mlrun>=1.0.0 jinja2~=3.1.2 +click>=8.0 pipenv myst_nb black>=24.3.0 From c8622ddba61c1f55c976dbe6332b2cb8ca144dbe Mon Sep 17 00:00:00 2001 From: Daniel Perez <100069700+danielperezz@users.noreply.github.com> Date: Tue, 30 Sep 2025 17:23:10 +0300 Subject: [PATCH 4/4] Update cli/common/generate_item_yaml.py Co-authored-by: Eyal Danieli --- cli/common/generate_item_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/common/generate_item_yaml.py b/cli/common/generate_item_yaml.py index ee319a2d6..9ce362c37 100644 --- a/cli/common/generate_item_yaml.py +++ b/cli/common/generate_item_yaml.py @@ -14,7 +14,7 @@ @click.argument("type", type=click.Choice(list(TEMPLATES.keys()))) @click.argument("name") @click.option("--overwrite", is_flag=True, help="Replace existing file instead of raising an error.") -def generate_item_yaml(type, name, overwrite: bool = False): +def generate_item_yaml(type: str, name: str, overwrite: bool = False): """ Generate an item.yaml file from a template.