Cherry pick new modules and fixes

.github/workflows/test-all.yaml

@@ -116,6 +116,10 @@ jobs:
     permissions:
       contents: write
     steps:
+      - name: Get the current branch name
+        shell: bash
+        run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
+        id: branch
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -128,7 +132,9 @@ jobs:
           pip install --upgrade pip
           pip install -r requirements.txt
       - name: Regenerate README tables
-        run: python -m cli.cli update-readme --asset functions --asset modules
+        env:
+          CHANNEL: ${{ steps.branch.outputs.branch }}
+        run: python -m cli.cli update-readme -c $CHANNEL --asset functions --asset modules
       - name: Commit & push (if changed)
         env:
           USERNAME: ${{ secrets.USERNAME }}

cli/README.md

@@ -60,7 +60,7 @@ Example:
 Regenerate the `README.md` files in each of the asset directories (functions/modules).
 
 Usage:
-    `python -m cli.cli update-readme --asset TYPE`
+    `python -m cli.cli update-readme -c CHANNEL --asset TYPE`
 
 Example:
-    `python -m cli.cli update-readme --asset functions --asset modules`
+    `python -m cli.cli update-readme -c master --asset functions --asset modules`

cli/common/update_readme.py

@@ -25,6 +25,7 @@
 COLUMNS = ("Name", "Description", "Kind", "Categories")
 
 @click.command("update-readme")
+@click.option("-c", "--channel", default="master", help="Name of build channel")
 @click.option(
     "--asset",
     multiple=True,
@@ -34,7 +35,7 @@
 )
 @click.option("--check", is_flag=True,
               help="Do not write; exit non‑zero if README(s) would change.")
-def update_readme(asset: Iterable[str],
+def update_readme(channel: str, asset: Iterable[str],
                       check: bool) -> None:
     """
     Regenerate the README tables for asset types from their item.yaml files.
@@ -50,15 +51,15 @@ def update_readme(asset: Iterable[str],
             root = Path(".").resolve()
             asset_dir = root / t
             readme = asset_dir / "README.md"
-            rows = _rows_for_asset_type(asset_dir)
+            rows = _rows_for_asset_type(channel, asset_dir)
             table_md = _build_table_md(rows)
             old = readme.read_text() if readme.exists() else f"# {t.title()}\n\n"
             new = _replace_block(old, table_md)
             if new != old:
                 changed_any = True
                 touched.append(str(readme))
         else:
-            if _update_one(t):
+            if _update_one(channel, t):
                 changed_any = True
                 touched.append(str((Path(t) / "README.md").as_posix()))
 
@@ -78,7 +79,7 @@ def update_readme(asset: Iterable[str],
             click.echo("No README changes.")
 
 
-def _rows_for_asset_type(asset_dir: Path) -> List[Tuple[str, str, str, str]]:
+def _rows_for_asset_type(channel: str, asset_dir: Path) -> List[Tuple[str, str, str, str]]:
     """Scan <asset>/src/*/item.yaml and return table rows."""
     src = asset_dir / "src"
     if not src.exists():
@@ -97,7 +98,9 @@ def _rows_for_asset_type(asset_dir: Path) -> List[Tuple[str, str, str, str]]:
         cats = data.get("categories") or []
         cats_str = ", ".join(c.strip() for c in cats) if isinstance(cats, list) else str(cats).strip()
         # Link the name to its source directory
-        link = f"[{asset_name}]({(asset_dir / 'src' / asset_name).as_posix()})"
+        # Construct the relative path from the repo root for the asset
+        rel_path = asset_dir.relative_to(Path(".").resolve())
+        link = f"[{asset_name}](https://github.com/mlrun/functions/tree/{channel}/{rel_path}/src/{asset_name})"
         rows.append((link, desc, kind, cats_str))
 
     rows.sort(key=lambda r: r[0].lower())
@@ -140,13 +143,13 @@ def _replace_block(readme_text: str, new_block: str) -> str:
     return readme_text[:start_close] + "\n" + new_block + "\n" + readme_text[ei:]
 
 
-def _update_one(asset_type: str) -> bool:
+def _update_one(channel: str, asset_type: str) -> bool:
     """Generate/replace the table in <asset_type>/README.md. Return True if changed."""
     root = Path(".").resolve()
     asset_dir = root / asset_type
     readme = asset_dir / "README.md"
 
-    rows = _rows_for_asset_type(asset_dir)
+    rows = _rows_for_asset_type(channel, asset_dir)
     table_md = _build_table_md(rows)
     old = readme.read_text() if readme.exists() else f"# {asset_type.title()}\n\n"
     new = _replace_block(old, table_md)

cli/marketplace/conf.template

@@ -15,8 +15,14 @@ import re
 import sys
 import os
 
-sys.path.insert(0, "{{sphinx_docs_target}}")
-sys.path.insert(0, os.path.abspath(os.path.join("{{sphinx_docs_target}}", "../functions")))
+import pathlib
+
+DOCS_DIR = pathlib.Path(__file__).resolve().parent
+REPO_ROOT = DOCS_DIR.parent
+
+# Add both source trees
+sys.path.insert(0, str(REPO_ROOT / "functions"))
+sys.path.insert(0, str(REPO_ROOT / "modules"))
 
 
 # -- Project information -----------------------------------------------------

functions/README.md

@@ -9,40 +9,40 @@ it is expected that contributors follow certain guidelines/protocols (please chi
 <!-- AUTOGEN:START (do not edit below) -->
 | Name | Description | Kind | Categories |
 | --- | --- | --- | --- |
-| [aggregate](/home/runner/work/functions/functions/functions/src/aggregate) | Rolling aggregation over Metrics and Lables according to specifications | job | data-preparation |
-| [arc_to_parquet](/home/runner/work/functions/functions/functions/src/arc_to_parquet) | retrieve remote archive, open and save as parquet | job | utils |
-| [auto_trainer](/home/runner/work/functions/functions/functions/src/auto_trainer) | Automatic train, evaluate and predict functions for the ML frameworks - Scikit-Learn, XGBoost and LightGBM. | job | machine-learning, model-training |
-| [azureml_serving](/home/runner/work/functions/functions/functions/src/azureml_serving) | AzureML serving function | serving | machine-learning, model-serving |
-| [azureml_utils](/home/runner/work/functions/functions/functions/src/azureml_utils) | Azure AutoML integration in MLRun, including utils functions for training models on Azure AutoML platfrom. | job | model-serving, utils |
-| [batch_inference](/home/runner/work/functions/functions/functions/src/batch_inference) | Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. | job | model-serving |
-| [batch_inference_v2](/home/runner/work/functions/functions/functions/src/batch_inference_v2) | Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. | job | model-serving |
-| [describe](/home/runner/work/functions/functions/functions/src/describe) | describe and visualizes dataset stats | job | data-analysis |
-| [describe_dask](/home/runner/work/functions/functions/functions/src/describe_dask) | describe and visualizes dataset stats | job | data-analysis |
-| [describe_spark](/home/runner/work/functions/functions/functions/src/describe_spark) |  | job | data-analysis |
-| [feature_selection](/home/runner/work/functions/functions/functions/src/feature_selection) | Select features through multiple Statistical and Model filters | job | data-preparation, machine-learning |
-| [gen_class_data](/home/runner/work/functions/functions/functions/src/gen_class_data) | Create a binary classification sample dataset and save. | job | data-generation |
-| [github_utils](/home/runner/work/functions/functions/functions/src/github_utils) | add comments to github pull request | job | utils |
-| [hugging_face_serving](/home/runner/work/functions/functions/functions/src/hugging_face_serving) | Generic Hugging Face model server. | serving | genai, model-serving |
-| [load_dataset](/home/runner/work/functions/functions/functions/src/load_dataset) | load a toy dataset from scikit-learn | job | data-preparation |
-| [mlflow_utils](/home/runner/work/functions/functions/functions/src/mlflow_utils) | Mlflow model server, and additional utils. | serving | model-serving, utils |
-| [model_server](/home/runner/work/functions/functions/functions/src/model_server) | generic sklearn model server | nuclio:serving | model-serving, machine-learning |
-| [model_server_tester](/home/runner/work/functions/functions/functions/src/model_server_tester) | test model servers | job | monitoring, model-serving |
-| [noise_reduction](/home/runner/work/functions/functions/functions/src/noise_reduction) | Reduce noise from audio files | job | data-preparation, audio |
-| [onnx_utils](/home/runner/work/functions/functions/functions/src/onnx_utils) | ONNX intigration in MLRun, some utils functions for the ONNX framework, optimizing and converting models from different framework to ONNX using MLRun. | job | utils, deep-learning |
-| [open_archive](/home/runner/work/functions/functions/functions/src/open_archive) | Open a file/object archive into a target directory | job | utils |
-| [pii_recognizer](/home/runner/work/functions/functions/functions/src/pii_recognizer) | This function is used to recognize PII in a directory of text files | job | data-preparation, NLP |
-| [pyannote_audio](/home/runner/work/functions/functions/functions/src/pyannote_audio) | pyannote's speech diarization of audio files | job | deep-learning, audio |
-| [question_answering](/home/runner/work/functions/functions/functions/src/question_answering) | GenAI approach of question answering on a given data | job | genai |
-| [send_email](/home/runner/work/functions/functions/functions/src/send_email) | Send Email messages through SMTP server | job | utils |
-| [silero_vad](/home/runner/work/functions/functions/functions/src/silero_vad) | Silero VAD (Voice Activity Detection) functions. | job | deep-learning, audio |
-| [sklearn_classifier](/home/runner/work/functions/functions/functions/src/sklearn_classifier) | train any classifier using scikit-learn's API | job | machine-learning, model-training |
-| [sklearn_classifier_dask](/home/runner/work/functions/functions/functions/src/sklearn_classifier_dask) | train any classifier using scikit-learn's API over Dask | job | machine-learning, model-training |
-| [structured_data_generator](/home/runner/work/functions/functions/functions/src/structured_data_generator) | GenAI approach of generating structured data according to a given schema | job | data-generation, genai |
-| [test_classifier](/home/runner/work/functions/functions/functions/src/test_classifier) | test a classifier using held-out or new data | job | machine-learning, model-testing |
-| [text_to_audio_generator](/home/runner/work/functions/functions/functions/src/text_to_audio_generator) | Generate audio file from text using different speakers | job | data-generation, audio |
-| [tf2_serving](/home/runner/work/functions/functions/functions/src/tf2_serving) | tf2 image classification server | nuclio:serving | model-serving, machine-learning |
-| [transcribe](/home/runner/work/functions/functions/functions/src/transcribe) | Transcribe audio files into text files | job | audio, genai |
-| [translate](/home/runner/work/functions/functions/functions/src/translate) | Translate text files from one language to another | job | genai, NLP |
-| [v2_model_server](/home/runner/work/functions/functions/functions/src/v2_model_server) | generic sklearn model server | serving | model-serving, machine-learning |
-| [v2_model_tester](/home/runner/work/functions/functions/functions/src/v2_model_tester) | test v2 model servers | job | model-testing, machine-learning |
+| [aggregate](https://github.com/mlrun/functions/tree/development/functions/src/aggregate) | Rolling aggregation over Metrics and Lables according to specifications | job | data-preparation |
+| [arc_to_parquet](https://github.com/mlrun/functions/tree/development/functions/src/arc_to_parquet) | retrieve remote archive, open and save as parquet | job | utils |
+| [auto_trainer](https://github.com/mlrun/functions/tree/development/functions/src/auto_trainer) | Automatic train, evaluate and predict functions for the ML frameworks - Scikit-Learn, XGBoost and LightGBM. | job | machine-learning, model-training |
+| [azureml_serving](https://github.com/mlrun/functions/tree/development/functions/src/azureml_serving) | AzureML serving function | serving | machine-learning, model-serving |
+| [azureml_utils](https://github.com/mlrun/functions/tree/development/functions/src/azureml_utils) | Azure AutoML integration in MLRun, including utils functions for training models on Azure AutoML platfrom. | job | model-serving, utils |
+| [batch_inference](https://github.com/mlrun/functions/tree/development/functions/src/batch_inference) | Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. | job | model-serving |
+| [batch_inference_v2](https://github.com/mlrun/functions/tree/development/functions/src/batch_inference_v2) | Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. | job | model-serving |
+| [describe](https://github.com/mlrun/functions/tree/development/functions/src/describe) | describe and visualizes dataset stats | job | data-analysis |
+| [describe_dask](https://github.com/mlrun/functions/tree/development/functions/src/describe_dask) | describe and visualizes dataset stats | job | data-analysis |
+| [describe_spark](https://github.com/mlrun/functions/tree/development/functions/src/describe_spark) |  | job | data-analysis |
+| [feature_selection](https://github.com/mlrun/functions/tree/development/functions/src/feature_selection) | Select features through multiple Statistical and Model filters | job | data-preparation, machine-learning |
+| [gen_class_data](https://github.com/mlrun/functions/tree/development/functions/src/gen_class_data) | Create a binary classification sample dataset and save. | job | data-generation |
+| [github_utils](https://github.com/mlrun/functions/tree/development/functions/src/github_utils) | add comments to github pull request | job | utils |
+| [hugging_face_serving](https://github.com/mlrun/functions/tree/development/functions/src/hugging_face_serving) | Generic Hugging Face model server. | serving | genai, model-serving |
+| [load_dataset](https://github.com/mlrun/functions/tree/development/functions/src/load_dataset) | load a toy dataset from scikit-learn | job | data-preparation |
+| [mlflow_utils](https://github.com/mlrun/functions/tree/development/functions/src/mlflow_utils) | Mlflow model server, and additional utils. | serving | model-serving, utils |
+| [model_server](https://github.com/mlrun/functions/tree/development/functions/src/model_server) | generic sklearn model server | nuclio:serving | model-serving, machine-learning |
+| [model_server_tester](https://github.com/mlrun/functions/tree/development/functions/src/model_server_tester) | test model servers | job | monitoring, model-serving |
+| [noise_reduction](https://github.com/mlrun/functions/tree/development/functions/src/noise_reduction) | Reduce noise from audio files | job | data-preparation, audio |
+| [onnx_utils](https://github.com/mlrun/functions/tree/development/functions/src/onnx_utils) | ONNX intigration in MLRun, some utils functions for the ONNX framework, optimizing and converting models from different framework to ONNX using MLRun. | job | utils, deep-learning |
+| [open_archive](https://github.com/mlrun/functions/tree/development/functions/src/open_archive) | Open a file/object archive into a target directory | job | utils |
+| [pii_recognizer](https://github.com/mlrun/functions/tree/development/functions/src/pii_recognizer) | This function is used to recognize PII in a directory of text files | job | data-preparation, NLP |
+| [pyannote_audio](https://github.com/mlrun/functions/tree/development/functions/src/pyannote_audio) | pyannote's speech diarization of audio files | job | deep-learning, audio |
+| [question_answering](https://github.com/mlrun/functions/tree/development/functions/src/question_answering) | GenAI approach of question answering on a given data | job | genai |
+| [send_email](https://github.com/mlrun/functions/tree/development/functions/src/send_email) | Send Email messages through SMTP server | job | utils |
+| [silero_vad](https://github.com/mlrun/functions/tree/development/functions/src/silero_vad) | Silero VAD (Voice Activity Detection) functions. | job | deep-learning, audio |
+| [sklearn_classifier](https://github.com/mlrun/functions/tree/development/functions/src/sklearn_classifier) | train any classifier using scikit-learn's API | job | machine-learning, model-training |
+| [sklearn_classifier_dask](https://github.com/mlrun/functions/tree/development/functions/src/sklearn_classifier_dask) | train any classifier using scikit-learn's API over Dask | job | machine-learning, model-training |
+| [structured_data_generator](https://github.com/mlrun/functions/tree/development/functions/src/structured_data_generator) | GenAI approach of generating structured data according to a given schema | job | data-generation, genai |
+| [test_classifier](https://github.com/mlrun/functions/tree/development/functions/src/test_classifier) | test a classifier using held-out or new data | job | machine-learning, model-testing |
+| [text_to_audio_generator](https://github.com/mlrun/functions/tree/development/functions/src/text_to_audio_generator) | Generate audio file from text using different speakers | job | data-generation, audio |
+| [tf2_serving](https://github.com/mlrun/functions/tree/development/functions/src/tf2_serving) | tf2 image classification server | nuclio:serving | model-serving, machine-learning |
+| [transcribe](https://github.com/mlrun/functions/tree/development/functions/src/transcribe) | Transcribe audio files into text files | job | audio, genai |
+| [translate](https://github.com/mlrun/functions/tree/development/functions/src/translate) | Translate text files from one language to another | job | genai, NLP |
+| [v2_model_server](https://github.com/mlrun/functions/tree/development/functions/src/v2_model_server) | generic sklearn model server | serving | model-serving, machine-learning |
+| [v2_model_tester](https://github.com/mlrun/functions/tree/development/functions/src/v2_model_tester) | test v2 model servers | job | model-testing, machine-learning |
 <!-- AUTOGEN:END -->

functions/src/translate/item.yaml

@@ -12,7 +12,7 @@ labels:
   author: Iguazio
 maintainers: []
 marketplaceType: ''
-mlrunVersion: 1.7.0
+mlrunVersion: 1.10.0-rc41
 name: translate
 platformVersion: 3.5.3
 spec:
@@ -23,8 +23,8 @@ spec:
   requirements:
     - transformers
     - sentencepiece
-    - torch
+    - torch>=2.6
     - tqdm
 url: ''
-version: 0.2.0
+version: 0.3.0
 test_valid: True

functions/src/translate/requirements.txt

@@ -1,4 +1,4 @@
 transformers
 tqdm
-torch
+torch>=2.6
 sentencepiece