diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index 162804863..d8eb6c6ed 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -116,6 +116,10 @@ jobs: permissions: contents: write steps: + - name: Get the current branch name + shell: bash + run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT + id: branch - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -128,7 +132,9 @@ jobs: pip install --upgrade pip pip install -r requirements.txt - name: Regenerate README tables - run: python -m cli.cli update-readme --asset functions --asset modules + env: + CHANNEL: ${{ steps.branch.outputs.branch }} + run: python -m cli.cli update-readme -c $CHANNEL --asset functions --asset modules - name: Commit & push (if changed) env: USERNAME: ${{ secrets.USERNAME }} diff --git a/cli/README.md b/cli/README.md index 4a3cd3bfc..69749536b 100644 --- a/cli/README.md +++ b/cli/README.md @@ -60,7 +60,7 @@ Example: Regenerate the `README.md` files in each of the asset directories (functions/modules). Usage: - `python -m cli.cli update-readme --asset TYPE` + `python -m cli.cli update-readme -c CHANNEL --asset TYPE` Example: - `python -m cli.cli update-readme --asset functions --asset modules` \ No newline at end of file + `python -m cli.cli update-readme -c master --asset functions --asset modules` diff --git a/cli/common/update_readme.py b/cli/common/update_readme.py index 6bcab8d33..89b6aa094 100644 --- a/cli/common/update_readme.py +++ b/cli/common/update_readme.py @@ -25,6 +25,7 @@ COLUMNS = ("Name", "Description", "Kind", "Categories") @click.command("update-readme") +@click.option("-c", "--channel", default="master", help="Name of build channel") @click.option( "--asset", multiple=True, @@ -34,7 +35,7 @@ ) @click.option("--check", is_flag=True, help="Do not write; exit non‑zero if README(s) would change.") -def update_readme(asset: Iterable[str], +def update_readme(channel: str, asset: Iterable[str], check: bool) -> None: """ Regenerate the README tables for asset types from their item.yaml files. @@ -50,7 +51,7 @@ def update_readme(asset: Iterable[str], root = Path(".").resolve() asset_dir = root / t readme = asset_dir / "README.md" - rows = _rows_for_asset_type(asset_dir) + rows = _rows_for_asset_type(channel, asset_dir) table_md = _build_table_md(rows) old = readme.read_text() if readme.exists() else f"# {t.title()}\n\n" new = _replace_block(old, table_md) @@ -58,7 +59,7 @@ def update_readme(asset: Iterable[str], changed_any = True touched.append(str(readme)) else: - if _update_one(t): + if _update_one(channel, t): changed_any = True touched.append(str((Path(t) / "README.md").as_posix())) @@ -78,7 +79,7 @@ def update_readme(asset: Iterable[str], click.echo("No README changes.") -def _rows_for_asset_type(asset_dir: Path) -> List[Tuple[str, str, str, str]]: +def _rows_for_asset_type(channel: str, asset_dir: Path) -> List[Tuple[str, str, str, str]]: """Scan /src/*/item.yaml and return table rows.""" src = asset_dir / "src" if not src.exists(): @@ -97,7 +98,9 @@ def _rows_for_asset_type(asset_dir: Path) -> List[Tuple[str, str, str, str]]: cats = data.get("categories") or [] cats_str = ", ".join(c.strip() for c in cats) if isinstance(cats, list) else str(cats).strip() # Link the name to its source directory - link = f"[{asset_name}]({(asset_dir / 'src' / asset_name).as_posix()})" + # Construct the relative path from the repo root for the asset + rel_path = asset_dir.relative_to(Path(".").resolve()) + link = f"[{asset_name}](https://github.com/mlrun/functions/tree/{channel}/{rel_path}/src/{asset_name})" rows.append((link, desc, kind, cats_str)) rows.sort(key=lambda r: r[0].lower()) @@ -140,13 +143,13 @@ def _replace_block(readme_text: str, new_block: str) -> str: return readme_text[:start_close] + "\n" + new_block + "\n" + readme_text[ei:] -def _update_one(asset_type: str) -> bool: +def _update_one(channel: str, asset_type: str) -> bool: """Generate/replace the table in /README.md. Return True if changed.""" root = Path(".").resolve() asset_dir = root / asset_type readme = asset_dir / "README.md" - rows = _rows_for_asset_type(asset_dir) + rows = _rows_for_asset_type(channel, asset_dir) table_md = _build_table_md(rows) old = readme.read_text() if readme.exists() else f"# {asset_type.title()}\n\n" new = _replace_block(old, table_md) diff --git a/cli/marketplace/conf.template b/cli/marketplace/conf.template index 93c83c9d3..e26f065aa 100644 --- a/cli/marketplace/conf.template +++ b/cli/marketplace/conf.template @@ -15,8 +15,14 @@ import re import sys import os -sys.path.insert(0, "{{sphinx_docs_target}}") -sys.path.insert(0, os.path.abspath(os.path.join("{{sphinx_docs_target}}", "../functions"))) +import pathlib + +DOCS_DIR = pathlib.Path(__file__).resolve().parent +REPO_ROOT = DOCS_DIR.parent + +# Add both source trees +sys.path.insert(0, str(REPO_ROOT / "functions")) +sys.path.insert(0, str(REPO_ROOT / "modules")) # -- Project information ----------------------------------------------------- diff --git a/functions/README.md b/functions/README.md index 08b1c7ad9..3618833a5 100644 --- a/functions/README.md +++ b/functions/README.md @@ -9,40 +9,40 @@ it is expected that contributors follow certain guidelines/protocols (please chi | Name | Description | Kind | Categories | | --- | --- | --- | --- | -| [aggregate](/home/runner/work/functions/functions/functions/src/aggregate) | Rolling aggregation over Metrics and Lables according to specifications | job | data-preparation | -| [arc_to_parquet](/home/runner/work/functions/functions/functions/src/arc_to_parquet) | retrieve remote archive, open and save as parquet | job | utils | -| [auto_trainer](/home/runner/work/functions/functions/functions/src/auto_trainer) | Automatic train, evaluate and predict functions for the ML frameworks - Scikit-Learn, XGBoost and LightGBM. | job | machine-learning, model-training | -| [azureml_serving](/home/runner/work/functions/functions/functions/src/azureml_serving) | AzureML serving function | serving | machine-learning, model-serving | -| [azureml_utils](/home/runner/work/functions/functions/functions/src/azureml_utils) | Azure AutoML integration in MLRun, including utils functions for training models on Azure AutoML platfrom. | job | model-serving, utils | -| [batch_inference](/home/runner/work/functions/functions/functions/src/batch_inference) | Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. | job | model-serving | -| [batch_inference_v2](/home/runner/work/functions/functions/functions/src/batch_inference_v2) | Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. | job | model-serving | -| [describe](/home/runner/work/functions/functions/functions/src/describe) | describe and visualizes dataset stats | job | data-analysis | -| [describe_dask](/home/runner/work/functions/functions/functions/src/describe_dask) | describe and visualizes dataset stats | job | data-analysis | -| [describe_spark](/home/runner/work/functions/functions/functions/src/describe_spark) | | job | data-analysis | -| [feature_selection](/home/runner/work/functions/functions/functions/src/feature_selection) | Select features through multiple Statistical and Model filters | job | data-preparation, machine-learning | -| [gen_class_data](/home/runner/work/functions/functions/functions/src/gen_class_data) | Create a binary classification sample dataset and save. | job | data-generation | -| [github_utils](/home/runner/work/functions/functions/functions/src/github_utils) | add comments to github pull request | job | utils | -| [hugging_face_serving](/home/runner/work/functions/functions/functions/src/hugging_face_serving) | Generic Hugging Face model server. | serving | genai, model-serving | -| [load_dataset](/home/runner/work/functions/functions/functions/src/load_dataset) | load a toy dataset from scikit-learn | job | data-preparation | -| [mlflow_utils](/home/runner/work/functions/functions/functions/src/mlflow_utils) | Mlflow model server, and additional utils. | serving | model-serving, utils | -| [model_server](/home/runner/work/functions/functions/functions/src/model_server) | generic sklearn model server | nuclio:serving | model-serving, machine-learning | -| [model_server_tester](/home/runner/work/functions/functions/functions/src/model_server_tester) | test model servers | job | monitoring, model-serving | -| [noise_reduction](/home/runner/work/functions/functions/functions/src/noise_reduction) | Reduce noise from audio files | job | data-preparation, audio | -| [onnx_utils](/home/runner/work/functions/functions/functions/src/onnx_utils) | ONNX intigration in MLRun, some utils functions for the ONNX framework, optimizing and converting models from different framework to ONNX using MLRun. | job | utils, deep-learning | -| [open_archive](/home/runner/work/functions/functions/functions/src/open_archive) | Open a file/object archive into a target directory | job | utils | -| [pii_recognizer](/home/runner/work/functions/functions/functions/src/pii_recognizer) | This function is used to recognize PII in a directory of text files | job | data-preparation, NLP | -| [pyannote_audio](/home/runner/work/functions/functions/functions/src/pyannote_audio) | pyannote's speech diarization of audio files | job | deep-learning, audio | -| [question_answering](/home/runner/work/functions/functions/functions/src/question_answering) | GenAI approach of question answering on a given data | job | genai | -| [send_email](/home/runner/work/functions/functions/functions/src/send_email) | Send Email messages through SMTP server | job | utils | -| [silero_vad](/home/runner/work/functions/functions/functions/src/silero_vad) | Silero VAD (Voice Activity Detection) functions. | job | deep-learning, audio | -| [sklearn_classifier](/home/runner/work/functions/functions/functions/src/sklearn_classifier) | train any classifier using scikit-learn's API | job | machine-learning, model-training | -| [sklearn_classifier_dask](/home/runner/work/functions/functions/functions/src/sklearn_classifier_dask) | train any classifier using scikit-learn's API over Dask | job | machine-learning, model-training | -| [structured_data_generator](/home/runner/work/functions/functions/functions/src/structured_data_generator) | GenAI approach of generating structured data according to a given schema | job | data-generation, genai | -| [test_classifier](/home/runner/work/functions/functions/functions/src/test_classifier) | test a classifier using held-out or new data | job | machine-learning, model-testing | -| [text_to_audio_generator](/home/runner/work/functions/functions/functions/src/text_to_audio_generator) | Generate audio file from text using different speakers | job | data-generation, audio | -| [tf2_serving](/home/runner/work/functions/functions/functions/src/tf2_serving) | tf2 image classification server | nuclio:serving | model-serving, machine-learning | -| [transcribe](/home/runner/work/functions/functions/functions/src/transcribe) | Transcribe audio files into text files | job | audio, genai | -| [translate](/home/runner/work/functions/functions/functions/src/translate) | Translate text files from one language to another | job | genai, NLP | -| [v2_model_server](/home/runner/work/functions/functions/functions/src/v2_model_server) | generic sklearn model server | serving | model-serving, machine-learning | -| [v2_model_tester](/home/runner/work/functions/functions/functions/src/v2_model_tester) | test v2 model servers | job | model-testing, machine-learning | +| [aggregate](https://github.com/mlrun/functions/tree/development/functions/src/aggregate) | Rolling aggregation over Metrics and Lables according to specifications | job | data-preparation | +| [arc_to_parquet](https://github.com/mlrun/functions/tree/development/functions/src/arc_to_parquet) | retrieve remote archive, open and save as parquet | job | utils | +| [auto_trainer](https://github.com/mlrun/functions/tree/development/functions/src/auto_trainer) | Automatic train, evaluate and predict functions for the ML frameworks - Scikit-Learn, XGBoost and LightGBM. | job | machine-learning, model-training | +| [azureml_serving](https://github.com/mlrun/functions/tree/development/functions/src/azureml_serving) | AzureML serving function | serving | machine-learning, model-serving | +| [azureml_utils](https://github.com/mlrun/functions/tree/development/functions/src/azureml_utils) | Azure AutoML integration in MLRun, including utils functions for training models on Azure AutoML platfrom. | job | model-serving, utils | +| [batch_inference](https://github.com/mlrun/functions/tree/development/functions/src/batch_inference) | Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. | job | model-serving | +| [batch_inference_v2](https://github.com/mlrun/functions/tree/development/functions/src/batch_inference_v2) | Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. | job | model-serving | +| [describe](https://github.com/mlrun/functions/tree/development/functions/src/describe) | describe and visualizes dataset stats | job | data-analysis | +| [describe_dask](https://github.com/mlrun/functions/tree/development/functions/src/describe_dask) | describe and visualizes dataset stats | job | data-analysis | +| [describe_spark](https://github.com/mlrun/functions/tree/development/functions/src/describe_spark) | | job | data-analysis | +| [feature_selection](https://github.com/mlrun/functions/tree/development/functions/src/feature_selection) | Select features through multiple Statistical and Model filters | job | data-preparation, machine-learning | +| [gen_class_data](https://github.com/mlrun/functions/tree/development/functions/src/gen_class_data) | Create a binary classification sample dataset and save. | job | data-generation | +| [github_utils](https://github.com/mlrun/functions/tree/development/functions/src/github_utils) | add comments to github pull request | job | utils | +| [hugging_face_serving](https://github.com/mlrun/functions/tree/development/functions/src/hugging_face_serving) | Generic Hugging Face model server. | serving | genai, model-serving | +| [load_dataset](https://github.com/mlrun/functions/tree/development/functions/src/load_dataset) | load a toy dataset from scikit-learn | job | data-preparation | +| [mlflow_utils](https://github.com/mlrun/functions/tree/development/functions/src/mlflow_utils) | Mlflow model server, and additional utils. | serving | model-serving, utils | +| [model_server](https://github.com/mlrun/functions/tree/development/functions/src/model_server) | generic sklearn model server | nuclio:serving | model-serving, machine-learning | +| [model_server_tester](https://github.com/mlrun/functions/tree/development/functions/src/model_server_tester) | test model servers | job | monitoring, model-serving | +| [noise_reduction](https://github.com/mlrun/functions/tree/development/functions/src/noise_reduction) | Reduce noise from audio files | job | data-preparation, audio | +| [onnx_utils](https://github.com/mlrun/functions/tree/development/functions/src/onnx_utils) | ONNX intigration in MLRun, some utils functions for the ONNX framework, optimizing and converting models from different framework to ONNX using MLRun. | job | utils, deep-learning | +| [open_archive](https://github.com/mlrun/functions/tree/development/functions/src/open_archive) | Open a file/object archive into a target directory | job | utils | +| [pii_recognizer](https://github.com/mlrun/functions/tree/development/functions/src/pii_recognizer) | This function is used to recognize PII in a directory of text files | job | data-preparation, NLP | +| [pyannote_audio](https://github.com/mlrun/functions/tree/development/functions/src/pyannote_audio) | pyannote's speech diarization of audio files | job | deep-learning, audio | +| [question_answering](https://github.com/mlrun/functions/tree/development/functions/src/question_answering) | GenAI approach of question answering on a given data | job | genai | +| [send_email](https://github.com/mlrun/functions/tree/development/functions/src/send_email) | Send Email messages through SMTP server | job | utils | +| [silero_vad](https://github.com/mlrun/functions/tree/development/functions/src/silero_vad) | Silero VAD (Voice Activity Detection) functions. | job | deep-learning, audio | +| [sklearn_classifier](https://github.com/mlrun/functions/tree/development/functions/src/sklearn_classifier) | train any classifier using scikit-learn's API | job | machine-learning, model-training | +| [sklearn_classifier_dask](https://github.com/mlrun/functions/tree/development/functions/src/sklearn_classifier_dask) | train any classifier using scikit-learn's API over Dask | job | machine-learning, model-training | +| [structured_data_generator](https://github.com/mlrun/functions/tree/development/functions/src/structured_data_generator) | GenAI approach of generating structured data according to a given schema | job | data-generation, genai | +| [test_classifier](https://github.com/mlrun/functions/tree/development/functions/src/test_classifier) | test a classifier using held-out or new data | job | machine-learning, model-testing | +| [text_to_audio_generator](https://github.com/mlrun/functions/tree/development/functions/src/text_to_audio_generator) | Generate audio file from text using different speakers | job | data-generation, audio | +| [tf2_serving](https://github.com/mlrun/functions/tree/development/functions/src/tf2_serving) | tf2 image classification server | nuclio:serving | model-serving, machine-learning | +| [transcribe](https://github.com/mlrun/functions/tree/development/functions/src/transcribe) | Transcribe audio files into text files | job | audio, genai | +| [translate](https://github.com/mlrun/functions/tree/development/functions/src/translate) | Translate text files from one language to another | job | genai, NLP | +| [v2_model_server](https://github.com/mlrun/functions/tree/development/functions/src/v2_model_server) | generic sklearn model server | serving | model-serving, machine-learning | +| [v2_model_tester](https://github.com/mlrun/functions/tree/development/functions/src/v2_model_tester) | test v2 model servers | job | model-testing, machine-learning | diff --git a/functions/src/translate/function.yaml b/functions/src/translate/function.yaml index 9595b77a3..eb1ffd345 100644 --- a/functions/src/translate/function.yaml +++ b/functions/src/translate/function.yaml @@ -1,4 +1,8 @@ +verbose: false spec: + description: Translate text files from one language to another + filename: /Users/Daniel_Perez/PycharmProjects/functions/functions/src/translate/translate.py + command: '' entry_points: open_mpi_handler: lineno: 56 @@ -8,24 +12,24 @@ spec: - name: root_worker_inputs type: Dict[str, Any] default: null - name: open_mpi_handler - has_kwargs: false doc: '' + has_kwargs: false has_varargs: false + name: open_mpi_handler decorator: lineno: 68 parameters: - name: handler - name: decorator - has_kwargs: false doc: '' + has_kwargs: false has_varargs: false + name: decorator wrapper: lineno: 73 - name: wrapper - has_kwargs: true doc: '' + has_kwargs: true has_varargs: false + name: wrapper translate: outputs: - doc: 'A tuple of:' @@ -75,8 +79,6 @@ spec: type: bool doc: 'Whether to present logs of a progress bar and errors. Default: True.' default: false - name: translate - has_kwargs: false doc: 'Translate text files using a transformer model from Huggingface''s hub according to the source and target languages @@ -89,27 +91,26 @@ spec: * text_file - The text file path. * translation_file - The translation text file name in the output directory.' + has_kwargs: false has_varargs: false + name: translate + disable_auto_mount: false + image: '' + default_handler: translate build: + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9wZXJhdG9yCmltcG9ydCBwYXRobGliCmZyb20gZnVuY3Rvb2xzIGltcG9ydCByZWR1Y2UsIHdyYXBzCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIFR1cGxlLCBVbmlvbgoKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgdHJhbnNmb3JtZXJzCmZyb20gdHFkbSBpbXBvcnQgdHFkbQoKIyBHZXQgdGhlIGdsb2JhbCBsb2dnZXI6Cl9MT0dHRVIgPSBsb2dnaW5nLmdldExvZ2dlcigpCgoKZGVmIF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKSAtPiBUdXBsZVsibWxydW4uTUxDbGllbnRDdHgiLCAibXBpNHB5Lk1QSS5JbnRyYWNvbW0iXToKICAgIGlzX21waSA9IEZhbHNlCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgaXNfbXBpID0gY29udGV4dC5sYWJlbHMuZ2V0KCJraW5kIiwgImpvYiIpID09ICJtcGlqb2IiCgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnJvbSBtcGk0cHkgaW1wb3J0IE1QSQoKICAgICAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBNUEkuQ09NTV9XT1JMRAogICAgICAgICAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtcGk0cHlfbm90X2ZvdW5kOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIlRvIGRpc3RyaWJ1dGUgdGhlIGZ1bmN0aW9uIHVzaW5nIE1MUnVuJ3MgJ21waWpvYicgeW91IG5lZWQgdG8gaGF2ZSBgbXBpNHB5YCBwYWNrYWdlIGluIHlvdXIgIgogICAgICAgICAgICAgICAgICAgICJpbnRlcnByZXRlci4gUGxlYXNlIHJ1biBgcGlwIGluc3RhbGwgbXBpNHB5YCBhbmQgbWFrZSBzdXJlIHlvdSBoYXZlIG9wZW4tbXBpLiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIG1waTRweV9ub3RfZm91bmQKICAgICAgICBlbHNlOgogICAgICAgICAgICByZXR1cm4gY29udGV4dCwgTm9uZQogICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3IgYXMgbW9kdWxlX25vdF9mb3VuZDoKICAgICAgICBpZiBpc19tcGk6CiAgICAgICAgICAgIHJhaXNlIG1vZHVsZV9ub3RfZm91bmQKICAgIHJldHVybiBOb25lLCBOb25lCgoKZGVmIG9wZW5fbXBpX2hhbmRsZXIoCiAgICB3b3JrZXJfaW5wdXRzOiBMaXN0W3N0cl0sIHJvb3Rfd29ya2VyX2lucHV0czogRGljdFtzdHIsIEFueV0gPSBOb25lCik6CiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgQ2hlY2sgZm9yIE1MUnVuIGFuZCBPcGVuTVBJIGF2YWlsYWJpbGl0eToKICAgIGNvbnRleHQsIGNvbW0gPSBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkKCiAgICAjIENoZWNrIGlmIE1MUnVuIGlzIGF2YWlsYWJsZSwgc2V0IHRoZSBnbG9iYWwgbG9nZ2VyIHRvIE1MUnVuJ3M6CiAgICBpZiBjb250ZXh0OgogICAgICAgIF9MT0dHRVIgPSBjb250ZXh0LmxvZ2dlcgoKICAgIGRlZiBkZWNvcmF0b3IoaGFuZGxlcik6CiAgICAgICAgaWYgY29tbSBpcyBOb25lIG9yIGNvbW0uR2V0X3NpemUoKSA9PSAxOgogICAgICAgICAgICByZXR1cm4gaGFuZGxlcgoKICAgICAgICBAd3JhcHMoaGFuZGxlcikKICAgICAgICBkZWYgd3JhcHBlcigqKmt3YXJncyk6CiAgICAgICAgICAgICMgR2V0IHRoZSBvcGVuIG1waSBlbnZpcm9ubWVudCBwcm9wZXJ0aWVzOgogICAgICAgICAgICBzaXplID0gY29tbS5HZXRfc2l6ZSgpCiAgICAgICAgICAgIHJhbmsgPSBjb21tLkdldF9yYW5rKCkKCiAgICAgICAgICAgICMgR2l2ZSB0aGUgY29ycmVjdCBjaHVuayBvZiB0aGUgd29ya2VycyBpbnB1dHM6CiAgICAgICAgICAgIGZvciB3b3JrZXJfaW5wdXQgaW4gd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0ga3dhcmdzW3dvcmtlcl9pbnB1dF0KICAgICAgICAgICAgICAgIGlmIGlucHV0X2FyZ3VtZW50IGlzIE5vbmU6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIChzdHIsIHBhdGhsaWIuUGF0aCkpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gX2dldF90ZXh0X2ZpbGVzKAogICAgICAgICAgICAgICAgICAgICAgICBkYXRhX3BhdGg9cGF0aGxpYi5QYXRoKGlucHV0X2FyZ3VtZW50KS5hYnNvbHV0ZSgpCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgbGVuKGlucHV0X2FyZ3VtZW50KSA8IHNpemU6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJDYW5ub3Qgc3BsaXQgdGhlIGlucHV0ICd7d29ya2VyX2lucHV0fScgb2YgbGVuZ3RoIHtsZW4oaW5wdXRfYXJndW1lbnQpfSB0byB7c2l6ZX0gd29ya2Vycy4gIgogICAgICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSByZWR1Y2UgdGhlIGFtb3VudCBvZiB3b3JrZXJzIGZvciB0aGlzIGlucHV0LiIKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBldmVuX2NodW5rX3NpemUgPSBsZW4oaW5wdXRfYXJndW1lbnQpIC8vIHNpemUKICAgICAgICAgICAgICAgIGNodW5rX3N0YXJ0ID0gcmFuayAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfZW5kID0gKAogICAgICAgICAgICAgICAgICAgIChyYW5rICsgMSkgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgICAgICBpZiByYW5rICsgMSA8IHNpemUKICAgICAgICAgICAgICAgICAgICBlbHNlIGxlbihpbnB1dF9hcmd1bWVudCkKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSYW5rICN7cmFua306IFByb2Nlc3NpbmcgaW5wdXQgY2h1bmsgb2YgJ3t3b3JrZXJfaW5wdXR9JyAiCiAgICAgICAgICAgICAgICAgICAgZiJmcm9tIGluZGV4IHtjaHVua19zdGFydH0gdG8ge2NodW5rX2VuZH0uIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgbGlzdCk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudFtjaHVua19zdGFydDpjaHVua19lbmRdCiAgICAgICAgICAgICAgICBlbGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHBkLkRhdGFGcmFtZSk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudC5pbG9jW2NodW5rX3N0YXJ0OmNodW5rX2VuZDosIDpdCiAgICAgICAgICAgICAgICBrd2FyZ3Nbd29ya2VyX2lucHV0XSA9IGlucHV0X2FyZ3VtZW50CgogICAgICAgICAgICAjIFNldCB0aGUgcm9vdCB3b3JrZXIgb25seSBhcmd1bWVudHM6CiAgICAgICAgICAgIGlmIHJhbmsgPT0gMCBhbmQgcm9vdF93b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAga3dhcmdzLnVwZGF0ZShyb290X3dvcmtlcl9pbnB1dHMpCgogICAgICAgICAgICAjIFJ1biB0aGUgd29ya2VyOgogICAgICAgICAgICBvdXRwdXQgPSBoYW5kbGVyKCoqa3dhcmdzKQoKICAgICAgICAgICAgIyBTZW5kIHRoZSBvdXRwdXQgdG8gdGhlIHJvb3QgcmFuayAocmFuayAjMCk6CiAgICAgICAgICAgIG91dHB1dCA9IGNvbW0uZ2F0aGVyKG91dHB1dCwgcm9vdD0wKQogICAgICAgICAgICBpZiByYW5rID09IDA6CiAgICAgICAgICAgICAgICAjIEpvaW4gdGhlIG91dHB1dHM6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJDb2xsZWN0aW5nIGRhdGEgZnJvbSB3b3JrZXJzIHRvIHJvb3Qgd29ya2VyLiIpCiAgICAgICAgICAgICAgICBvdXRwdXRfZGlyZWN0b3J5ID0gb3V0cHV0WzBdWzBdCiAgICAgICAgICAgICAgICBkYXRhZnJhbWUgPSBwZC5jb25jYXQob2Jqcz1bZGYgZm9yIF8sIGRmLCBfIGluIG91dHB1dF0sIGF4aXM9MCkKICAgICAgICAgICAgICAgIGVycm9yc19kaWN0aW9uYXJ5ID0gcmVkdWNlKAogICAgICAgICAgICAgICAgICAgIG9wZXJhdG9yLmlvciwgW2VyciBmb3IgXywgXywgZXJyIGluIG91dHB1dF0sIHt9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByZXR1cm4gb3V0cHV0X2RpcmVjdG9yeSwgZGF0YWZyYW1lLCBlcnJvcnNfZGljdGlvbmFyeQogICAgICAgICAgICByZXR1cm4gTm9uZQoKICAgICAgICByZXR1cm4gd3JhcHBlcgoKICAgIHJldHVybiBkZWNvcmF0b3IKCgpAb3Blbl9tcGlfaGFuZGxlcih3b3JrZXJfaW5wdXRzPVsiZGF0YV9wYXRoIl0sIHJvb3Rfd29ya2VyX2lucHV0cz17InZlcmJvc2UiOiBUcnVlfSkKZGVmIHRyYW5zbGF0ZSgKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl0sIHBhdGhsaWIuUGF0aF0sCiAgICBvdXRwdXRfZGlyZWN0b3J5OiBzdHIsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgc291cmNlX2xhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgdGFyZ2V0X2xhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgZGV2aWNlOiBzdHIgPSBOb25lLAogICAgbW9kZWxfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIGJhdGNoX3NpemU6IGludCA9IDEsCiAgICB0cmFuc2xhdGlvbl9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopIC0+IFR1cGxlW3N0ciwgcGQuRGF0YUZyYW1lLCBkaWN0XToKICAgICIiIgogICAgVHJhbnNsYXRlIHRleHQgZmlsZXMgdXNpbmcgYSB0cmFuc2Zvcm1lciBtb2RlbCBmcm9tIEh1Z2dpbmdmYWNlJ3MgaHViIGFjY29yZGluZyB0byB0aGUgc291cmNlIGFuZCB0YXJnZXQgbGFuZ3VhZ2VzCiAgICBnaXZlbiAob3IgdXNpbmcgdGhlIGRpcmVjdGx5IHByb3ZpZGVkIG1vZGVsIG5hbWUpLiBUaGUgZW5kIHJlc3VsdCBpcyBhIGRpcmVjdG9yeSBvZiB0cmFuc2xhdGVkIHRleHQgZmlsZXMgYW5kIGEKICAgIGRhdGFmcmFtZSBjb250YWluaW5nIHRoZSBmb2xsb3dpbmcgY29sdW1uczoKCiAgICAqIHRleHRfZmlsZSAtIFRoZSB0ZXh0IGZpbGUgcGF0aC4KICAgICogdHJhbnNsYXRpb25fZmlsZSAtIFRoZSB0cmFuc2xhdGlvbiB0ZXh0IGZpbGUgbmFtZSBpbiB0aGUgb3V0cHV0IGRpcmVjdG9yeS4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICBBIGRpcmVjdG9yeSBvZiB0ZXh0IGZpbGVzIG9yIGEgc2luZ2xlIGZpbGUgb3IgYSBsaXN0IG9mIGZpbGVzIHRvIHRyYW5zbGF0ZS4KICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiAgIERpcmVjdG9yeSB3aGVyZSB0aGUgdHJhbnNsYXRlZCBmaWxlcyB3aWxsIGJlIHNhdmVkLgogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgVGhlIG5hbWUgb2YgYSBtb2RlbCB0byBsb2FkLiBJZiBOb25lLCB0aGUgbW9kZWwgbmFtZSBpcyBjb25zdHJ1Y3RlZCB1c2luZyB0aGUgc291cmNlIGFuZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGFyZ2V0IGxhbmd1YWdlcyBwYXJhbWV0ZXJzLgogICAgOnBhcmFtIHNvdXJjZV9sYW5ndWFnZTogICAgVGhlIHNvdXJjZSBsYW5ndWFnZSBjb2RlIChlLmcuLCAnZW4nIGZvciBFbmdsaXNoKS4KICAgIDpwYXJhbSB0YXJnZXRfbGFuZ3VhZ2U6ICAgIFRoZSB0YXJnZXQgbGFuZ3VhZ2UgY29kZSAoZS5nLiwgJ2VuJyBmb3IgRW5nbGlzaCkuCiAgICA6cGFyYW0gbW9kZWxfa3dhcmdzOiAgICAgICBLZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHJlZ2FyZGluZyB0aGUgbG9hZGluZyBvZiB0aGUgbW9kZWwgaW4gSHVnZ2luZ0ZhY2UncyBgcGlwZWxpbmVgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmdW5jdGlvbi4KICAgIDpwYXJhbSBkZXZpY2U6ICAgICAgICAgICAgIFRoZSBkZXZpY2UgaW5kZXggZm9yIHRyYW5zZm9ybWVycy4gRGVmYXVsdCB3aWxsIHByZWZlciBjdWRhIGlmIGF2YWlsYWJsZS4KICAgIDpwYXJhbSBiYXRjaF9zaXplOiAgICAgICAgIFRoZSBudW1iZXIgb2YgYmF0Y2hlcyB0byB1c2UgaW4gdHJhbnNsYXRpb24uIFRoZSBmaWxlcyBhcmUgdHJhbnNsYXRlZCBvbmUgYnkgb25lLCBidXQgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZW50ZW5jZXMgY2FuIGJlIGJhdGNoZWQuCiAgICA6cGFyYW0gdHJhbnNsYXRpb25fa3dhcmdzOiBBZGRpdGlvbmFsIGtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgdG8gYSBgdHJhbnNmb3JtZXJzLlRyYW5zbGF0aW9uUGlwZWxpbmVgIHdoZW4gZG9pbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZSB0cmFuc2xhdGlvbiBpbmZlcmVuY2UuIE5vdGljZSB0aGUgYmF0Y2ggc2l6ZSBoZXJlIGlzIGJlaW5nIGFkZGVkIGF1dG9tYXRpY2FsbHkuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICBXaGV0aGVyIHRvIHByZXNlbnQgbG9ncyBvZiBhIHByb2dyZXNzIGJhciBhbmQgZXJyb3JzLiBEZWZhdWx0OiBUcnVlLgoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mOgoKICAgICAgICAgICAgICAqIFBhdGggdG8gdGhlIG91dHB1dCBkaXJlY3RvcnkuCiAgICAgICAgICAgICAgKiBBIGRhdGFmcmFtZSBkYXRhc2V0IG9mIHRoZSB0cmFuc2xhdGVkIGZpbGUgbmFtZXMuCiAgICAgICAgICAgICAgKiBBIGRpY3Rpb25hcnkgb2YgZXJyb3JlZCBmaWxlcyB0aGF0IHdlcmUgbm90IHRyYW5zbGF0ZWQuCiAgICAiIiIKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBHZXQgdGhlIGlucHV0IHRleHQgZmlsZXMgdG8gdHJhbnNsYXRlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgdGV4dCBmaWxlcy4iKQogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgICAgIHRleHRfZmlsZXMgPSBfZ2V0X3RleHRfZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGVsc2U6CiAgICAgICAgdGV4dF9maWxlcyA9IGRhdGFfcGF0aAogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbih0ZXh0X2ZpbGVzKX0gdGV4dCBmaWxlcy4iKQoKICAgICMgR2V0IHRoZSB0cmFuc2xhdGlvbiBwaXBlbGluZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTG9hZGluZyBtb2RlbCAtIHVzaW5nIGRldmljZSAne2RldmljZX0nLiIpCiAgICB0cmFuc2xhdGlvbl9waXBlbGluZSwgbW9kZWxfbmFtZSA9IF9nZXRfdHJhbnNsYXRpb25fcGlwZWxpbmUoCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIHNvdXJjZV9sYW5ndWFnZT1zb3VyY2VfbGFuZ3VhZ2UsCiAgICAgICAgdGFyZ2V0X2xhbmd1YWdlPXRhcmdldF9sYW5ndWFnZSwKICAgICAgICBkZXZpY2U9ZGV2aWNlLAogICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3MsCiAgICAgICAgYmF0Y2hfc2l6ZT1iYXRjaF9zaXplIGlmIGJhdGNoX3NpemUgIT0gMSBlbHNlIE5vbmUsCiAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIk1vZGVsICd7bW9kZWxfbmFtZX0nIHdhcyBsb2FkZWQgc3VjY2Vzc2Z1bGx5LiIpCgogICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgIHN1Y2Nlc3NlcyA9IFtdCiAgICBlcnJvcnMgPSB7fQoKICAgICMgQ3JlYXRlIHRoZSBvdXRwdXQgZGlyZWN0b3J5OgogICAgb3V0cHV0X2RpcmVjdG9yeSA9IHBhdGhsaWIuUGF0aChvdXRwdXRfZGlyZWN0b3J5KQogICAgb3V0cHV0X2RpcmVjdG9yeS5ta2RpcihwYXJlbnRzPVRydWUsIGV4aXN0X29rPVRydWUpCgogICAgIyBQcmVwYXJlIHRoZSB0cmFuc2xhdGlvbiBrZXl3b3JkIGFyZ3VtZW50czoKICAgIHRyYW5zbGF0aW9uX2t3YXJncyA9IHRyYW5zbGF0aW9uX2t3YXJncyBvciB7fQoKICAgICMgR28gb3ZlciB0aGUgYXVkaW8gZmlsZXMgYW5kIHRyYW5zY3JpYmU6CiAgICBmb3IgdGV4dF9maWxlIGluIHRxZG0oCiAgICAgICAgdGV4dF9maWxlcywgZGVzYz0iVHJhbnNsYXRpbmciLCB1bml0PSJmaWxlIiwgZGlzYWJsZT1ub3QgdmVyYm9zZQogICAgKToKICAgICAgICB0cnk6CiAgICAgICAgICAgICMgVHJhbnNsYXRlOgogICAgICAgICAgICB0cmFuc2xhdGlvbiA9IF90cmFuc2xhdGUoCiAgICAgICAgICAgICAgICB0ZXh0X2ZpbGU9dGV4dF9maWxlLAogICAgICAgICAgICAgICAgdHJhbnNsYXRpb25fcGlwZWxpbmU9dHJhbnNsYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgICAgICB0cmFuc2xhdGlvbl9rd2FyZ3M9dHJhbnNsYXRpb25fa3dhcmdzLAogICAgICAgICAgICApCiAgICAgICAgICAgICMgV3JpdGUgdGhlIHRyYW5zY3JpcHRpb24gdG8gZmlsZToKICAgICAgICAgICAgdHJhbnNsYXRpb25fZmlsZSA9IF9zYXZlX3RvX2ZpbGUoCiAgICAgICAgICAgICAgICB0cmFuc2xhdGlvbj10cmFuc2xhdGlvbiwKICAgICAgICAgICAgICAgIGZpbGVfbmFtZT10ZXh0X2ZpbGUuc3RlbSwKICAgICAgICAgICAgICAgIG91dHB1dF9kaXJlY3Rvcnk9b3V0cHV0X2RpcmVjdG9yeSwKICAgICAgICAgICAgKQogICAgICAgICAgICAjIE5vdGUgYXMgYSBzdWNjZXNzIGluIHRoZSBsaXN0OgogICAgICAgICAgICBzdWNjZXNzZXMuYXBwZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRleHRfZmlsZS5uYW1lLAogICAgICAgICAgICAgICAgICAgIHRyYW5zbGF0aW9uX2ZpbGUubmFtZSwKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIE5vdGUgdGhlIGV4Y2VwdGlvbiBhcyBlcnJvciBpbiB0aGUgZGljdGlvbmFyeToKICAgICAgICAgICAgaWYgdmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIkVycm9yIGluIGZpbGU6ICd7dGV4dF9maWxlLm5hbWV9JyIpCiAgICAgICAgICAgIGVycm9yc1tzdHIodGV4dF9maWxlLm5hbWUpXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBDb25zdHJ1Y3QgdGhlIHRyYW5zbGF0aW9ucyBkYXRhZnJhbWU6CiAgICBjb2x1bW5zID0gWwogICAgICAgICJ0ZXh0X2ZpbGUiLAogICAgICAgICJ0cmFuc2xhdGlvbl9maWxlIiwKICAgIF0KICAgIHN1Y2Nlc3NlcyA9IHBkLkRhdGFGcmFtZSgKICAgICAgICBzdWNjZXNzZXMsCiAgICAgICAgY29sdW1ucz1jb2x1bW5zLAogICAgKQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oCiAgICAgICAgICAgIGYiRG9uZSAoe3N1Y2Nlc3Nlcy5zaGFwZVswXX0ve2xlbih0ZXh0X2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiVHJhbnNsYXRpb25zIHN1bW1hcnk6XG4iCiAgICAgICAgICAgIGYie3N1Y2Nlc3Nlcy5oZWFkKCl9IgogICAgICAgICkKICAgIHJldHVybiBzdHIob3V0cHV0X2RpcmVjdG9yeSksIHN1Y2Nlc3NlcywgZXJyb3JzCgoKZGVmIF9nZXRfdGV4dF9maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IExpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICB0ZXh0X2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgdGV4dF9maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgZmlsZSBwYXRoLiAiCiAgICAgICAgICAgIGYiR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gdGV4dF9maWxlcwoKCmRlZiBfZ2V0X3RyYW5zbGF0aW9uX3BpcGVsaW5lKAogICAgbW9kZWxfbmFtZTogc3RyID0gTm9uZSwKICAgIHNvdXJjZV9sYW5ndWFnZTogc3RyID0gTm9uZSwKICAgIHRhcmdldF9sYW5ndWFnZTogc3RyID0gTm9uZSwKICAgIGRldmljZTogc3RyID0gTm9uZSwKICAgIG1vZGVsX2t3YXJnczogZGljdCA9IE5vbmUsCiAgICBiYXRjaF9zaXplOiBpbnQgPSBOb25lLAopIC0+IFR1cGxlW3RyYW5zZm9ybWVycy5QaXBlbGluZSwgc3RyXToKICAgICMgQ29uc3RydWN0IHRoZSBtb2RlbCBuYW1lIC0gaWYgbW9kZWwgbmFtZSBpcyBwcm92aWRlZCAobm90IE5vbmUpIHRoZW4gd2UgdGFrZSBpdCwgb3RoZXJ3aXNlIHdlIGNoZWNrIGJvdGggc291cmNlCiAgICAjIGFuZCB0YXJnZXQgd2VyZSBwcm92aWRlZCB0byBjb25zdHJ1Y3QgdGhlIG1vZGVsIG5hbWU6CiAgICBpZiBtb2RlbF9uYW1lIGlzIE5vbmUgYW5kIChzb3VyY2VfbGFuZ3VhZ2UgaXMgTm9uZSBvciB0YXJnZXRfbGFuZ3VhZ2UgaXMgTm9uZSk6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgIk5vIG1vZGVsIG5hbWUgd2VyZSBnaXZlbiBhbmQgbWlzc2luZyBzb3VyY2UgYW5kIC8gb3IgdGFyZ2V0IGxhbmd1YWdlcy4gSW4gb3JkZXIgdG8gdHJhbnNsYXRlIHlvdSBtdXN0ICIKICAgICAgICAgICAgInBhc3MgYSBgbW9kZWxfbmFtZWAgb3IgYm90aCBgc291cmNlX2xhbmd1YWdlYCBhbmQgYHRhcmdldF9sYW5ndWFnZWAuIgogICAgICAgICkKICAgIGVsaWYgbW9kZWxfbmFtZSBpcyBOb25lOgogICAgICAgIG1vZGVsX25hbWUgPSBmIkhlbHNpbmtpLU5MUC9vcHVzLW10LXtzb3VyY2VfbGFuZ3VhZ2V9LXt0YXJnZXRfbGFuZ3VhZ2V9IgoKICAgICMgSW5pdGlhbGl6ZSB0aGUgdHJhbnNsYXRpb24gcGlwZWxpbmU6CiAgICB0cnk6CiAgICAgICAgdHJhbnNsYXRpb25fcGlwZWxpbmUgPSB0cmFuc2Zvcm1lcnMucGlwZWxpbmUoCiAgICAgICAgICAgIHRhc2s9InRyYW5zbGF0aW9uIiwKICAgICAgICAgICAgbW9kZWw9bW9kZWxfbmFtZSwKICAgICAgICAgICAgdG9rZW5pemVyPW1vZGVsX25hbWUsCiAgICAgICAgICAgIGRldmljZT1kZXZpY2UsCiAgICAgICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3MsCiAgICAgICAgICAgIGJhdGNoX3NpemU9YmF0Y2hfc2l6ZSwKICAgICAgICApCiAgICBleGNlcHQgT1NFcnJvciBhcyBsb2FkX2V4Y2VwdGlvbjoKICAgICAgICBpZiAoCiAgICAgICAgICAgICJpcyBub3QgYSB2YWxpZCBtb2RlbCBpZGVudGlmaWVyIGxpc3RlZCBvbiAnaHR0cHM6Ly9odWdnaW5nZmFjZS5jby9tb2RlbHMnIgogICAgICAgICAgICBpbiBzdHIobG9hZF9leGNlcHRpb24pCiAgICAgICAgICAgIGFuZCBzb3VyY2VfbGFuZ3VhZ2UKICAgICAgICApOgogICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgZiJUaGUgbW9kZWwgJ3ttb2RlbF9uYW1lfScgaXMgbm90IGEgdmFsaWQgbW9kZWwgaWRlbnRpZmllci4gIgogICAgICAgICAgICAgICAgZiJUaGUgcGFyYW1ldGVycyBgc291cmNlX2xhbmd1YWdlYCBhbmQgYHRhcmdldF9sYW5ndWFnZWAgYXJlIHVzZWQgdG8gY29uc3RydWN0IGEgSGVsc2lua2kgbW9kZWwgZm9yICIKICAgICAgICAgICAgICAgIGYidGV4dCB0byB0ZXh0IGdlbmVyYXRpb24sIGJ1dCB0aGUgbW9kZWwgY3JlYXRlZCBmcm9tIHRoZSBnaXZlbiBsYW5ndWFnZXMgZG9lcyBub3QgZXhpc3QuICIKICAgICAgICAgICAgICAgIGYiWW91IG1heSBjaGVjayBsYW5ndWFnZSBpZGVudGlmaWVycyBhdCAiCiAgICAgICAgICAgICAgICBmImh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL2FkbWluLXNkay9kaXJlY3RvcnkvdjEvbGFuZ3VhZ2VzLCBhbmQgaWYgdGhlIGVycm9yIHdhcyBub3QgZml4ZWQsIG9uZSAiCiAgICAgICAgICAgICAgICBmIm9yIG1vcmUgbGFuZ3VhZ2UgY29kZSBtaWdodCBiZSB3aXRoIDMgbGV0dGVycyBhbmQgbmVlZHMgdG8gYmUgZm91bmQgb25saW5lLiAiCiAgICAgICAgICAgICAgICBmIlJlbWVtYmVyLCB5b3UgY2FuIGFsd2F5cyBjaG9vc2UgYSBtb2RlbCBkaXJlY3RseSBmcm9tIHRoZSBIdWdnaW5nZmFjZSBodWIgYnkgdXNpbmcgdGhlIGBtb2RlbF9uYW1lYCAiCiAgICAgICAgICAgICAgICBmInBhcmFtZXRlci4iCiAgICAgICAgICAgICkgZnJvbSBsb2FkX2V4Y2VwdGlvbgogICAgICAgIHJhaXNlIGxvYWRfZXhjZXB0aW9uCgogICAgcmV0dXJuIHRyYW5zbGF0aW9uX3BpcGVsaW5lLCBtb2RlbF9uYW1lCgoKZGVmIF90cmFuc2xhdGUoCiAgICB0ZXh0X2ZpbGU6IHBhdGhsaWIuUGF0aCwKICAgIHRyYW5zbGF0aW9uX3BpcGVsaW5lOiB0cmFuc2Zvcm1lcnMuUGlwZWxpbmUsCiAgICB0cmFuc2xhdGlvbl9rd2FyZ3M6IGRpY3QsCikgLT4gc3RyOgogICAgIyBSZWFkIHRoZSB0ZXh0IGZyb20gZmlsZToKICAgIHdpdGggb3Blbih0ZXh0X2ZpbGUsICJyIikgYXMgZnA6CiAgICAgICAgdGV4dCA9IGZwLnJlYWQoKQoKICAgICMgU3BsaXQgdG8gcGFyYWdyYXBocyBhbmQgZWFjaCBwYXJhZ3JhcGggdG8gc2VudGVuY2VzOgogICAgcGFyYWdyYXBocyA9IFtwYXJhZ3JhcGguc3BsaXQoIi4iKSBmb3IgcGFyYWdyYXBoIGluIHRleHQuc3BsaXQoIlxuIildCgogICAgIyBEaXNjb3ZlciB0aGUgbmV3bGluZSBpbmRleGVzIHRvIHJlc3RvcmUgdGhlIGZpbGUgdG8gaXRzIHN0cnVjdHVyZSBwb3N0IHRyYW5zbGF0aW9uOgogICAgbmV3bGluZXNfaW5kZXhlcyA9IFtdCiAgICBmb3IgcGFyYWdyYXBoIGluIHBhcmFncmFwaHNbOi0xXToKICAgICAgICBpZiBsZW4obmV3bGluZXNfaW5kZXhlcykgPT0gMDoKICAgICAgICAgICAgbmV3bGluZXNfaW5kZXhlcy5hcHBlbmQobGVuKHBhcmFncmFwaCkgLSAxKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIG5ld2xpbmVzX2luZGV4ZXMuYXBwZW5kKG5ld2xpbmVzX2luZGV4ZXNbLTFdICsgbGVuKHBhcmFncmFwaCkpCgogICAgIyBQcmVwYXJlIHRoZSBiYXRjaGVzIChlYWNoIHNlbnRlbmNlIGZyb20gdGhlIHBhcmFncmFwaHMpLiBOb3RpY2Ugd2UgYWRkIGEgZG90IG5vdCBvbmx5IHRvIHJlc3RvcmUgdGhlIHNlbnRlbmNlCiAgICAjIHN0cnVjdHVyZSBidXQgdG8gaWdub3JlIGVtcHR5IHN0cmluZ3MgYXMgaXQgd2lsbCBydWluIHRoZSB0cmFuc2xhdGlvbjoKICAgIHNlbnRlbmNlcyA9IFtmIntsaW5lfS4iIGZvciBwYXJhZ3JhcGggaW4gcGFyYWdyYXBocyBmb3IgbGluZSBpbiBwYXJhZ3JhcGhdCgogICAgIyBUcmFuc2xhdGUgdGhlIHNlbnRlbmNlczoKICAgIHRyYW5zbGF0aW9ucyA9IHRyYW5zbGF0aW9uX3BpcGVsaW5lKHNlbnRlbmNlcywgKip0cmFuc2xhdGlvbl9rd2FyZ3MpCgogICAgIyBSZXN0cnVjdHVyZSB0aGUgZnVsbCB0ZXh0IGZyb20gdGhlIHNlbnRlbmNlczoKICAgIHRyYW5zbGF0ZWRfdGV4dCA9IFtdCiAgICBuZXdsaW5lX2luZGV4ID0gbmV3bGluZXNfaW5kZXhlcy5wb3AoMCkgaWYgbmV3bGluZXNfaW5kZXhlcyBlbHNlIE5vbmUKICAgIGZvciBpLCB0cmFuc2xhdGlvbiBpbiBlbnVtZXJhdGUodHJhbnNsYXRpb25zKToKICAgICAgICAjIEdldCB0aGUgdHJhbnNsYXRpb246CiAgICAgICAgdGV4dCA9IHRyYW5zbGF0aW9uWyJ0cmFuc2xhdGlvbl90ZXh0Il0KICAgICAgICAjIFZhbGlkYXRlIGlmIGl0IHdhcyBhbiBlbXB0eSBzZW50ZW5jZSBiZWZvcmU6CiAgICAgICAgaWYgdGV4dCA9PSAiLiI6CiAgICAgICAgICAgIHRleHQgPSAiIgogICAgICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIGluc2VydCBhIG5ld2xpbmU6CiAgICAgICAgaWYgbmV3bGluZV9pbmRleCBhbmQgbmV3bGluZV9pbmRleCA9PSBpOgogICAgICAgICAgICB0ZXh0ICs9ICJcbiIKICAgICAgICAgICAgbmV3bGluZV9pbmRleCA9IG5ld2xpbmVzX2luZGV4ZXMucG9wKDApIGlmIG5ld2xpbmVzX2luZGV4ZXMgZWxzZSBOb25lCiAgICAgICAgIyBDb2xsZWN0IGl0OgogICAgICAgIHRyYW5zbGF0ZWRfdGV4dC5hcHBlbmQodGV4dCkKICAgIHRyYW5zbGF0ZWRfdGV4dCA9ICIiLmpvaW4odHJhbnNsYXRlZF90ZXh0KQoKICAgIHJldHVybiB0cmFuc2xhdGVkX3RleHQKCgpkZWYgX3NhdmVfdG9fZmlsZSgKICAgIHRyYW5zbGF0aW9uOiBzdHIsIGZpbGVfbmFtZTogc3RyLCBvdXRwdXRfZGlyZWN0b3J5OiBwYXRobGliLlBhdGgKKSAtPiBwYXRobGliLlBhdGg6CiAgICAjIFByZXBhcmUgdGhlIGZpbGUgZnVsbCBwYXRoIChjaGVja2luZyBmb3Igbm8gZHVwbGljYXRpb25zKToKICAgIHRyYW5zbGF0aW9uX2ZpbGUgPSBvdXRwdXRfZGlyZWN0b3J5IC8gZiJ7ZmlsZV9uYW1lfS50eHQiCiAgICBpID0gMQogICAgd2hpbGUgdHJhbnNsYXRpb25fZmlsZS5leGlzdHMoKToKICAgICAgICBpICs9IDEKICAgICAgICB0cmFuc2xhdGlvbl9maWxlID0gb3V0cHV0X2RpcmVjdG9yeSAvIGYie2ZpbGVfbmFtZX1fe2l9LnR4dCIKCiAgICAjIE1ha2Ugc3VyZSBhbGwgZGlyZWN0b3JpZXMgYXJlIGNyZWF0ZWQ6CiAgICB0cmFuc2xhdGlvbl9maWxlLnBhcmVudC5ta2RpcihleGlzdF9vaz1UcnVlLCBwYXJlbnRzPVRydWUpCgogICAgIyBXcml0ZSB0byBmaWxlOgogICAgd2l0aCBvcGVuKHRyYW5zbGF0aW9uX2ZpbGUsICJ3IikgYXMgZnA6CiAgICAgICAgZnAud3JpdGUodHJhbnNsYXRpb24pCgogICAgcmV0dXJuIHRyYW5zbGF0aW9uX2ZpbGUK + origin_filename: '' + base_image: mlrun/mlrun requirements: - transformers - sentencepiece - - torch + - torch>=2.6 - tqdm code_origin: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9wZXJhdG9yCmltcG9ydCBwYXRobGliCmZyb20gZnVuY3Rvb2xzIGltcG9ydCByZWR1Y2UsIHdyYXBzCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIFR1cGxlLCBVbmlvbgoKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgdHJhbnNmb3JtZXJzCmZyb20gdHFkbSBpbXBvcnQgdHFkbQoKIyBHZXQgdGhlIGdsb2JhbCBsb2dnZXI6Cl9MT0dHRVIgPSBsb2dnaW5nLmdldExvZ2dlcigpCgoKZGVmIF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKSAtPiBUdXBsZVsibWxydW4uTUxDbGllbnRDdHgiLCAibXBpNHB5Lk1QSS5JbnRyYWNvbW0iXToKICAgIGlzX21waSA9IEZhbHNlCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgaXNfbXBpID0gY29udGV4dC5sYWJlbHMuZ2V0KCJraW5kIiwgImpvYiIpID09ICJtcGlqb2IiCgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnJvbSBtcGk0cHkgaW1wb3J0IE1QSQoKICAgICAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBNUEkuQ09NTV9XT1JMRAogICAgICAgICAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtcGk0cHlfbm90X2ZvdW5kOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIlRvIGRpc3RyaWJ1dGUgdGhlIGZ1bmN0aW9uIHVzaW5nIE1MUnVuJ3MgJ21waWpvYicgeW91IG5lZWQgdG8gaGF2ZSBgbXBpNHB5YCBwYWNrYWdlIGluIHlvdXIgIgogICAgICAgICAgICAgICAgICAgICJpbnRlcnByZXRlci4gUGxlYXNlIHJ1biBgcGlwIGluc3RhbGwgbXBpNHB5YCBhbmQgbWFrZSBzdXJlIHlvdSBoYXZlIG9wZW4tbXBpLiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIG1waTRweV9ub3RfZm91bmQKICAgICAgICBlbHNlOgogICAgICAgICAgICByZXR1cm4gY29udGV4dCwgTm9uZQogICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3IgYXMgbW9kdWxlX25vdF9mb3VuZDoKICAgICAgICBpZiBpc19tcGk6CiAgICAgICAgICAgIHJhaXNlIG1vZHVsZV9ub3RfZm91bmQKICAgIHJldHVybiBOb25lLCBOb25lCgoKZGVmIG9wZW5fbXBpX2hhbmRsZXIoCiAgICB3b3JrZXJfaW5wdXRzOiBMaXN0W3N0cl0sIHJvb3Rfd29ya2VyX2lucHV0czogRGljdFtzdHIsIEFueV0gPSBOb25lCik6CiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgQ2hlY2sgZm9yIE1MUnVuIGFuZCBPcGVuTVBJIGF2YWlsYWJpbGl0eToKICAgIGNvbnRleHQsIGNvbW0gPSBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkKCiAgICAjIENoZWNrIGlmIE1MUnVuIGlzIGF2YWlsYWJsZSwgc2V0IHRoZSBnbG9iYWwgbG9nZ2VyIHRvIE1MUnVuJ3M6CiAgICBpZiBjb250ZXh0OgogICAgICAgIF9MT0dHRVIgPSBjb250ZXh0LmxvZ2dlcgoKICAgIGRlZiBkZWNvcmF0b3IoaGFuZGxlcik6CiAgICAgICAgaWYgY29tbSBpcyBOb25lIG9yIGNvbW0uR2V0X3NpemUoKSA9PSAxOgogICAgICAgICAgICByZXR1cm4gaGFuZGxlcgoKICAgICAgICBAd3JhcHMoaGFuZGxlcikKICAgICAgICBkZWYgd3JhcHBlcigqKmt3YXJncyk6CiAgICAgICAgICAgICMgR2V0IHRoZSBvcGVuIG1waSBlbnZpcm9ubWVudCBwcm9wZXJ0aWVzOgogICAgICAgICAgICBzaXplID0gY29tbS5HZXRfc2l6ZSgpCiAgICAgICAgICAgIHJhbmsgPSBjb21tLkdldF9yYW5rKCkKCiAgICAgICAgICAgICMgR2l2ZSB0aGUgY29ycmVjdCBjaHVuayBvZiB0aGUgd29ya2VycyBpbnB1dHM6CiAgICAgICAgICAgIGZvciB3b3JrZXJfaW5wdXQgaW4gd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0ga3dhcmdzW3dvcmtlcl9pbnB1dF0KICAgICAgICAgICAgICAgIGlmIGlucHV0X2FyZ3VtZW50IGlzIE5vbmU6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIChzdHIsIHBhdGhsaWIuUGF0aCkpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gX2dldF90ZXh0X2ZpbGVzKAogICAgICAgICAgICAgICAgICAgICAgICBkYXRhX3BhdGg9cGF0aGxpYi5QYXRoKGlucHV0X2FyZ3VtZW50KS5hYnNvbHV0ZSgpCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgbGVuKGlucHV0X2FyZ3VtZW50KSA8IHNpemU6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJDYW5ub3Qgc3BsaXQgdGhlIGlucHV0ICd7d29ya2VyX2lucHV0fScgb2YgbGVuZ3RoIHtsZW4oaW5wdXRfYXJndW1lbnQpfSB0byB7c2l6ZX0gd29ya2Vycy4gIgogICAgICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSByZWR1Y2UgdGhlIGFtb3VudCBvZiB3b3JrZXJzIGZvciB0aGlzIGlucHV0LiIKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBldmVuX2NodW5rX3NpemUgPSBsZW4oaW5wdXRfYXJndW1lbnQpIC8vIHNpemUKICAgICAgICAgICAgICAgIGNodW5rX3N0YXJ0ID0gcmFuayAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfZW5kID0gKAogICAgICAgICAgICAgICAgICAgIChyYW5rICsgMSkgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgICAgICBpZiByYW5rICsgMSA8IHNpemUKICAgICAgICAgICAgICAgICAgICBlbHNlIGxlbihpbnB1dF9hcmd1bWVudCkKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSYW5rICN7cmFua306IFByb2Nlc3NpbmcgaW5wdXQgY2h1bmsgb2YgJ3t3b3JrZXJfaW5wdXR9JyAiCiAgICAgICAgICAgICAgICAgICAgZiJmcm9tIGluZGV4IHtjaHVua19zdGFydH0gdG8ge2NodW5rX2VuZH0uIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgbGlzdCk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudFtjaHVua19zdGFydDpjaHVua19lbmRdCiAgICAgICAgICAgICAgICBlbGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHBkLkRhdGFGcmFtZSk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudC5pbG9jW2NodW5rX3N0YXJ0OmNodW5rX2VuZDosIDpdCiAgICAgICAgICAgICAgICBrd2FyZ3Nbd29ya2VyX2lucHV0XSA9IGlucHV0X2FyZ3VtZW50CgogICAgICAgICAgICAjIFNldCB0aGUgcm9vdCB3b3JrZXIgb25seSBhcmd1bWVudHM6CiAgICAgICAgICAgIGlmIHJhbmsgPT0gMCBhbmQgcm9vdF93b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAga3dhcmdzLnVwZGF0ZShyb290X3dvcmtlcl9pbnB1dHMpCgogICAgICAgICAgICAjIFJ1biB0aGUgd29ya2VyOgogICAgICAgICAgICBvdXRwdXQgPSBoYW5kbGVyKCoqa3dhcmdzKQoKICAgICAgICAgICAgIyBTZW5kIHRoZSBvdXRwdXQgdG8gdGhlIHJvb3QgcmFuayAocmFuayAjMCk6CiAgICAgICAgICAgIG91dHB1dCA9IGNvbW0uZ2F0aGVyKG91dHB1dCwgcm9vdD0wKQogICAgICAgICAgICBpZiByYW5rID09IDA6CiAgICAgICAgICAgICAgICAjIEpvaW4gdGhlIG91dHB1dHM6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJDb2xsZWN0aW5nIGRhdGEgZnJvbSB3b3JrZXJzIHRvIHJvb3Qgd29ya2VyLiIpCiAgICAgICAgICAgICAgICBvdXRwdXRfZGlyZWN0b3J5ID0gb3V0cHV0WzBdWzBdCiAgICAgICAgICAgICAgICBkYXRhZnJhbWUgPSBwZC5jb25jYXQob2Jqcz1bZGYgZm9yIF8sIGRmLCBfIGluIG91dHB1dF0sIGF4aXM9MCkKICAgICAgICAgICAgICAgIGVycm9yc19kaWN0aW9uYXJ5ID0gcmVkdWNlKAogICAgICAgICAgICAgICAgICAgIG9wZXJhdG9yLmlvciwgW2VyciBmb3IgXywgXywgZXJyIGluIG91dHB1dF0sIHt9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByZXR1cm4gb3V0cHV0X2RpcmVjdG9yeSwgZGF0YWZyYW1lLCBlcnJvcnNfZGljdGlvbmFyeQogICAgICAgICAgICByZXR1cm4gTm9uZQoKICAgICAgICByZXR1cm4gd3JhcHBlcgoKICAgIHJldHVybiBkZWNvcmF0b3IKCgpAb3Blbl9tcGlfaGFuZGxlcih3b3JrZXJfaW5wdXRzPVsiZGF0YV9wYXRoIl0sIHJvb3Rfd29ya2VyX2lucHV0cz17InZlcmJvc2UiOiBUcnVlfSkKZGVmIHRyYW5zbGF0ZSgKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl0sIHBhdGhsaWIuUGF0aF0sCiAgICBvdXRwdXRfZGlyZWN0b3J5OiBzdHIsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgc291cmNlX2xhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgdGFyZ2V0X2xhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgZGV2aWNlOiBzdHIgPSBOb25lLAogICAgbW9kZWxfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIGJhdGNoX3NpemU6IGludCA9IDEsCiAgICB0cmFuc2xhdGlvbl9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopIC0+IFR1cGxlW3N0ciwgcGQuRGF0YUZyYW1lLCBkaWN0XToKICAgICIiIgogICAgVHJhbnNsYXRlIHRleHQgZmlsZXMgdXNpbmcgYSB0cmFuc2Zvcm1lciBtb2RlbCBmcm9tIEh1Z2dpbmdmYWNlJ3MgaHViIGFjY29yZGluZyB0byB0aGUgc291cmNlIGFuZCB0YXJnZXQgbGFuZ3VhZ2VzCiAgICBnaXZlbiAob3IgdXNpbmcgdGhlIGRpcmVjdGx5IHByb3ZpZGVkIG1vZGVsIG5hbWUpLiBUaGUgZW5kIHJlc3VsdCBpcyBhIGRpcmVjdG9yeSBvZiB0cmFuc2xhdGVkIHRleHQgZmlsZXMgYW5kIGEKICAgIGRhdGFmcmFtZSBjb250YWluaW5nIHRoZSBmb2xsb3dpbmcgY29sdW1uczoKCiAgICAqIHRleHRfZmlsZSAtIFRoZSB0ZXh0IGZpbGUgcGF0aC4KICAgICogdHJhbnNsYXRpb25fZmlsZSAtIFRoZSB0cmFuc2xhdGlvbiB0ZXh0IGZpbGUgbmFtZSBpbiB0aGUgb3V0cHV0IGRpcmVjdG9yeS4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICBBIGRpcmVjdG9yeSBvZiB0ZXh0IGZpbGVzIG9yIGEgc2luZ2xlIGZpbGUgb3IgYSBsaXN0IG9mIGZpbGVzIHRvIHRyYW5zbGF0ZS4KICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiAgIERpcmVjdG9yeSB3aGVyZSB0aGUgdHJhbnNsYXRlZCBmaWxlcyB3aWxsIGJlIHNhdmVkLgogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgVGhlIG5hbWUgb2YgYSBtb2RlbCB0byBsb2FkLiBJZiBOb25lLCB0aGUgbW9kZWwgbmFtZSBpcyBjb25zdHJ1Y3RlZCB1c2luZyB0aGUgc291cmNlIGFuZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGFyZ2V0IGxhbmd1YWdlcyBwYXJhbWV0ZXJzLgogICAgOnBhcmFtIHNvdXJjZV9sYW5ndWFnZTogICAgVGhlIHNvdXJjZSBsYW5ndWFnZSBjb2RlIChlLmcuLCAnZW4nIGZvciBFbmdsaXNoKS4KICAgIDpwYXJhbSB0YXJnZXRfbGFuZ3VhZ2U6ICAgIFRoZSB0YXJnZXQgbGFuZ3VhZ2UgY29kZSAoZS5nLiwgJ2VuJyBmb3IgRW5nbGlzaCkuCiAgICA6cGFyYW0gbW9kZWxfa3dhcmdzOiAgICAgICBLZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHJlZ2FyZGluZyB0aGUgbG9hZGluZyBvZiB0aGUgbW9kZWwgaW4gSHVnZ2luZ0ZhY2UncyBgcGlwZWxpbmVgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmdW5jdGlvbi4KICAgIDpwYXJhbSBkZXZpY2U6ICAgICAgICAgICAgIFRoZSBkZXZpY2UgaW5kZXggZm9yIHRyYW5zZm9ybWVycy4gRGVmYXVsdCB3aWxsIHByZWZlciBjdWRhIGlmIGF2YWlsYWJsZS4KICAgIDpwYXJhbSBiYXRjaF9zaXplOiAgICAgICAgIFRoZSBudW1iZXIgb2YgYmF0Y2hlcyB0byB1c2UgaW4gdHJhbnNsYXRpb24uIFRoZSBmaWxlcyBhcmUgdHJhbnNsYXRlZCBvbmUgYnkgb25lLCBidXQgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZW50ZW5jZXMgY2FuIGJlIGJhdGNoZWQuCiAgICA6cGFyYW0gdHJhbnNsYXRpb25fa3dhcmdzOiBBZGRpdGlvbmFsIGtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgdG8gYSBgdHJhbnNmb3JtZXJzLlRyYW5zbGF0aW9uUGlwZWxpbmVgIHdoZW4gZG9pbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZSB0cmFuc2xhdGlvbiBpbmZlcmVuY2UuIE5vdGljZSB0aGUgYmF0Y2ggc2l6ZSBoZXJlIGlzIGJlaW5nIGFkZGVkIGF1dG9tYXRpY2FsbHkuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICBXaGV0aGVyIHRvIHByZXNlbnQgbG9ncyBvZiBhIHByb2dyZXNzIGJhciBhbmQgZXJyb3JzLiBEZWZhdWx0OiBUcnVlLgoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mOgoKICAgICAgICAgICAgICAqIFBhdGggdG8gdGhlIG91dHB1dCBkaXJlY3RvcnkuCiAgICAgICAgICAgICAgKiBBIGRhdGFmcmFtZSBkYXRhc2V0IG9mIHRoZSB0cmFuc2xhdGVkIGZpbGUgbmFtZXMuCiAgICAgICAgICAgICAgKiBBIGRpY3Rpb25hcnkgb2YgZXJyb3JlZCBmaWxlcyB0aGF0IHdlcmUgbm90IHRyYW5zbGF0ZWQuCiAgICAiIiIKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBHZXQgdGhlIGlucHV0IHRleHQgZmlsZXMgdG8gdHJhbnNsYXRlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgdGV4dCBmaWxlcy4iKQogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgICAgIHRleHRfZmlsZXMgPSBfZ2V0X3RleHRfZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGVsc2U6CiAgICAgICAgdGV4dF9maWxlcyA9IGRhdGFfcGF0aAogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbih0ZXh0X2ZpbGVzKX0gdGV4dCBmaWxlcy4iKQoKICAgICMgR2V0IHRoZSB0cmFuc2xhdGlvbiBwaXBlbGluZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTG9hZGluZyBtb2RlbCAtIHVzaW5nIGRldmljZSAne2RldmljZX0nLiIpCiAgICB0cmFuc2xhdGlvbl9waXBlbGluZSwgbW9kZWxfbmFtZSA9IF9nZXRfdHJhbnNsYXRpb25fcGlwZWxpbmUoCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIHNvdXJjZV9sYW5ndWFnZT1zb3VyY2VfbGFuZ3VhZ2UsCiAgICAgICAgdGFyZ2V0X2xhbmd1YWdlPXRhcmdldF9sYW5ndWFnZSwKICAgICAgICBkZXZpY2U9ZGV2aWNlLAogICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3MsCiAgICAgICAgYmF0Y2hfc2l6ZT1iYXRjaF9zaXplIGlmIGJhdGNoX3NpemUgIT0gMSBlbHNlIE5vbmUsCiAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIk1vZGVsICd7bW9kZWxfbmFtZX0nIHdhcyBsb2FkZWQgc3VjY2Vzc2Z1bGx5LiIpCgogICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgIHN1Y2Nlc3NlcyA9IFtdCiAgICBlcnJvcnMgPSB7fQoKICAgICMgQ3JlYXRlIHRoZSBvdXRwdXQgZGlyZWN0b3J5OgogICAgb3V0cHV0X2RpcmVjdG9yeSA9IHBhdGhsaWIuUGF0aChvdXRwdXRfZGlyZWN0b3J5KQogICAgb3V0cHV0X2RpcmVjdG9yeS5ta2RpcihwYXJlbnRzPVRydWUsIGV4aXN0X29rPVRydWUpCgogICAgIyBQcmVwYXJlIHRoZSB0cmFuc2xhdGlvbiBrZXl3b3JkIGFyZ3VtZW50czoKICAgIHRyYW5zbGF0aW9uX2t3YXJncyA9IHRyYW5zbGF0aW9uX2t3YXJncyBvciB7fQoKICAgICMgR28gb3ZlciB0aGUgYXVkaW8gZmlsZXMgYW5kIHRyYW5zY3JpYmU6CiAgICBmb3IgdGV4dF9maWxlIGluIHRxZG0oCiAgICAgICAgdGV4dF9maWxlcywgZGVzYz0iVHJhbnNsYXRpbmciLCB1bml0PSJmaWxlIiwgZGlzYWJsZT1ub3QgdmVyYm9zZQogICAgKToKICAgICAgICB0cnk6CiAgICAgICAgICAgICMgVHJhbnNsYXRlOgogICAgICAgICAgICB0cmFuc2xhdGlvbiA9IF90cmFuc2xhdGUoCiAgICAgICAgICAgICAgICB0ZXh0X2ZpbGU9dGV4dF9maWxlLAogICAgICAgICAgICAgICAgdHJhbnNsYXRpb25fcGlwZWxpbmU9dHJhbnNsYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgICAgICB0cmFuc2xhdGlvbl9rd2FyZ3M9dHJhbnNsYXRpb25fa3dhcmdzLAogICAgICAgICAgICApCiAgICAgICAgICAgICMgV3JpdGUgdGhlIHRyYW5zY3JpcHRpb24gdG8gZmlsZToKICAgICAgICAgICAgdHJhbnNsYXRpb25fZmlsZSA9IF9zYXZlX3RvX2ZpbGUoCiAgICAgICAgICAgICAgICB0cmFuc2xhdGlvbj10cmFuc2xhdGlvbiwKICAgICAgICAgICAgICAgIGZpbGVfbmFtZT10ZXh0X2ZpbGUuc3RlbSwKICAgICAgICAgICAgICAgIG91dHB1dF9kaXJlY3Rvcnk9b3V0cHV0X2RpcmVjdG9yeSwKICAgICAgICAgICAgKQogICAgICAgICAgICAjIE5vdGUgYXMgYSBzdWNjZXNzIGluIHRoZSBsaXN0OgogICAgICAgICAgICBzdWNjZXNzZXMuYXBwZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRleHRfZmlsZS5uYW1lLAogICAgICAgICAgICAgICAgICAgIHRyYW5zbGF0aW9uX2ZpbGUubmFtZSwKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIE5vdGUgdGhlIGV4Y2VwdGlvbiBhcyBlcnJvciBpbiB0aGUgZGljdGlvbmFyeToKICAgICAgICAgICAgaWYgdmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIkVycm9yIGluIGZpbGU6ICd7dGV4dF9maWxlLm5hbWV9JyIpCiAgICAgICAgICAgIGVycm9yc1tzdHIodGV4dF9maWxlLm5hbWUpXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBDb25zdHJ1Y3QgdGhlIHRyYW5zbGF0aW9ucyBkYXRhZnJhbWU6CiAgICBjb2x1bW5zID0gWwogICAgICAgICJ0ZXh0X2ZpbGUiLAogICAgICAgICJ0cmFuc2xhdGlvbl9maWxlIiwKICAgIF0KICAgIHN1Y2Nlc3NlcyA9IHBkLkRhdGFGcmFtZSgKICAgICAgICBzdWNjZXNzZXMsCiAgICAgICAgY29sdW1ucz1jb2x1bW5zLAogICAgKQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oCiAgICAgICAgICAgIGYiRG9uZSAoe3N1Y2Nlc3Nlcy5zaGFwZVswXX0ve2xlbih0ZXh0X2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiVHJhbnNsYXRpb25zIHN1bW1hcnk6XG4iCiAgICAgICAgICAgIGYie3N1Y2Nlc3Nlcy5oZWFkKCl9IgogICAgICAgICkKICAgIHJldHVybiBzdHIob3V0cHV0X2RpcmVjdG9yeSksIHN1Y2Nlc3NlcywgZXJyb3JzCgoKZGVmIF9nZXRfdGV4dF9maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IExpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICB0ZXh0X2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgdGV4dF9maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgZmlsZSBwYXRoLiAiCiAgICAgICAgICAgIGYiR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gdGV4dF9maWxlcwoKCmRlZiBfZ2V0X3RyYW5zbGF0aW9uX3BpcGVsaW5lKAogICAgbW9kZWxfbmFtZTogc3RyID0gTm9uZSwKICAgIHNvdXJjZV9sYW5ndWFnZTogc3RyID0gTm9uZSwKICAgIHRhcmdldF9sYW5ndWFnZTogc3RyID0gTm9uZSwKICAgIGRldmljZTogc3RyID0gTm9uZSwKICAgIG1vZGVsX2t3YXJnczogZGljdCA9IE5vbmUsCiAgICBiYXRjaF9zaXplOiBpbnQgPSBOb25lLAopIC0+IFR1cGxlW3RyYW5zZm9ybWVycy5QaXBlbGluZSwgc3RyXToKICAgICMgQ29uc3RydWN0IHRoZSBtb2RlbCBuYW1lIC0gaWYgbW9kZWwgbmFtZSBpcyBwcm92aWRlZCAobm90IE5vbmUpIHRoZW4gd2UgdGFrZSBpdCwgb3RoZXJ3aXNlIHdlIGNoZWNrIGJvdGggc291cmNlCiAgICAjIGFuZCB0YXJnZXQgd2VyZSBwcm92aWRlZCB0byBjb25zdHJ1Y3QgdGhlIG1vZGVsIG5hbWU6CiAgICBpZiBtb2RlbF9uYW1lIGlzIE5vbmUgYW5kIChzb3VyY2VfbGFuZ3VhZ2UgaXMgTm9uZSBvciB0YXJnZXRfbGFuZ3VhZ2UgaXMgTm9uZSk6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgIk5vIG1vZGVsIG5hbWUgd2VyZSBnaXZlbiBhbmQgbWlzc2luZyBzb3VyY2UgYW5kIC8gb3IgdGFyZ2V0IGxhbmd1YWdlcy4gSW4gb3JkZXIgdG8gdHJhbnNsYXRlIHlvdSBtdXN0ICIKICAgICAgICAgICAgInBhc3MgYSBgbW9kZWxfbmFtZWAgb3IgYm90aCBgc291cmNlX2xhbmd1YWdlYCBhbmQgYHRhcmdldF9sYW5ndWFnZWAuIgogICAgICAgICkKICAgIGVsaWYgbW9kZWxfbmFtZSBpcyBOb25lOgogICAgICAgIG1vZGVsX25hbWUgPSBmIkhlbHNpbmtpLU5MUC9vcHVzLW10LXtzb3VyY2VfbGFuZ3VhZ2V9LXt0YXJnZXRfbGFuZ3VhZ2V9IgoKICAgICMgSW5pdGlhbGl6ZSB0aGUgdHJhbnNsYXRpb24gcGlwZWxpbmU6CiAgICB0cnk6CiAgICAgICAgdHJhbnNsYXRpb25fcGlwZWxpbmUgPSB0cmFuc2Zvcm1lcnMucGlwZWxpbmUoCiAgICAgICAgICAgIHRhc2s9InRyYW5zbGF0aW9uIiwKICAgICAgICAgICAgbW9kZWw9bW9kZWxfbmFtZSwKICAgICAgICAgICAgdG9rZW5pemVyPW1vZGVsX25hbWUsCiAgICAgICAgICAgIGRldmljZT1kZXZpY2UsCiAgICAgICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3MsCiAgICAgICAgICAgIGJhdGNoX3NpemU9YmF0Y2hfc2l6ZSwKICAgICAgICApCiAgICBleGNlcHQgT1NFcnJvciBhcyBsb2FkX2V4Y2VwdGlvbjoKICAgICAgICBpZiAoCiAgICAgICAgICAgICJpcyBub3QgYSB2YWxpZCBtb2RlbCBpZGVudGlmaWVyIGxpc3RlZCBvbiAnaHR0cHM6Ly9odWdnaW5nZmFjZS5jby9tb2RlbHMnIgogICAgICAgICAgICBpbiBzdHIobG9hZF9leGNlcHRpb24pCiAgICAgICAgICAgIGFuZCBzb3VyY2VfbGFuZ3VhZ2UKICAgICAgICApOgogICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgZiJUaGUgbW9kZWwgJ3ttb2RlbF9uYW1lfScgaXMgbm90IGEgdmFsaWQgbW9kZWwgaWRlbnRpZmllci4gIgogICAgICAgICAgICAgICAgZiJUaGUgcGFyYW1ldGVycyBgc291cmNlX2xhbmd1YWdlYCBhbmQgYHRhcmdldF9sYW5ndWFnZWAgYXJlIHVzZWQgdG8gY29uc3RydWN0IGEgSGVsc2lua2kgbW9kZWwgZm9yICIKICAgICAgICAgICAgICAgIGYidGV4dCB0byB0ZXh0IGdlbmVyYXRpb24sIGJ1dCB0aGUgbW9kZWwgY3JlYXRlZCBmcm9tIHRoZSBnaXZlbiBsYW5ndWFnZXMgZG9lcyBub3QgZXhpc3QuICIKICAgICAgICAgICAgICAgIGYiWW91IG1heSBjaGVjayBsYW5ndWFnZSBpZGVudGlmaWVycyBhdCAiCiAgICAgICAgICAgICAgICBmImh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL2FkbWluLXNkay9kaXJlY3RvcnkvdjEvbGFuZ3VhZ2VzLCBhbmQgaWYgdGhlIGVycm9yIHdhcyBub3QgZml4ZWQsIG9uZSAiCiAgICAgICAgICAgICAgICBmIm9yIG1vcmUgbGFuZ3VhZ2UgY29kZSBtaWdodCBiZSB3aXRoIDMgbGV0dGVycyBhbmQgbmVlZHMgdG8gYmUgZm91bmQgb25saW5lLiAiCiAgICAgICAgICAgICAgICBmIlJlbWVtYmVyLCB5b3UgY2FuIGFsd2F5cyBjaG9vc2UgYSBtb2RlbCBkaXJlY3RseSBmcm9tIHRoZSBIdWdnaW5nZmFjZSBodWIgYnkgdXNpbmcgdGhlIGBtb2RlbF9uYW1lYCAiCiAgICAgICAgICAgICAgICBmInBhcmFtZXRlci4iCiAgICAgICAgICAgICkgZnJvbSBsb2FkX2V4Y2VwdGlvbgogICAgICAgIHJhaXNlIGxvYWRfZXhjZXB0aW9uCgogICAgcmV0dXJuIHRyYW5zbGF0aW9uX3BpcGVsaW5lLCBtb2RlbF9uYW1lCgoKZGVmIF90cmFuc2xhdGUoCiAgICB0ZXh0X2ZpbGU6IHBhdGhsaWIuUGF0aCwKICAgIHRyYW5zbGF0aW9uX3BpcGVsaW5lOiB0cmFuc2Zvcm1lcnMuUGlwZWxpbmUsCiAgICB0cmFuc2xhdGlvbl9rd2FyZ3M6IGRpY3QsCikgLT4gc3RyOgogICAgIyBSZWFkIHRoZSB0ZXh0IGZyb20gZmlsZToKICAgIHdpdGggb3Blbih0ZXh0X2ZpbGUsICJyIikgYXMgZnA6CiAgICAgICAgdGV4dCA9IGZwLnJlYWQoKQoKICAgICMgU3BsaXQgdG8gcGFyYWdyYXBocyBhbmQgZWFjaCBwYXJhZ3JhcGggdG8gc2VudGVuY2VzOgogICAgcGFyYWdyYXBocyA9IFtwYXJhZ3JhcGguc3BsaXQoIi4iKSBmb3IgcGFyYWdyYXBoIGluIHRleHQuc3BsaXQoIlxuIildCgogICAgIyBEaXNjb3ZlciB0aGUgbmV3bGluZSBpbmRleGVzIHRvIHJlc3RvcmUgdGhlIGZpbGUgdG8gaXRzIHN0cnVjdHVyZSBwb3N0IHRyYW5zbGF0aW9uOgogICAgbmV3bGluZXNfaW5kZXhlcyA9IFtdCiAgICBmb3IgcGFyYWdyYXBoIGluIHBhcmFncmFwaHNbOi0xXToKICAgICAgICBpZiBsZW4obmV3bGluZXNfaW5kZXhlcykgPT0gMDoKICAgICAgICAgICAgbmV3bGluZXNfaW5kZXhlcy5hcHBlbmQobGVuKHBhcmFncmFwaCkgLSAxKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIG5ld2xpbmVzX2luZGV4ZXMuYXBwZW5kKG5ld2xpbmVzX2luZGV4ZXNbLTFdICsgbGVuKHBhcmFncmFwaCkpCgogICAgIyBQcmVwYXJlIHRoZSBiYXRjaGVzIChlYWNoIHNlbnRlbmNlIGZyb20gdGhlIHBhcmFncmFwaHMpLiBOb3RpY2Ugd2UgYWRkIGEgZG90IG5vdCBvbmx5IHRvIHJlc3RvcmUgdGhlIHNlbnRlbmNlCiAgICAjIHN0cnVjdHVyZSBidXQgdG8gaWdub3JlIGVtcHR5IHN0cmluZ3MgYXMgaXQgd2lsbCBydWluIHRoZSB0cmFuc2xhdGlvbjoKICAgIHNlbnRlbmNlcyA9IFtmIntsaW5lfS4iIGZvciBwYXJhZ3JhcGggaW4gcGFyYWdyYXBocyBmb3IgbGluZSBpbiBwYXJhZ3JhcGhdCgogICAgIyBUcmFuc2xhdGUgdGhlIHNlbnRlbmNlczoKICAgIHRyYW5zbGF0aW9ucyA9IHRyYW5zbGF0aW9uX3BpcGVsaW5lKHNlbnRlbmNlcywgKip0cmFuc2xhdGlvbl9rd2FyZ3MpCgogICAgIyBSZXN0cnVjdHVyZSB0aGUgZnVsbCB0ZXh0IGZyb20gdGhlIHNlbnRlbmNlczoKICAgIHRyYW5zbGF0ZWRfdGV4dCA9IFtdCiAgICBuZXdsaW5lX2luZGV4ID0gbmV3bGluZXNfaW5kZXhlcy5wb3AoMCkgaWYgbmV3bGluZXNfaW5kZXhlcyBlbHNlIE5vbmUKICAgIGZvciBpLCB0cmFuc2xhdGlvbiBpbiBlbnVtZXJhdGUodHJhbnNsYXRpb25zKToKICAgICAgICAjIEdldCB0aGUgdHJhbnNsYXRpb246CiAgICAgICAgdGV4dCA9IHRyYW5zbGF0aW9uWyJ0cmFuc2xhdGlvbl90ZXh0Il0KICAgICAgICAjIFZhbGlkYXRlIGlmIGl0IHdhcyBhbiBlbXB0eSBzZW50ZW5jZSBiZWZvcmU6CiAgICAgICAgaWYgdGV4dCA9PSAiLiI6CiAgICAgICAgICAgIHRleHQgPSAiIgogICAgICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIGluc2VydCBhIG5ld2xpbmU6CiAgICAgICAgaWYgbmV3bGluZV9pbmRleCBhbmQgbmV3bGluZV9pbmRleCA9PSBpOgogICAgICAgICAgICB0ZXh0ICs9ICJcbiIKICAgICAgICAgICAgbmV3bGluZV9pbmRleCA9IG5ld2xpbmVzX2luZGV4ZXMucG9wKDApIGlmIG5ld2xpbmVzX2luZGV4ZXMgZWxzZSBOb25lCiAgICAgICAgIyBDb2xsZWN0IGl0OgogICAgICAgIHRyYW5zbGF0ZWRfdGV4dC5hcHBlbmQodGV4dCkKICAgIHRyYW5zbGF0ZWRfdGV4dCA9ICIiLmpvaW4odHJhbnNsYXRlZF90ZXh0KQoKICAgIHJldHVybiB0cmFuc2xhdGVkX3RleHQKCgpkZWYgX3NhdmVfdG9fZmlsZSgKICAgIHRyYW5zbGF0aW9uOiBzdHIsIGZpbGVfbmFtZTogc3RyLCBvdXRwdXRfZGlyZWN0b3J5OiBwYXRobGliLlBhdGgKKSAtPiBwYXRobGliLlBhdGg6CiAgICAjIFByZXBhcmUgdGhlIGZpbGUgZnVsbCBwYXRoIChjaGVja2luZyBmb3Igbm8gZHVwbGljYXRpb25zKToKICAgIHRyYW5zbGF0aW9uX2ZpbGUgPSBvdXRwdXRfZGlyZWN0b3J5IC8gZiJ7ZmlsZV9uYW1lfS50eHQiCiAgICBpID0gMQogICAgd2hpbGUgdHJhbnNsYXRpb25fZmlsZS5leGlzdHMoKToKICAgICAgICBpICs9IDEKICAgICAgICB0cmFuc2xhdGlvbl9maWxlID0gb3V0cHV0X2RpcmVjdG9yeSAvIGYie2ZpbGVfbmFtZX1fe2l9LnR4dCIKCiAgICAjIE1ha2Ugc3VyZSBhbGwgZGlyZWN0b3JpZXMgYXJlIGNyZWF0ZWQ6CiAgICB0cmFuc2xhdGlvbl9maWxlLnBhcmVudC5ta2RpcihleGlzdF9vaz1UcnVlLCBwYXJlbnRzPVRydWUpCgogICAgIyBXcml0ZSB0byBmaWxlOgogICAgd2l0aCBvcGVuKHRyYW5zbGF0aW9uX2ZpbGUsICJ3IikgYXMgZnA6CiAgICAgICAgZnAud3JpdGUodHJhbnNsYXRpb24pCgogICAgcmV0dXJuIHRyYW5zbGF0aW9uX2ZpbGUK - base_image: mlrun/mlrun - origin_filename: '' - image: '' - default_handler: translate - disable_auto_mount: false - command: '' - description: Translate text files from one language to another -verbose: false +kind: job metadata: + tag: '' categories: - genai - NLP - tag: '' name: translate -kind: job diff --git a/functions/src/translate/item.yaml b/functions/src/translate/item.yaml index eb0e821e4..68f176ac2 100644 --- a/functions/src/translate/item.yaml +++ b/functions/src/translate/item.yaml @@ -12,7 +12,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0-rc41 name: translate platformVersion: 3.5.3 spec: @@ -23,8 +23,8 @@ spec: requirements: - transformers - sentencepiece - - torch + - torch>=2.6 - tqdm url: '' -version: 0.2.0 +version: 0.3.0 test_valid: True diff --git a/functions/src/translate/requirements.txt b/functions/src/translate/requirements.txt index 94e548463..746da576c 100644 --- a/functions/src/translate/requirements.txt +++ b/functions/src/translate/requirements.txt @@ -1,4 +1,4 @@ transformers tqdm -torch +torch>=2.6 sentencepiece \ No newline at end of file diff --git a/modules/README.md b/modules/README.md index 05e7cfefd..c3ed1c597 100644 --- a/modules/README.md +++ b/modules/README.md @@ -6,6 +6,19 @@ | Name | Description | Kind | Categories | | --- | --- | --- | --- | -| [count_events](/home/runner/work/functions/functions/modules/src/count_events) | Count events in each time window | monitoring_application | model-serving | -| [histogram_data_drift](/home/runner/work/functions/functions/modules/src/histogram_data_drift) | Model-monitoring application for detecting and visualizing data drift | monitoring_application | model-serving, structured-ML | +| [count_events](https://github.com/mlrun/functions/tree/development/modules/src/count_events) | Count events in each time window | monitoring_application | model-serving | +| [evidently_iris](https://github.com/mlrun/functions/tree/development/modules/src/evidently_iris) | Demonstrates Evidently integration in MLRun for data quality and drift monitoring using the Iris dataset | monitoring_application | model-serving, structured-ML | +| [histogram_data_drift](https://github.com/mlrun/functions/tree/development/modules/src/histogram_data_drift) | Model-monitoring application for detecting and visualizing data drift | monitoring_application | model-serving, structured-ML | +| [openai_proxy_app](https://github.com/mlrun/functions/tree/development/modules/src/openai_proxy_app) | OpenAI application runtime based on fastapi | generic | genai | + + +## Catalog + + +| Name | Description | Kind | Categories | +| --- | --- | --- | --- | +| [count_events](https://github.com/mlrun/functions/tree/development/modules/src/count_events) | Count events in each time window | monitoring_application | model-serving | +| [evidently_iris](https://github.com/mlrun/functions/tree/development/modules/src/evidently_iris) | Demonstrates Evidently integration in MLRun for data quality and drift monitoring using the Iris dataset | monitoring_application | model-serving, structured-ML | +| [histogram_data_drift](https://github.com/mlrun/functions/tree/development/modules/src/histogram_data_drift) | Model-monitoring application for detecting and visualizing data drift | monitoring_application | model-serving, structured-ML | +| [openai_proxy_app](https://github.com/mlrun/functions/tree/development/modules/src/openai_proxy_app) | OpenAI application runtime based on fastapi | generic | genai | diff --git a/modules/src/evidently_iris/evidently_iris.ipynb b/modules/src/evidently_iris/evidently_iris.ipynb new file mode 100644 index 000000000..c3299f82f --- /dev/null +++ b/modules/src/evidently_iris/evidently_iris.ipynb @@ -0,0 +1,1295 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8f92a6bb-e4b4-4b5d-91c7-2e99c97798c6", + "metadata": {}, + "source": [ + "# Evidently Iris Demo\n", + "\n", + "In this notebook, we’ll import the hub’s Evidently demo app, which monitors data quality and drift on Scikit-Learn’s Iris dataset. We’ll run it using the `evaluate()` method with a slightly modified dataset as the monitored data.\n", + "\n", + "The Evidently Iris module demonstrates a simple example of integrating MLRun with Evidently for data monitoring, which you can adapt to fit your own project needs or use as a reference implementation." + ] + }, + { + "cell_type": "markdown", + "id": "a6775277-5f4f-4261-9a06-5c6d87cb85c7", + "metadata": {}, + "source": [ + "## Set up an MLRun project and prepare the data" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "d7a8c256-035f-4261-b494-f3f3cbd8c77c", + "metadata": {}, + "outputs": [], + "source": [ + "import mlrun\n", + "project = mlrun.get_or_create_project(\"evidently-demo\",'./evidently-demo')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "1e89667f-f84e-492a-a886-61104bc5ce49", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.datasets import load_iris\n", + "import pandas as pd\n", + "from mlrun.feature_store.api import norm_column_name\n", + "\n", + "iris = load_iris()\n", + "columns = [norm_column_name(col) for col in iris.feature_names]\n", + "current_df = pd.DataFrame(iris.data, columns=columns)\n", + "current_df[\"sepal_length_cm\"] += 0.3 # simulate drift" + ] + }, + { + "cell_type": "markdown", + "id": "af6e56af-c99d-481e-a32e-f7e5eac4ae3a", + "metadata": {}, + "source": [ + "## Get the module from the hub and edit its defaults" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "35a4bb6b-d15e-4bfd-8d04-2fa188cb36cc", + "metadata": {}, + "outputs": [], + "source": [ + "hub_mod = mlrun.get_hub_module(\"hub://evidently_iris\", download_files=True)\n", + "src_file_path = hub_mod.get_module_file_path()" + ] + }, + { + "cell_type": "markdown", + "id": "ba0c043b-7356-44da-b6d2-84eb02718482", + "metadata": {}, + "source": [ + "We need to modify the class defaults to include the Evidently workspace path and project ID parameters. This can be done in one of two ways: either by editing the downloaded source file directly and then evaluating with the standard class, or - as we’ll do now - by adding an inheriting class to the same file and evaluating using that new class.\n", + "\n", + "(Note: this is only needed when runnning the app using `evaluate()`. When setting it as a real-time function we can simply pass the parameters)." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "4e9253a9-58bd-4732-8eb1-80a7d15b2e7a", + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import uuid\n", + "\n", + "ws = Path(\"./evidently_workspace\")\n", + "ws.mkdir(parents=True, exist_ok=True) # will create if missing\n", + "evidently_project_id = str(uuid.uuid4())\n", + "\n", + "wrapper_code = f\"\"\"\n", + "class EvidentlyIrisMonitoringAppWithWorkspaceSet(EvidentlyIrisMonitoringApp):\n", + " def __init__(self) -> None:\n", + " super().__init__(evidently_workspace_path=\"{ws}\", evidently_project_id=\"{evidently_project_id}\")\n", + " \"\"\"\n", + "\n", + "with open(src_file_path, \"a\") as f:\n", + " f.write(wrapper_code)" + ] + }, + { + "cell_type": "markdown", + "id": "5776541f-2d6f-4c10-9246-75fe14e1bbea", + "metadata": {}, + "source": [ + "Now we can actually import it as a module, using the `module()` method" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "3742576d-6da2-423d-8c1c-2861712a698f", + "metadata": {}, + "outputs": [], + "source": [ + "app_module = hub_mod.module()\n", + "evidently_app = app_module.EvidentlyIrisMonitoringAppWithWorkspaceSet" + ] + }, + { + "cell_type": "markdown", + "id": "57a81ea8-f203-4152-9492-a0f7b916d02b", + "metadata": {}, + "source": [ + "## Run the app\n", + "We are ready to call `evaluate()` (notice that the run is linked to the current (active) project that we created at the beggining of the notebook)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d8103577-8523-4b64-bd67-e93bbde8dd06", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 2025-11-17 09:14:43,241 [info] Changing function name - adding `\"-batch\"` suffix: {\"func_name\":\"evidentlyirismonitoringappwithworkspaceset-batch\"}\n", + "> 2025-11-17 09:14:43,580 [info] Storing function: {\"db\":\"http://mlrun-api:8080\",\"name\":\"evidentlyirismonitoringappwithworkspaceset-batch--handler\",\"uid\":\"9ecf72a1bd82498c92d5897809b6a438\"}\n", + "> 2025-11-17 09:14:43,856 [info] downloading v3io:///projects/evidently-demo/artifacts/evidentlyirismonitoringappwithworkspaceset-batch_sample_data.parquet to local temp file\n", + "> 2025-11-17 09:14:43,890 [info] Running evidently app\n", + "> 2025-11-17 09:14:46,214 [info] Logged evidently object\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
projectuiditerstartendstatekindnamelabelsinputsparametersresultsartifact_uris
evidently-demo0Nov 17 09:14:43NaTcompletedrunevidentlyirismonitoringappwithworkspaceset-batch--handler
v3io_user=iguazio
kind=local
owner=iguazio
host=jupyter-97c64f97b-8qtcv
sample_data
write_output=False
existing_data_handling=fail_on_overlap
stream_profile=None
return={result_name: 'data_drift_test', result_value: 0.5, result_kind: 0, result_status: 1, result_extra_data: '{}'}
evidently_report=store://artifacts/evidently-demo/evidentlyirismonitoringappwithworkspaceset-batch--handler_evidently_report#0@9ecf72a1bd82498c92d5897809b6a438^2f82c069b396f23b4daae81540ffa386b44f165c
\n", + "
\n", + "
\n", + "
\n", + " Title\n", + " ×\n", + "
\n", + " \n", + "
\n", + "
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/html": [ + " > to track results use the .show() or .logs() methods or click here to open in UI" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 2025-11-17 09:14:46,354 [info] Run execution finished: {\"name\":\"evidentlyirismonitoringappwithworkspaceset-batch--handler\",\"status\":\"completed\"}\n" + ] + } + ], + "source": [ + "# Evaluate directly on the sample data\n", + "run_result = evidently_app.evaluate(\n", + " func_path=hub_mod.get_module_file_path(),\n", + " sample_data=current_df,\n", + " run_local=True)" + ] + }, + { + "cell_type": "markdown", + "id": "2c6843cd-70d4-4e1a-8aa2-52b6ef5b0ec9", + "metadata": {}, + "source": [ + "## Examine the results\n", + "Notice that the 0.5 value in the demo run result is not derived from Evidently’s drift metrics, but is a constant placeholder added for demonstration only.\n", + "\n", + "Let's take a look at the artifact the app generated for us:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "7f1680f5-0ee7-4a82-a351-f8348bf398cc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "artifact_key = f\"{run_result.metadata.name}_evidently_report\"\n", + "artifact = project.get_artifact(artifact_key)\n", + "artifact.to_dataitem().show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "mlrun-base-py311", + "language": "python", + "name": "conda-env-mlrun-base-py311-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/modules/src/evidently_iris/evidently_iris.py b/modules/src/evidently_iris/evidently_iris.py new file mode 100644 index 000000000..e7a9f3ef9 --- /dev/null +++ b/modules/src/evidently_iris/evidently_iris.py @@ -0,0 +1,117 @@ +# Copyright 2025 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import pandas as pd +from sklearn.datasets import load_iris + +import mlrun.model_monitoring.applications.context as mm_context +from mlrun.common.schemas.model_monitoring.constants import ( + ResultKindApp, + ResultStatusApp, +) +from mlrun.feature_store.api import norm_column_name +from mlrun.model_monitoring.applications import ModelMonitoringApplicationResult +from mlrun.model_monitoring.applications.evidently import EvidentlyModelMonitoringApplicationBase + +from evidently.core.report import Report, Snapshot +from evidently.metrics import DatasetMissingValueCount, ValueDrift +from evidently.presets import DataDriftPreset, DataSummaryPreset +from evidently.ui.workspace import ( + STR_UUID, + OrgID, +) + +_PROJECT_NAME = "Iris Monitoring" +_PROJECT_DESCRIPTION = "Test project using iris dataset" + + +class EvidentlyIrisMonitoringApp(EvidentlyModelMonitoringApplicationBase): + """ + This model monitoring application is a simple example of integrating MLRun with Evidently for data monitoring, + which you can adapt to fit your own project needs or use as a reference implementation. + """ + NAME = "Evidently-App-Test" + + def __init__( + self, + evidently_project_id: Optional["STR_UUID"] = None, + evidently_workspace_path: Optional[str] = None, + cloud_workspace: bool = False, + evidently_organization_id: Optional["OrgID"] = None, + ) -> None: + self.org_id = evidently_organization_id + self._init_iris_data() + super().__init__( + evidently_project_id=evidently_project_id, + evidently_workspace_path=evidently_workspace_path, + cloud_workspace=cloud_workspace, + ) + + def _init_iris_data(self) -> None: + iris = load_iris() + self.columns = [norm_column_name(col) for col in iris.feature_names] + self.train_set = pd.DataFrame(iris.data, columns=self.columns) + + def do_tracking( + self, monitoring_context: mm_context.MonitoringApplicationContext + ) -> ModelMonitoringApplicationResult: + monitoring_context.logger.info("Running evidently app") + + sample_df = monitoring_context.sample_df[self.columns] + + data_drift_report_run = self.create_report_run( + sample_df, monitoring_context.end_infer_time + ) + self.evidently_workspace.add_run( + self.evidently_project_id, data_drift_report_run + ) + + self.log_evidently_object( + monitoring_context, data_drift_report_run, "evidently_report" + ) + monitoring_context.logger.info("Logged evidently object") + + return ModelMonitoringApplicationResult( + name="data_drift_test", + value=0.5, + kind=ResultKindApp.data_drift, + status=ResultStatusApp.potential_detection, + ) + + def create_report_run( + self, sample_df: pd.DataFrame, schedule_time: pd.Timestamp + ) -> "Snapshot": + metrics = [ + DataDriftPreset(), + DatasetMissingValueCount(), + DataSummaryPreset(), + ] + metrics.extend( + [ + ValueDrift(column=col_name, method="wasserstein") + for col_name in self.columns + ] + ) + + data_drift_report = Report( + metrics=metrics, + metadata={"timestamp": str(schedule_time)}, + include_tests=True, + ) + + return data_drift_report.run( + current_data=sample_df, reference_data=self.train_set + ) diff --git a/modules/src/evidently_iris/item.yaml b/modules/src/evidently_iris/item.yaml new file mode 100644 index 000000000..262b7e1b7 --- /dev/null +++ b/modules/src/evidently_iris/item.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +categories: +- model-serving +- structured-ML +description: Demonstrates Evidently integration in MLRun for data quality and drift monitoring using the Iris dataset +example: evidently_iris.ipynb +generationDate: 2025-11-09 +hidden: false +labels: + author: Iguazio +mlrunVersion: 1.10.0-rc41 +name: evidently_iris +spec: + filename: evidently_iris.py + image: mlrun/mlrun + kind: monitoring_application + requirements: + - scikit-learn~=1.5.2 + - evidently~=0.7.5 + - pandas +version: 1.0.0 \ No newline at end of file diff --git a/modules/src/evidently_iris/requirements.txt b/modules/src/evidently_iris/requirements.txt new file mode 100644 index 000000000..6bd12d901 --- /dev/null +++ b/modules/src/evidently_iris/requirements.txt @@ -0,0 +1,3 @@ +scikit-learn~=1.5.2 +evidently~=0.7.5 +pandas \ No newline at end of file diff --git a/modules/src/evidently_iris/test_evidently_iris.py b/modules/src/evidently_iris/test_evidently_iris.py new file mode 100644 index 000000000..6488768fd --- /dev/null +++ b/modules/src/evidently_iris/test_evidently_iris.py @@ -0,0 +1,72 @@ +# Copyright 2023 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from contextlib import AbstractContextManager +from contextlib import nullcontext as does_not_raise +from pathlib import Path +from uuid import uuid4 + +import pytest +import semver + +from mlrun.errors import MLRunIncompatibleVersionError +from mlrun.model_monitoring.applications.evidently.base import ( + _check_evidently_version, +) + +from evidently_iris import EvidentlyIrisMonitoringApp + + +@pytest.mark.parametrize( + ("cur", "ref", "expectation"), + [ + ("0.4.11", "0.4.11", does_not_raise()), + ("0.4.12", "0.4.11", does_not_raise()), + ("1.23.0", "1.1.32", does_not_raise()), + ("0.4.11", "0.4.12", pytest.raises(MLRunIncompatibleVersionError)), + ("0.4.11", "0.4.12", pytest.raises(MLRunIncompatibleVersionError)), + ("1.0.3", "0.9.9", pytest.raises(MLRunIncompatibleVersionError)), + ("0.6.0", "0.3.0", pytest.warns(UserWarning)), + pytest.param("0.6.0", "0.3.0", does_not_raise(), marks=pytest.mark.xfail), + ], +) +def test_version_check( + cur: str, + ref: str, + expectation: AbstractContextManager, +) -> None: + with warnings.catch_warnings(): + warnings.simplefilter("error") + with expectation: + _check_evidently_version( + cur=semver.Version.parse(cur), ref=semver.Version.parse(ref) + ) + + +def test_demo_evidently_app(tmpdir: Path) -> None: + """Test that the workspace and the project's dashboards are created""" + evidently_app = EvidentlyIrisMonitoringApp( + evidently_project_id=uuid4(), evidently_workspace_path=str(tmpdir) + ) + run = evidently_app.create_report_run( + sample_df=evidently_app.train_set, schedule_time=None + ) + added_run_uid = evidently_app.evidently_workspace.add_run( + project_id=evidently_app.evidently_project_id, + run=run, + ).id + assert evidently_app.evidently_workspace.list_runs( + project_id=evidently_app.evidently_project_id + ) == [added_run_uid], "Different project runs than expected" diff --git a/modules/src/openai_proxy_app/item.yaml b/modules/src/openai_proxy_app/item.yaml new file mode 100644 index 000000000..bf295cf2a --- /dev/null +++ b/modules/src/openai_proxy_app/item.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +categories: +- genai +description: OpenAI application runtime based on fastapi +example: openai_proxy_app.ipynb +generationDate: 2025-11-11:12-25 +hidden: false +labels: + author: Iguazio +mlrunVersion: 1.10.0 +name: openai_proxy_app +spec: + filename: openai_proxy_app.py + image: mlrun/mlrun + requirements: + - fastapi>=0.110,<1.0 + - requests>=2.31,<3.0 + kind: generic +version: 1.0.0 diff --git a/modules/src/openai_proxy_app/openai_proxy_app.ipynb b/modules/src/openai_proxy_app/openai_proxy_app.ipynb new file mode 100644 index 000000000..123934fbd --- /dev/null +++ b/modules/src/openai_proxy_app/openai_proxy_app.ipynb @@ -0,0 +1,72 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "220629c8-17aa-45f6-ac81-0ca31e165412", + "metadata": {}, + "source": [ + "# OpenAI Module Demo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "967b4d5d-7250-40bf-8149-de11e1e3244c", + "metadata": {}, + "outputs": [], + "source": [ + "import mlrun\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17d208f4-a00a-42ef-a849-0fa79bed10cb", + "metadata": {}, + "outputs": [], + "source": [ + "project = mlrun.get_or_create_project(\"fastapi-openai\",user_project=True,context=\"./src\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67c93a0d-8240-48b8-808e-9cd0af418309", + "metadata": {}, + "outputs": [], + "source": [ + "app = mlrun.import_module(\"hub://openai\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "93e67d6a-5f53-4bda-b0b5-4e2977088139", + "metadata": {}, + "outputs": [], + "source": "app.OpenAIModule.deploy()" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/modules/src/openai_proxy_app/openai_proxy_app.py b/modules/src/openai_proxy_app/openai_proxy_app.py new file mode 100644 index 000000000..a0e9df7ac --- /dev/null +++ b/modules/src/openai_proxy_app/openai_proxy_app.py @@ -0,0 +1,56 @@ +# Copyright 2025 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#This module acts as a lightweight gateway to OpenAI-compatible APIs. +#You can send chat prompts, create embeddings, or get model responses without worrying about authentication or endpoint differences. +#It simplifies access so you can test, analyze, or integrate AI features directly into your projects or notebooks with minimal setup. + + +BASE64 = "IyBvcGVuYWlfcHJveHkvb3BlbmFpLnB5CgppbXBvcnQgb3MKaW1wb3J0IGpzb24KZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybGpvaW4KZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdCwgT3B0aW9uYWwKCmltcG9ydCByZXF1ZXN0cwpmcm9tIGZhc3RhcGkgaW1wb3J0IEZhc3RBUEksIFJlcXVlc3QsIFJlc3BvbnNlLCBCb2R5CgphcHAgPSBGYXN0QVBJKAogICAgdGl0bGU9Ik9wZW5BSSBQcm94eSBBcHAiLAogICAgZGVzY3JpcHRpb249IkxvY2FsIEZhc3RBUEkgcHJveHkgZm9yIE9wZW5BSSBzdHlsZSBlbmRwb2ludHMiLAogICAgdmVyc2lvbj0iMS4wLjAiLAopCgpPUEVOQUlfQkFTRV9VUkwgPSBvcy5nZXRlbnYoIk9QRU5BSV9CQVNFX1VSTCIsICJodHRwczovL2FwaS5vcGVuYWkuY29tIikucnN0cmlwKCIvIikKT1BFTkFJX0FQSV9LRVkgPSBvcy5nZXRlbnYoIk9QRU5BSV9BUElfS0VZIiwgIiIpCk9QRU5BSV9ERUZBVUxUX01PREVMID0gb3MuZ2V0ZW52KCJPUEVOQUlfREVGQVVMVF9NT0RFTCIsICJncHQtNG8tbWluaSIpCgoKZGVmIGJ1aWxkX2hlYWRlcnMoaW5jb21pbmc6IGRpY3QpIC0+IGRpY3Q6CiAgICBoZWFkZXJzID0ge30KICAgIGF1dGggPSBpbmNvbWluZy5nZXQoImF1dGhvcml6YXRpb24iKSBvciBpbmNvbWluZy5nZXQoIkF1dGhvcml6YXRpb24iKQogICAgaWYgYXV0aDoKICAgICAgICBoZWFkZXJzWyJBdXRob3JpemF0aW9uIl0gPSBhdXRoCiAgICBlbGlmIE9QRU5BSV9BUElfS0VZOgogICAgICAgIGhlYWRlcnNbIkF1dGhvcml6YXRpb24iXSA9IGYiQmVhcmVyIHtPUEVOQUlfQVBJX0tFWX0iCiAgICBjdHlwZSA9IGluY29taW5nLmdldCgiY29udGVudC10eXBlIikgb3IgaW5jb21pbmcuZ2V0KCJDb250ZW50LVR5cGUiKSBvciAiYXBwbGljYXRpb24vanNvbiIKICAgIGhlYWRlcnNbIkNvbnRlbnQtVHlwZSJdID0gY3R5cGUKICAgIHJldHVybiBoZWFkZXJzCgoKZGVmIGJ1aWxkX3RhcmdldChwYXRoOiBzdHIpIC0+IHN0cjoKICAgIGJhc2UgPSBPUEVOQUlfQkFTRV9VUkwKICAgIGlmIGJhc2UuZW5kc3dpdGgoIi92MSIpIG9yIGJhc2UuZW5kc3dpdGgoIi92MS8iKToKICAgICAgICBiYXNlID0gYmFzZVs6LTNdIGlmIGJhc2UuZW5kc3dpdGgoIi92MSIpIGVsc2UgYmFzZVs6LTRdCiAgICByZXR1cm4gdXJsam9pbihiYXNlICsgIi8iLCBwYXRoLmxzdHJpcCgiLyIpKQoKCmRlZiBmb3J3YXJkX2pzb24ocGF0aDogc3RyLCBib2R5OiBkaWN0LCBoZWFkZXJzOiBkaWN0LCBxdWVyeTogZGljdCk6CiAgICB0YXJnZXQgPSBidWlsZF90YXJnZXQocGF0aCkKICAgIHJlc3AgPSByZXF1ZXN0cy5wb3N0KAogICAgICAgIHRhcmdldCwKICAgICAgICBoZWFkZXJzPWhlYWRlcnMsCiAgICAgICAgcGFyYW1zPXF1ZXJ5LAogICAgICAgIGpzb249Ym9keSwKICAgICAgICB0aW1lb3V0PTYwLAogICAgKQogICAgcmV0dXJuIHJlc3AKCkBhcHAuZ2V0KCIvIikKZGVmIGhlYWx0aCgpOgogICAgcmV0dXJuIHsic3RhdHVzIjogIm9rIn0KCgojIHJlbGF4ZWQgY2hhdCBlbmRwb2ludCwgYWNjZXB0cyBhbnkgSlNPTiB0aGF0IGluY2x1ZGVzIG1lc3NhZ2VzCkBhcHAucG9zdCgiL3YxL2NoYXQvY29tcGxldGlvbnMiKQphc3luYyBkZWYgY2hhdF9jb21wbGV0aW9ucygKICAgIHJlcXVlc3Q6IFJlcXVlc3QsCiAgICBwYXlsb2FkOiBEaWN0W3N0ciwgQW55XSA9IEJvZHkoLi4uKSwKKToKICAgIGlmICJtZXNzYWdlcyIgbm90IGluIHBheWxvYWQgb3Igbm90IGlzaW5zdGFuY2UocGF5bG9hZFsibWVzc2FnZXMiXSwgbGlzdCk6CiAgICAgICAgcmV0dXJuIFJlc3BvbnNlKAogICAgICAgICAgICBjb250ZW50PWpzb24uZHVtcHMoeyJlcnJvciI6ICJtZXNzYWdlcyBtdXN0IGJlIGEgbGlzdCBvZiBjaGF0IG1lc3NhZ2VzIn0pLAogICAgICAgICAgICBzdGF0dXNfY29kZT00MDAsCiAgICAgICAgICAgIG1lZGlhX3R5cGU9ImFwcGxpY2F0aW9uL2pzb24iLAogICAgICAgICkKCiAgICBpZiAibW9kZWwiIG5vdCBpbiBwYXlsb2FkIG9yIHBheWxvYWRbIm1vZGVsIl0gaXMgTm9uZToKICAgICAgICBwYXlsb2FkWyJtb2RlbCJdID0gT1BFTkFJX0RFRkFVTFRfTU9ERUwKCiAgICBoZWFkZXJzID0gYnVpbGRfaGVhZGVycyhkaWN0KHJlcXVlc3QuaGVhZGVycykpCiAgICByZXNwID0gZm9yd2FyZF9qc29uKCIvdjEvY2hhdC9jb21wbGV0aW9ucyIsIHBheWxvYWQsIGhlYWRlcnMsIGRpY3QocmVxdWVzdC5xdWVyeV9wYXJhbXMpKQogICAgcmV0dXJuIFJlc3BvbnNlKAogICAgICAgIGNvbnRlbnQ9cmVzcC5jb250ZW50LAogICAgICAgIHN0YXR1c19jb2RlPXJlc3Auc3RhdHVzX2NvZGUsCiAgICAgICAgbWVkaWFfdHlwZT1yZXNwLmhlYWRlcnMuZ2V0KCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpLAogICAgKQoKCkBhcHAucG9zdCgiL3YxL2VtYmVkZGluZ3MiKQphc3luYyBkZWYgZW1iZWRkaW5ncygKICAgIHJlcXVlc3Q6IFJlcXVlc3QsCiAgICBwYXlsb2FkOiBEaWN0W3N0ciwgQW55XSA9IEJvZHkoLi4uKSwKKToKICAgIGlmICJtb2RlbCIgbm90IGluIHBheWxvYWQgb3Igbm90IHBheWxvYWRbIm1vZGVsIl06CiAgICAgICAgcGF5bG9hZFsibW9kZWwiXSA9ICJ0ZXh0LWVtYmVkZGluZy0zLXNtYWxsIgogICAgaGVhZGVycyA9IGJ1aWxkX2hlYWRlcnMoZGljdChyZXF1ZXN0LmhlYWRlcnMpKQogICAgcmVzcCA9IGZvcndhcmRfanNvbigiL3YxL2VtYmVkZGluZ3MiLCBwYXlsb2FkLCBoZWFkZXJzLCBkaWN0KHJlcXVlc3QucXVlcnlfcGFyYW1zKSkKICAgIHJldHVybiBSZXNwb25zZSgKICAgICAgICBjb250ZW50PXJlc3AuY29udGVudCwKICAgICAgICBzdGF0dXNfY29kZT1yZXNwLnN0YXR1c19jb2RlLAogICAgICAgIG1lZGlhX3R5cGU9cmVzcC5oZWFkZXJzLmdldCgiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKSwKICAgICkKCgpAYXBwLnBvc3QoIi92MS9yZXNwb25zZXMiKQphc3luYyBkZWYgcmVzcG9uc2VzX2FwaSgKICAgIHJlcXVlc3Q6IFJlcXVlc3QsCiAgICBwYXlsb2FkOiBEaWN0W3N0ciwgQW55XSA9IEJvZHkoLi4uKSwKKToKICAgIGlmICJtb2RlbCIgbm90IGluIHBheWxvYWQgb3IgcGF5bG9hZFsibW9kZWwiXSBpcyBOb25lOgogICAgICAgIHBheWxvYWRbIm1vZGVsIl0gPSBPUEVOQUlfREVGQVVMVF9NT0RFTAogICAgaGVhZGVycyA9IGJ1aWxkX2hlYWRlcnMoZGljdChyZXF1ZXN0LmhlYWRlcnMpKQogICAgcmVzcCA9IGZvcndhcmRfanNvbigiL3YxL3Jlc3BvbnNlcyIsIHBheWxvYWQsIGhlYWRlcnMsIGRpY3QocmVxdWVzdC5xdWVyeV9wYXJhbXMpKQogICAgcmV0dXJuIFJlc3BvbnNlKAogICAgICAgIGNvbnRlbnQ9cmVzcC5jb250ZW50LAogICAgICAgIHN0YXR1c19jb2RlPXJlc3Auc3RhdHVzX2NvZGUsCiAgICAgICAgbWVkaWFfdHlwZT1yZXNwLmhlYWRlcnMuZ2V0KCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpLAogICAgKQoKCiMgLS0tLS0tLS0tLS0tLS0tLSBjbGllbnQgLS0tLS0tLS0tLS0tLS0tLQpjbGFzcyBPcGVuQUlQcm94eUNsaWVudDoKICAgICIiIgogICAgU2ltcGxlIGNsaWVudCBmb3IgdGhlIGxvY2FsIHByb3h5LgogICAgRGVmYXVsdCBiYXNlIHVybCBpcyBodHRwOi8vbG9jYWxob3N0OjgwMDAKICAgIElmIGFwaV9rZXkgaXMgbm90IHByb3ZpZGVkLCBpdCB1c2VzIE9QRU5BSV9BUElfS0VZIGZyb20gZW52aXJvbm1lbnQuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgYmFzZV91cmw6IHN0ciA9ICJodHRwOi8vbG9jYWxob3N0OjgwMDAiLCBhcGlfa2V5OiBPcHRpb25hbFtzdHJdID0gTm9uZSk6CiAgICAgICAgc2VsZi5iYXNlX3VybCA9IGJhc2VfdXJsLnJzdHJpcCgiLyIpCiAgICAgICAgc2VsZi5hcGlfa2V5ID0gYXBpX2tleQoKICAgIGRlZiBfaGVhZGVycyhzZWxmKSAtPiBEaWN0W3N0ciwgc3RyXToKICAgICAgICBoZWFkZXJzID0geyJDb250ZW50LVR5cGUiOiAiYXBwbGljYXRpb24vanNvbiJ9CiAgICAgICAga2V5ID0gc2VsZi5hcGlfa2V5IG9yIG9zLmdldGVudigiT1BFTkFJX0FQSV9LRVkiLCAiIikKICAgICAgICBpZiBrZXk6CiAgICAgICAgICAgIGhlYWRlcnNbIkF1dGhvcml6YXRpb24iXSA9IGYiQmVhcmVyIHtrZXl9IgogICAgICAgIHJldHVybiBoZWFkZXJzCgogICAgZGVmIGNoYXQoc2VsZiwgbWVzc2FnZXM6IExpc3RbRGljdFtzdHIsIHN0cl1dLCBtb2RlbDogT3B0aW9uYWxbc3RyXSA9IE5vbmUpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgICAgIGJvZHk6IERpY3Rbc3RyLCBBbnldID0geyJtZXNzYWdlcyI6IG1lc3NhZ2VzfQogICAgICAgIGlmIG1vZGVsOgogICAgICAgICAgICBib2R5WyJtb2RlbCJdID0gbW9kZWwKICAgICAgICByZXNwID0gcmVxdWVzdHMucG9zdCgKICAgICAgICAgICAgZiJ7c2VsZi5iYXNlX3VybH0vdjEvY2hhdC9jb21wbGV0aW9ucyIsCiAgICAgICAgICAgIGhlYWRlcnM9c2VsZi5faGVhZGVycygpLAogICAgICAgICAgICBqc29uPWJvZHksCiAgICAgICAgICAgIHRpbWVvdXQ9NjAsCiAgICAgICAgKQogICAgICAgIHJlc3AucmFpc2VfZm9yX3N0YXR1cygpCiAgICAgICAgcmV0dXJuIHJlc3AuanNvbigpCgogICAgZGVmIGVtYmVkZGluZ3Moc2VsZiwgdGV4dDogQW55LCBtb2RlbDogT3B0aW9uYWxbc3RyXSA9IE5vbmUpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgICAgIGJvZHk6IERpY3Rbc3RyLCBBbnldID0geyJpbnB1dCI6IHRleHR9CiAgICAgICAgaWYgbW9kZWw6CiAgICAgICAgICAgIGJvZHlbIm1vZGVsIl0gPSBtb2RlbAogICAgICAgIHJlc3AgPSByZXF1ZXN0cy5wb3N0KAogICAgICAgICAgICBmIntzZWxmLmJhc2VfdXJsfS92MS9lbWJlZGRpbmdzIiwKICAgICAgICAgICAgaGVhZGVycz1zZWxmLl9oZWFkZXJzKCksCiAgICAgICAgICAgIGpzb249Ym9keSwKICAgICAgICAgICAgdGltZW91dD02MCwKICAgICAgICApCiAgICAgICAgcmVzcC5yYWlzZV9mb3Jfc3RhdHVzKCkKICAgICAgICByZXR1cm4gcmVzcC5qc29uKCkKCiAgICBkZWYgcmVzcG9uc2VzKHNlbGYsIGlucHV0X3RleHQ6IEFueSwgbW9kZWw6IE9wdGlvbmFsW3N0cl0gPSBOb25lKSAtPiBEaWN0W3N0ciwgQW55XToKICAgICAgICBib2R5OiBEaWN0W3N0ciwgQW55XSA9IHsiaW5wdXQiOiBpbnB1dF90ZXh0fQogICAgICAgIGlmIG1vZGVsOgogICAgICAgICAgICBib2R5WyJtb2RlbCJdID0gbW9kZWwKICAgICAgICByZXNwID0gcmVxdWVzdHMucG9zdCgKICAgICAgICAgICAgZiJ7c2VsZi5iYXNlX3VybH0vdjEvcmVzcG9uc2VzIiwKICAgICAgICAgICAgaGVhZGVycz1zZWxmLl9oZWFkZXJzKCksCiAgICAgICAgICAgIGpzb249Ym9keSwKICAgICAgICAgICAgdGltZW91dD02MCwKICAgICAgICApCiAgICAgICAgcmVzcC5yYWlzZV9mb3Jfc3RhdHVzKCkKICAgICAgICByZXR1cm4gcmVzcC5qc29uKCkKCgojIG9wdGlvbmFsIHF1aWNrIHNlbGYgdGVzdCB3aGVuIHJ1bm5pbmcgdGhpcyBmaWxlIGRpcmVjdGx5CmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICAjIHN0YXJ0IHRoZSBzZXJ2ZXIgaW4gYW5vdGhlciB0ZXJtaW5hbCBmaXJzdDoKICAgICMgdXZpY29ybiBvcGVuYWlfcHJveHkub3BlbmFpOmFwcCAtLWhvc3QgMC4wLjAuMCAtLXBvcnQgODAwMCAtLXJlbG9hZAogICAgYyA9IE9wZW5BSVByb3h5Q2xpZW50KCkKICAgIHRyeToKICAgICAgICBwcmludCgiSGVhbHRoOiIsIHJlcXVlc3RzLmdldChmIntjLmJhc2VfdXJsfS8iKS5qc29uKCkpCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgcHJpbnQoIlNlcnZlciBub3QgcnVubmluZzoiLCBlKQo=" +CMD = r''' +set -e +python - <<'PY' +import os, base64, pathlib +code = os.environ["BASE64"] +pathlib.Path("/opt/app").mkdir(parents=True, exist_ok=True) +with open("/opt/app/openai_proxy_app.py","wb") as f: + f.write(base64.b64decode(code)) +print("Wrote /opt/app/openai_proxy_app.py") +PY + +exec gunicorn openai:app \ + --chdir /opt/app \ + --bind 0.0.0.0:8000 \ + --worker-class uvicorn.workers.UvicornWorker \ + --log-level info +'''.strip() +class OpenAIModule: + def __init__(self,project): + self.project = project + self.fastapi_app = self.project.set_function(name="openai",kind="application",image="python:3.11") + self.fastapi_app.with_requirements([ + "fastapi>=0.110,<1.0", + "uvicorn[standard]>=0.29,<1.0", + "gunicorn>=21.2,<22.0", + "requests>=2.31,<3.0", + ]) + self.fastapi_app.set_env("BASE64",BASE64) + self.fastapi_app.set_internal_application_port(8000) + self.fastapi_app.spec.command = "/bin/sh" + self.fastapi_app.spec.args = ["-c", CMD] + + + + +