diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..c1275ae --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,2 @@ +# Require review from the Maintainers team for any change +* @your-org/DWA_Owner \ No newline at end of file diff --git a/README.md b/README.md index 28eb686..dd8e1b3 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,37 @@ -# DWA - Data Warehouse Automation Framework
(Fabric Edition) +# DWA - Data Warehouse Automation Framework (Fabric Edition) + ## Welcome -Welcome to the [Prodata](https://www.prodata.ie) Fabric Data warheouse Automation Framework. DWA is a meta data automation framework for data warheousing and data engineering. -We have been developing and improving this framework for 15 years and now it is available for Open Source for you to try. -
+ +Welcome to the Prodata Fabric Data Warehouse Automation Framework. DWA is a metadata automation framework for data warehousing and data engineering. We have been developing and improving this framework for 15 years, and it is now available as open source for you to try. + ## Overview + ![image](https://github.com/user-attachments/assets/678020fe-ead9-41f9-a77e-597350fa5e45) -The DWA Framework supports the entire enterprise data warheouse lifecycle or other data engineering project by reducing data engineering pipelines and tasks ot meta data and enforcing common -Enterprise Features (logging, lineage, error handling), orchestration and a very flexible template library +The DWA framework supports the entire enterprise data warehouse lifecycle (and other data engineering projects) by reducing data engineering pipelines and tasks to metadata and enforcing common enterprise features such as logging, lineage, and error handling, along with orchestration and a flexible template library. ## Getting Started and Documentation -You can browse this Github repos to use some of the Templates for ideas on your project.
-If you wwant to install the framework and try it out, or look deeper into documentation, examples and demos then -read the wiki below + +You can browse this GitHub repository to use some of the templates for ideas on your project. + +For installation instructions, examples, demos, and deeper documentation, see the project wiki: https://github.com/ProdataSQL/DWA/wiki -
## Engaging with Prodata for a Quick Start -if you want to try the Framework on your own Fabric environment, then Prodata offers a Fabric Data Warehouse Quick Start. -This is 2-3 weeks ot install framework and create a sample Proof of Concept using your actual data and then provide walk through and training on the concepts -to accelerate your getting started with Fabric. -
-
-More details on the link below + +If you want to try the framework in your Fabric environment, Prodata offers a Fabric Data Warehouse Quick Start. This is a 2–3 week engagement to install the framework, create a sample proof of concept using your actual data, and provide a walkthrough and training to accelerate your adoption of Fabric. + +More details: https://prodata.ie/fabric-poc/ -
+ ## Videos ### Introduction to DWA in Fabric [![image](https://github.com/user-attachments/assets/0cce133b-1d61-4cc0-9f58-70eac999de5b)](https://www.youtube.com/watch?v=9hkCDL8TKSQ) -
+ ### 20-Minute Demo of automated Ingest, Extract, Transform and Load [![image](https://github.com/user-attachments/assets/47026239-97f7-48a9-81b5-da2d4d070d9f)](https://www.youtube.com/watch?v=Wc_JE8YsT90) -
-### Other Videos and Tutorials -[Tutorials Wiki Site](https://github.com/ProdataSQL/DWA/wiki/8.-Tutorials) +### Other Videos and Tutorials +Tutorials Wiki Site: https://github.com/ProdataSQL/DWA/wiki/8.-Tutorials \ No newline at end of file diff --git a/Scripts/Setup-DWA.ipynb b/Scripts/Setup-DWA.ipynb index ad84b66..329c315 100644 --- a/Scripts/Setup-DWA.ipynb +++ b/Scripts/Setup-DWA.ipynb @@ -1,1759 +1 @@ - -{ - "cells": [ - { - "cell_type": "markdown", - "id": "9a9979c7-719f-4bf4-9e63-f8a6aaf6ac67", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# **Data Warehouse Automation (DWA) Setup in Microsoft Fabric**\n", - "- Modify the parameter values as needed.\n", - "- Create your connections on your workspace before running." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19abb929-7fea-4ae9-bb22-f9026054affe", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "tags": [ - "parameters" - ] - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-08-03T16:24:42.1399604Z", - "execution_start_time": "2025-08-03T16:24:41.7518405Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "5b143256-486a-4134-bb89-e8fd2ab1c05b", - "queued_time": "2025-08-03T16:24:32.5130133Z", - "session_id": "4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb", - "session_start_time": "2025-08-03T16:24:32.5140606Z", - "spark_pool": null, - "state": "finished", - "statement_id": 3, - "statement_ids": [ - 3 - ] - }, - "text/plain": [ - "StatementMeta(, 4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb, 3, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "lakehouse_name = \"LH\" # Lakehouse name e.g. FinanceLH, FinanceBronze, \n", - "warehouse_name = \"DW\" # Warehouse name e.g. FinanceDW, FinanceGold\n", - "metadata_db_name = \"Meta\" # SQL Database name to store metadata information for the framework\n", - "lakehouse_schema_enabled = True # If False then lakehouse and warehouse objects will need to be created manually\n", - "warehouse_case_sensitive = False # To make data warehouse case sensitive, default is False\n", - "deploy_aw = True # To deploy AdventureWorks files and objects set to True\n", - "branch= \"main\" # main or dev for experimental release\n", - "\n", - "# Create these connections on your workspace before running. https://github.com/ProdataSQL/DWA/wiki/Create-Cloud-Connection\n", - "fabricsqldb_connection_name = \"Meta-DWA-Training\" #Connection name for Fabric SQL Database shareable cloud\n", - "fabricdatapipelines_connection_name = \"Pipelines-DWA-Training\" #Connection name for Fabric Data Pipelines shareable cloud\n", - "pbi_connection_name =\"PBI-DWA-Training\" #Connection name for Power BI Semantic Models (used for refresh)\n" - ] - }, - { - "cell_type": "markdown", - "id": "0e0e17fa-939d-433c-87be-af7c49a0b07c", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### Import of needed libraries" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "ee02c50f-f044-4ad4-81c3-d1afaa40738e", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-08-03T16:25:17.4084423Z", - "execution_start_time": "2025-08-03T16:25:09.7066262Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "853b0e4a-bb86-4bb8-89c3-cf8623f282b7", - "queued_time": "2025-08-03T16:25:09.7054579Z", - "session_id": "4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 8, - "statement_ids": [ - 8 - ] - }, - "text/plain": [ - "StatementMeta(, 4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb, 8, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import struct\n", - "import sqlalchemy\n", - "from sqlalchemy.sql import text\n", - "from notebookutils import mssparkutils\n", - "import sempy.fabric as fabric\n", - "import sempy.fabric as sf\n", - "import base64\n", - "from azure.core.credentials import AccessToken\n", - "from azure.storage.filedatalake import DataLakeServiceClient\n", - "from azure.identity import DefaultAzureCredential\n", - "import os\n", - "import pyodbc\n", - "import shutil\n", - "from git import Repo\n", - "import requests\n", - "import json\n", - "import fnmatch\n", - "import time\n", - "from enum import Enum\n", - "from sqlalchemy.engine import Engine" - ] - }, - { - "cell_type": "markdown", - "id": "63fbf6f1-4603-4634-a9fc-222098ce0ca6", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "##### Setting up global parameters \n", - "Also Clone GIT repos" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "c1d0a5c9-1d97-4597-9dc1-c9aa6e7bd49a", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-08-03T16:25:26.9434809Z", - "execution_start_time": "2025-08-03T16:25:17.4105868Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "1b871056-2e40-4561-95e9-634789e25a06", - "queued_time": "2025-08-03T16:25:12.6467262Z", - "session_id": "4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 9, - "statement_ids": [ - 9 - ] - }, - "text/plain": [ - "StatementMeta(, 4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb, 9, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "WORKSPACE_ID = fabric.get_workspace_id()\n", - "WORKSPACE_NAME = fabric.resolve_workspace_name()\n", - "repo_url = \"https://github.com/ProdataSQL/DWA\"\n", - "repo_dir = \"DWA_repo\"\n", - "\n", - "BASE_URL = \"https://api.fabric.microsoft.com/v1\"\n", - "access_token = notebookutils.credentials.getToken('pbi')\n", - "\n", - "headers = {\n", - " \"Authorization\": f\"Bearer {access_token}\",\n", - " \"Content-Type\": \"application/json\"\n", - "}\n", - "\n", - "\n", - "#Clone Repo into local storage\n", - "if os.path.exists(repo_dir):\n", - " shutil.rmtree(repo_dir)\n", - "Repo.clone_from(url=repo_url, to_path=repo_dir,branch=branch, single_branch=True)\n", - "os.chdir(repo_dir)\n", - "\n", - "\n", - " \n" - ] - }, - { - "cell_type": "markdown", - "id": "ecfe6963-fe87-4597-93cb-9bbd40254978", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 1. Create Lakehouse" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "fa181b62-1c13-45fd-909d-c415eda39a1e", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-08-03T16:25:27.6921027Z", - "execution_start_time": "2025-08-03T16:25:26.9456577Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "79556a9c-b19d-4c47-9d39-2edad3511941", - "queued_time": "2025-08-03T16:25:16.7959238Z", - "session_id": "4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 10, - "statement_ids": [ - 10 - ] - }, - "text/plain": [ - "StatementMeta(, 4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb, 10, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Lakehouse LH already exists. No change was made.\n" - ] - } - ], - "source": [ - "lakehouse_url = f\"{BASE_URL}/workspaces/{WORKSPACE_ID}/lakehouses\"\n", - "payload = {\n", - " \"displayName\": f\"{lakehouse_name}\", \n", - " \"description\": \"A schema-enabled lakehouse.\",\n", - " \"creationPayload\": {\"enableSchemas\": f\"{lakehouse_schema_enabled}\"} \n", - "}\n", - "response = requests.post(lakehouse_url, headers=headers, json=payload)\n", - "\n", - "if response.status_code == 400:\n", - " print(f\"Lakehouse {lakehouse_name} already exists. No change was made.\")\n", - "elif response.status_code != 201:\n", - " raise RuntimeError(f\"Failed to create Lakehouse {lakehouse_name} : {response.status_code}, {response.text}\")" - ] - }, - { - "cell_type": "markdown", - "id": "e66b8de5-1aee-4278-ad8d-f3baf2a2554b", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 2. Create Metadata SQL Database\n", - "#### **Warning:** Verify the SQL DB creation before executing the next cell." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "786c758b-d46c-462c-a5ff-3a9bf6edf8b0", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-08-03T16:25:28.4825305Z", - "execution_start_time": "2025-08-03T16:25:27.6944232Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "802148b1-c435-464d-8473-1bf508be0a30", - "queued_time": "2025-08-03T16:25:20.2367187Z", - "session_id": "4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 11, - "statement_ids": [ - 11 - ] - }, - "text/plain": [ - "StatementMeta(, 4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb, 11, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SQL DB Meta already exists. No change was made.\n" - ] - } - ], - "source": [ - "payload = {\n", - " \"displayName\": f\"{metadata_db_name}\",\n", - " \"type\": \"SQLDatabase\",\n", - " \"description\": \"SQL Database to store metadata for Framework\"\n", - "}\n", - "\n", - "sqldb_url = f'{BASE_URL}/workspaces/{WORKSPACE_ID}/items'\n", - "response = requests.post(sqldb_url, headers=headers, json=payload)\n", - "\n", - "if response.status_code == 400:\n", - " print(f\"SQL DB {metadata_db_name} already exists. No change was made.\")\n", - "else:\n", - " start_time = time.time() \n", - " max_wait_time = 600 \n", - " check_interval = 60\n", - " \n", - " while time.time() - start_time < max_wait_time: #Sleep timer to wait for SQL DB creation\n", - " time.sleep(check_interval)\n", - " db_exists = fabric.resolve_item_id(metadata_db_name, \"SqlDatabase\", WORKSPACE_ID) \n", - "\n", - " if db_exists:\n", - " break\n", - " else: \n", - " raise RuntimeError(f\"Failed to create database {metadata_db_name} within the timeout period.\")\n", - " \n", - " if response.status_code not in [200, 201, 202]:\n", - " raise RuntimeError(f\"Failed to create database {metadata_db_name}: {response.status_code}, {response.text}\")" - ] - }, - { - "cell_type": "markdown", - "id": "177d070a-a9c4-4beb-8eac-7f2a40374907", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 3. Create Data Warehouse\n", - "#### **Warning:** Verify the Data Warehouse creation before executing the next cell." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "bcc75513-071f-4146-b283-9fddeaa22a6b", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-08-03T16:25:29.2198234Z", - "execution_start_time": "2025-08-03T16:25:28.4846689Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "1b7e4328-aac7-434c-a30f-83548d5a2cb2", - "queued_time": "2025-08-03T16:25:25.1158545Z", - "session_id": "4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 12, - "statement_ids": [ - 12 - ] - }, - "text/plain": [ - "StatementMeta(, 4ede4f56-0b0c-4b8c-b639-cdc6d836dfeb, 12, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Warehouse DW already exists. No change was made.\n" - ] - } - ], - "source": [ - "if warehouse_case_sensitive == True:\n", - " warehouse_collation = \"Latin1_General_100_BIN2_UTF8\"\n", - "else:\n", - " warehouse_collation = \"Latin1_General_100_CI_AS_KS_WS_SC_UTF8\"\n", - "\n", - "payload = {\n", - " \"displayName\": f\"{warehouse_name}\",\n", - " \"type\": \"warehouse\",\n", - " \"properties\": {\n", - " \"collation\": f\"{warehouse_collation}\" \n", - " }\n", - "}\n", - "\n", - "dw_url = f'{BASE_URL}/workspaces/{WORKSPACE_ID}/items'\n", - "response = requests.post(dw_url, headers=headers, json=payload)\n", - "\n", - "if response.status_code == 400:\n", - " print(f\"Warehouse {warehouse_name} already exists. No change was made.\")\n", - "else:\n", - " start_time = time.time() \n", - " max_wait_time = 600 \n", - " check_interval = 60\n", - " \n", - " while time.time() - start_time < max_wait_time: #Sleep timer to wait for SQL DB creation\n", - " time.sleep(check_interval)\n", - " dw_exists = fabric.resolve_item_id(warehouse_name, \"Warehouse\", WORKSPACE_ID) \n", - "\n", - " if dw_exists:\n", - " break\n", - " else: \n", - " raise RuntimeError(f\"Failed to create warehouse {warehouse_name} within the timeout period.\")\n", - " \n", - " if response.status_code not in [200, 201, 202]:\n", - " raise RuntimeError(f\"Failed to create warehouse {warehouse_name}: {response.status_code}, {response.text}\") " - ] - }, - { - "cell_type": "markdown", - "id": "954779cc-808c-4fb1-bdbd-94360f997005", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 4. Upload Notebook Templates" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "bc8a1f0f-c7c9-4a10-96d9-cbe4ebf02181", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-07-14T19:07:55.427008Z", - "execution_start_time": "2025-07-14T19:05:48.7227784Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "4243a159-74de-4e51-989c-84543b0809cb", - "queued_time": "2025-07-14T19:03:17.9649949Z", - "session_id": "929963e1-609d-4bbe-8728-c790daed3f81", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 9, - "statement_ids": [ - 9 - ] - }, - "text/plain": [ - "StatementMeta(, 929963e1-609d-4bbe-8728-c790daed3f81, 9, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Extract-Excel created.\n", - "Extract-SP-Excel created.\n", - "Extract-Parquet-CDC created.\n", - "Extract-O365-API created.\n", - "Extract-CSV-Pandas created.\n", - "Extract-CSV created.\n", - "Extract-XML created.\n", - "Extract-JSON created.\n", - "Export-SQL created.\n", - "Export-CSV created.\n", - "Export-SP created.\n", - "Export-Excel created.\n", - "Export-Parquet created.\n", - "Refresh-Fabric created.\n", - "Copy-Blob created.\n", - "Ingest-SFTP created.\n", - "Ingest-SP created.\n", - "MD-Sync Python created.\n", - "MD-Sync created.\n", - "SQL-Connection-Shared-Functions created.\n", - "SharePoint-Shared-Functions created.\n", - "Transform-LH-DuckDB created.\n", - "ArchiveFiles created.\n", - "Extract-Dictionary created.\n", - "Extract-Fabric-Logs created.\n", - "Extract-Artefacts created.\n" - ] - } - ], - "source": [ - "import sempy.fabric as sf\n", - "import re\n", - "lh_artifact_id = fabric.resolve_item_id(lakehouse_name, \"Lakehouse\", WORKSPACE_ID)\n", - "\n", - "def get_notebooks(directory, pattern):\n", - " notebook_files = []\n", - " for root, dirs, files in os.walk(directory):\n", - " for file in fnmatch.filter(files, pattern):\n", - " notebook_files.append(os.path.join(root, file))\n", - " return notebook_files\n", - "\n", - "import json\n", - "\n", - "def parse_python_to_cells(py_content):\n", - " cells = []\n", - " current_cell_lines = []\n", - " cell_type = \"code\"\n", - " is_param_cell = False\n", - " in_metadata_block = False\n", - " skipped_first_comment = False\n", - " metadata_lines = []\n", - " is_default_lakehouse = False \n", - "\n", - " def add_cell():\n", - " if not current_cell_lines:\n", - " return\n", - " metadata = {}\n", - " if is_param_cell:\n", - " metadata = {\n", - " \"microsoft\": {\n", - " \"language\": \"python\",\n", - " \"language_group\": \"synapse_pyspark\"\n", - " },\n", - " \"tags\": [\"parameters\"],\n", - " }\n", - " cells.append({\n", - " \"cell_type\": cell_type,\n", - " \"metadata\": metadata,\n", - " \"source\": current_cell_lines.copy(),\n", - " \"execution_count\": None,\n", - " \"outputs\": [] if cell_type == \"code\" else None\n", - " })\n", - " current_cell_lines.clear()\n", - "\n", - " for line in py_content.splitlines(keepends=True):\n", - " stripped = line.strip()\n", - " if not skipped_first_comment and stripped.startswith(\"#\"):\n", - " skipped_first_comment = True\n", - " continue\n", - " if stripped.startswith(\"# META\"):\n", - " in_metadata_block = True\n", - " metadata_lines.append(stripped[6:] if stripped.startswith(\"# META \") else \"\")\n", - " continue\n", - " elif in_metadata_block:\n", - " if not stripped.startswith(\"#\") or stripped == \"\":\n", - " in_metadata_block = False\n", - " metadata_str = \"\\n\".join(metadata_lines)\n", - " try:\n", - " meta_json = json.loads(metadata_str)\n", - " dependencies = meta_json.get(\"dependencies\", {})\n", - " lakehouse = dependencies.get(\"lakehouse\", {})\n", - " if lakehouse.get(\"default_lakehouse_name\"):\n", - " is_default_lakehouse = True\n", - " except json.JSONDecodeError:\n", - " pass\n", - " metadata_lines.clear()\n", - " else:\n", - " metadata_lines.append(stripped[6:] if stripped.startswith(\"# META \") else \"\")\n", - " continue\n", - " if \"# PARAMETERS CELL\" in stripped:\n", - " add_cell()\n", - " cell_type = \"code\"\n", - " is_param_cell = True\n", - " continue\n", - " elif \"# CELL\" in stripped:\n", - " add_cell()\n", - " cell_type = \"code\"\n", - " is_param_cell = False\n", - " continue\n", - " elif \"# MARKDOWN\" in stripped or stripped.startswith(\"# %% [markdown]\"):\n", - " add_cell()\n", - " cell_type = \"markdown\"\n", - " is_param_cell = False\n", - " continue\n", - " if stripped:\n", - " if cell_type == \"markdown\" and stripped.startswith(\"#\"):\n", - " cleaned_line = re.sub(r'^#\\s?(.*)', r'\\1', stripped) + \"\\n\"\n", - " current_cell_lines.append(cleaned_line)\n", - " else:\n", - " current_cell_lines.append(line)\n", - " add_cell()\n", - " return cells, is_default_lakehouse\n", - "\n", - "def py_to_notebook(py_content, notebook_name):\n", - " notebook_cells, is_default_lakehouse = parse_python_to_cells(py_content)\n", - " metadata = {\n", - " \"language_info\": {\n", - " \"name\": \"python\",\n", - " \"language_group\": \"synapse_pyspark\"\n", - " }\n", - " }\n", - " if is_default_lakehouse:\n", - " metadata[\"dependencies\"] = {\n", - " \"lakehouse\": {\n", - " \"default_lakehouse\": lh_artifact_id,\n", - " \"default_lakehouse_name\": lakehouse_name,\n", - " \"default_lakehouse_workspace_id\": WORKSPACE_ID\n", - " }\n", - " }\n", - " notebook_content = {\n", - " \"nbformat\": 4,\n", - " \"nbformat_minor\": 5,\n", - " \"cells\": notebook_cells,\n", - " \"metadata\": metadata\n", - " }\n", - " return notebook_content\n", - "\n", - "def create_folder(folder_name: str, parent_folder_id: str=None, workspace_id: str = None):\n", - " if not workspace_id:\n", - " workspace_id = sf.get_notebook_workspace_id()\n", - " client = fabric.FabricRestClient()\n", - " response = client.get( f\"v1/workspaces/{workspace_id}/folders\").json()\n", - " folders = response.get(\"value\", [])\n", - " folder_id=next((f[\"id\"] for f in folders if f[\"displayName\"].lower() == folder_name.lower()), None)\n", - " if folder_id:\n", - " return folder_id\n", - " else:\n", - " client = fabric.FabricRestClient()\n", - " if parent_folder_id:\n", - " json={ \n", - " \"displayName\": folder_name,\n", - " \"parentFolderId\": parent_folder_id\n", - " \n", - " }\n", - " else:\n", - " json={ \n", - " \"displayName\": folder_name,\n", - " } \n", - " response = client.post(f\"v1/workspaces/{workspace_id}/folders\",json=json).json()\n", - " return response['id']\n", - "\n", - "def upload_notebook(notebook_name, notebook_content, folder_id):\n", - " notebook_json = json.dumps(notebook_content)\n", - " notebook_base64 = base64.b64encode(notebook_json.encode('utf-8')).decode('utf-8')\n", - " notebook_url = f\"{BASE_URL}/workspaces/{WORKSPACE_ID}/notebooks\"\n", - " payload = {\n", - " \"displayName\": notebook_name,\n", - " \"type\":\"Notebook\",\n", - " \"folderId\": folder_id,\n", - " \"definition\": {\n", - " \"format\": \"ipynb\",\n", - " \"parts\": [\n", - " {\n", - " \"path\": \"artifact.content.ipynb\",\n", - " \"payload\": notebook_base64,\n", - " \"payloadType\": \"InlineBase64\"\n", - " }\n", - " ]\n", - " }\n", - " }\n", - " fabric_response = requests.post(\n", - " notebook_url,\n", - " headers=headers,\n", - " data=json.dumps(payload)\n", - " )\n", - " if fabric_response.status_code == 400:\n", - " print(f\"The {notebook_name} already exists. No changes were made\")\n", - " elif fabric_response.status_code not in [200,201,202]:\n", - " raise RuntimeError(f\"Failed to upload {notebook_name}: {fabric_response.status_code} - {fabric_response.text}\")\n", - " else:\n", - " print(f\"{notebook_name} created.\")\n", - "\n", - "directory = \"Workspaces/DWA/\"\n", - "pattern = \"*.py\"\n", - "notebook_files = get_notebooks(directory, pattern)\n", - "folder_cache = {}\n", - "i=0\n", - "\n", - "for notebook_file in notebook_files:\n", - " i+=1\n", - " parent_folder_id=None\n", - " directory, file_name = os.path.split(notebook_file)\n", - " path = \"/\".join( directory.split(\"/\")[2:-1])\n", - " if path in folder_cache:\n", - " parent_folder_id=folder_cache[path]\n", - " if not parent_folder_id:\n", - " for folder in path.split('/'):\n", - " parent_folder_id=create_folder(folder, parent_folder_id)\n", - " folder_cache[path] = parent_folder_id\n", - " base_name = os.path.basename(directory)\n", - " if base_name.endswith('.Notebook'):\n", - " notebook_name = base_name.split('.')[0]\n", - " with open(notebook_file, 'r') as file:\n", - " py_content = file.read()\n", - " notebook_content = py_to_notebook(py_content, notebook_name)\n", - " upload_notebook(notebook_name, notebook_content, parent_folder_id)\n", - " " - ] - }, - { - "cell_type": "markdown", - "id": "cb600388-b06e-4f66-bebf-e843454f14b6", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 5. Create Metadata and Data Warehouse SQL Objects" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "de3b1a99-895a-4e74-8a0a-32132f617a34", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-07-14T19:08:01.4628139Z", - "execution_start_time": "2025-07-14T19:07:55.4294553Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "f68dafca-6afb-4543-9fbd-6046fee70178", - "queued_time": "2025-07-14T19:03:18.0395327Z", - "session_id": "929963e1-609d-4bbe-8728-c790daed3f81", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 10, - "statement_ids": [ - 10 - ] - }, - "text/plain": [ - "StatementMeta(, 929963e1-609d-4bbe-8728-c790daed3f81, 10, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "META_DB_NAME = metadata_db_name\n", - "DW_NAME = warehouse_name\n", - "original_dir = os.getcwd()\n", - "engine_pool = {}\n", - "\n", - "class DBType(Enum):\n", - " SQLDatabase = \"SQLDatabases\"\n", - " Warehouse = \"Warehouses\"\n", - "\n", - " def get_connection_string(self) -> str: \n", - " display_name = DW_NAME if self == DBType.Warehouse else META_DB_NAME\n", - " client = fabric.FabricRestClient()\n", - " endpoint = f\"/v1/workspaces/{WORKSPACE_ID}/{self.value}\"\n", - " databases = client.get(endpoint).json()\n", - "\n", - " selected_database = next((db for db in databases.get(\"value\", []) if db.get(\"displayName\") == display_name), None)\n", - " if not selected_database:\n", - " raise ValueError(f\"No {self.value} with displayName '{display_name}' found.\")\n", - " \n", - " server = selected_database['properties'].get('serverFqdn') or selected_database['properties'].get('connectionString')\n", - " database = selected_database['properties'].get('databaseName', warehouse_name)\n", - " \n", - " return f\"Driver={{ODBC Driver 18 for SQL Server}};Server={server};database={database};LongAsMax=YES\"\n", - "\n", - " def get_engine(self) -> Engine:\n", - " if self not in engine_pool:\n", - " token = mssparkutils.credentials.getToken('https://analysis.windows.net/powerbi/api').encode(\"UTF-16-LE\")\n", - " token_struct = struct.pack(f' list:\n", - " if not os.path.exists(file_path):\n", - " return []\n", - " \n", - " with open(file_path, \"r\") as f:\n", - " return [line.strip() for line in f if line.strip()]\n", - "\n", - "def process_sql_script(db_type: DBType, script_path: str):\n", - " if not os.path.exists(script_path):\n", - " return\n", - " \n", - " filename = os.path.basename(script_path)\n", - " if \"prodata.ie\" in filename:\n", - " print(f\"Skipping {filename} because it ends with 'prodata.ie'\")\n", - " return\n", - "\n", - " with open(script_path, \"r\") as file:\n", - " script = file.read()\n", - "\n", - " statements = script.split(\"\\nGO\\n\")\n", - " \n", - " object_name = script_path.split(\"/\")[-1].split(\".\")[0]\n", - " db_name = script_path.split(\"/\")[0].replace(\".\", \" \")\n", - " schema_name = script_path.split(\"/\")[1]\n", - "\n", - " if \"Tables/\" in script_path:\n", - " check_query = f\"SELECT COUNT(*) FROM sys.tables WHERE name = '{object_name}' AND schema_id = SCHEMA_ID('{schema_name}')\"\n", - " elif \"Views/\" in script_path:\n", - " check_query = f\"SELECT COUNT(*) FROM sys.views WHERE name = '{object_name}' AND schema_id = SCHEMA_ID('{schema_name}')\"\n", - " elif \"StoredProcedures/\" in script_path:\n", - " check_query = f\"SELECT COUNT(*) FROM sys.procedures WHERE name = '{object_name}' AND schema_id = SCHEMA_ID('{schema_name}')\"\n", - " elif \"Security/\" in script_path: \n", - " check_query = f\"SELECT COUNT(*) FROM sys.schemas WHERE name = '{object_name}'\"\n", - " else:\n", - " check_query = f\"SELECT COUNT(*) FROM sys.schemas WHERE name = '{schema_name}'\"\n", - " \n", - " with db_type.get_engine().connect() as conn:\n", - " object_exists = conn.execute(text(check_query)).scalar() > 0\n", - "\n", - " if not object_exists:\n", - " try:\n", - " for statement in filter(None, map(str.strip, statements)):\n", - " conn.execute(text(statement))\n", - " conn.commit()\n", - " except Exception as e:\n", - " conn.rollback()\n", - " print(f\"Failed to create {schema_name}.{object_name}: {e}\")\n", - " raise\n", - " else:\n", - " print(f\"The object {schema_name}.{object_name} already exists in {db_name}. No changes made.\")\n", - "\n", - "def iterate_sql_objects(db_type: DBType, object_type: str, objects_created: set):\n", - " base_path = \"Meta.SQLDatabase\" if db_type == DBType.SQLDatabase else \"DW.Warehouse\"\n", - " visited_dirs = set()\n", - " \n", - " for root, _, files in os.walk(base_path):\n", - " if root in visited_dirs:\n", - " continue\n", - " visited_dirs.add(root)\n", - " \n", - " for file in filter(lambda f: f.endswith(\".sql\"), files):\n", - " script_path = os.path.join(root, file).lstrip(\"./\")\n", - " \n", - " if script_path in objects_created:\n", - " continue\n", - "\n", - " parts = script_path.split(os.sep)\n", - " if object_type == \"Schema\":\n", - " if any(folder in parts for folder in [\"Tables\", \"Views\", \"StoredProcedures\"]):\n", - " continue \n", - " else:\n", - " if object_type not in parts:\n", - " continue \n", - "\n", - " process_sql_script(db_type, script_path)\n", - " objects_created.add(script_path)\n", - "\n", - "def process_sql_objects(db_type: DBType, obj_type: str = None):\n", - " table_orders_cache = {}\n", - " has_changed_directory = False \n", - " current_dir = os.path.abspath(os.getcwd())\n", - "\n", - " table_order_files = {\n", - " DBType.SQLDatabase: os.path.join(current_dir, \"Setup\", \"Files\", \"SQLDatabase\", \"MetaTableOrder.txt\"),\n", - " DBType.Warehouse: os.path.join(current_dir, \"Setup\", \"Files\", \"Warehouse\", \"DWTableOrder.txt\")\n", - " }\n", - " \n", - " for db_type_key, file_path in table_order_files.items(): \n", - " if os.path.exists(file_path):\n", - " table_orders_cache[db_type_key] = read_table_order(file_path)\n", - "\n", - " table_order = table_orders_cache.get(db_type, [])\n", - "\n", - " if not has_changed_directory:\n", - " workspaces_dwa_path = os.path.join(current_dir, \"Workspaces\", \"DWA\")\n", - " \n", - " if os.path.exists(workspaces_dwa_path):\n", - " os.chdir(workspaces_dwa_path)\n", - " has_changed_directory = True \n", - " \n", - " objects_created = set()\n", - " base_path = \"Meta.SQLDatabase\" if db_type == DBType.SQLDatabase else \"DW.Warehouse\"\n", - " \n", - " for table in table_order:\n", - " schema = table.split(\".\", maxsplit=1)[0].strip(\"[]\")\n", - " table_name = table.split(\".\", maxsplit=1)[1].strip(\"[]\")\n", - " script_path = f\"{base_path}/{schema}/{obj_type}/{table_name}.sql\"\n", - " \n", - " if os.path.exists(script_path) and script_path not in objects_created:\n", - " process_sql_script(db_type, script_path)\n", - " objects_created.add(script_path)\n", - " \n", - " iterate_sql_objects(db_type, obj_type, objects_created)\n", - " os.chdir(original_dir)\n", - "\n", - "#Create meta databse objects\n", - "process_sql_objects(DBType.SQLDatabase, \"Security\")\n", - "process_sql_objects(DBType.SQLDatabase, \"Tables\")\n", - "process_sql_objects(DBType.SQLDatabase, \"Views\")\n", - "process_sql_objects(DBType.SQLDatabase, \"StoredProcedures\") \n", - "\n", - "#Create warehouse objects\n", - "process_sql_objects(DBType.Warehouse, \"Schema\")\n", - "process_sql_objects(DBType.Warehouse, \"StoredProcedures\")" - ] - }, - { - "cell_type": "markdown", - "id": "da333f14-9428-48c3-aead-65c5d0bf1117", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 6. Upload Data Pipelines" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "eee6400d-7f6f-4b5f-a5a0-3b5433065a31", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-07-14T19:08:03.7752646Z", - "execution_start_time": "2025-07-14T19:08:01.4651434Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "9eeee6f1-19df-4a76-85e6-7e54b1f76d2b", - "queued_time": "2025-07-14T19:03:18.141491Z", - "session_id": "929963e1-609d-4bbe-8728-c790daed3f81", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 11, - "statement_ids": [ - 11 - ] - }, - "text/plain": [ - "StatementMeta(, 929963e1-609d-4bbe-8728-c790daed3f81, 11, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "# Stores new references for updating old data pipelines connections \n", - "lh_artifact_id = fabric.resolve_item_id(lakehouse_name, \"Lakehouse\", WORKSPACE_ID) #Lakehouse ID\n", - "meta_artifact_id = fabric.resolve_item_id(metadata_db_name, \"SqlDatabase\", WORKSPACE_ID) #SQL DB ID and Connection String\n", - "sqldb_url = f'{BASE_URL}/workspaces/{WORKSPACE_ID}/SQLDatabases/{meta_artifact_id}'\n", - "response = requests.get(sqldb_url, headers=headers)\n", - "\n", - "meta_endpoint = response.json().get(\"properties\", {}).get(\"serverFqdn\")\n", - "meta_databasename = response.json().get(\"properties\", {}).get(\"databaseName\")\n", - "\n", - "dw_artifact_id = fabric.resolve_item_id(warehouse_name, \"Warehouse\", WORKSPACE_ID) #Warehouse ID and Connection String\n", - "\n", - "dw_url = f'{BASE_URL}/workspaces/{WORKSPACE_ID}/warehouses/{dw_artifact_id}'\n", - "response = requests.get(dw_url, headers=headers)\n", - "\n", - "dw_endpoint = response.json().get(\"properties\", {}).get(\"connectionString\")\n", - "\n", - "def create_folder(folder_name: str, parent_folder_id: str=None, workspace_id: str = None):\n", - " if not workspace_id:\n", - " workspace_id = sf.get_notebook_workspace_id()\n", - " client = fabric.FabricRestClient()\n", - " response = client.get( f\"v1/workspaces/{workspace_id}/folders\").json()\n", - " folders = response.get(\"value\", [])\n", - " folder_id=next((f[\"id\"] for f in folders if f[\"displayName\"].lower() == folder_name.lower()), None)\n", - " if folder_id:\n", - " return folder_id\n", - " else:\n", - " client = fabric.FabricRestClient()\n", - " if parent_folder_id:\n", - " json={ \n", - " \"displayName\": folder_name,\n", - " \"parentFolderId\": parent_folder_id\n", - " \n", - " }\n", - " else:\n", - " json={ \n", - " \"displayName\": folder_name,\n", - " } \n", - " response = client.post(f\"v1/workspaces/{workspace_id}/folders\",json=json).json()\n", - " return response['id']\n", - "\n", - "def get_connection_id_by_name(connection_name: str) -> str:\n", - " url = f\"{BASE_URL}/connections\"\n", - "\n", - " response = requests.get(url, headers=headers)\n", - " response.raise_for_status()\n", - "\n", - " connections = response.json().get(\"value\", [])\n", - "\n", - " for conn in connections:\n", - " if conn.get(\"displayName\") == connection_name:\n", - " return conn.get(\"id\")\n", - "\n", - " raise ValueError(f\"Connection with name '{connection_name}' not found in workspace {WORKSPACE_ID}.\")\n", - "\n", - "fabricsqldb_connection_id = get_connection_id_by_name(fabricsqldb_connection_name)\n", - "fabricdatapipelines_connection_id = get_connection_id_by_name(fabricdatapipelines_connection_name)\n", - "pbi_connection_id = get_connection_id_by_name(pbi_connection_name)\n", - "\n", - "current_dir = os.path.abspath(os.getcwd())\n", - "file_path = os.path.join(current_dir, \"Setup\", \"Files\", \"Workspace\", \"AWDataPipelines.txt\")\n", - "if os.path.exists(file_path):\n", - " with open(file_path, \"r\") as f:\n", - " aw_pipelines = [line.strip() for line in f if line.strip()]\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "19d74a8c-843a-4ba4-bcc8-f55a28ebf92f", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-07-14T19:09:25.5796781Z", - "execution_start_time": "2025-07-14T19:08:03.7779434Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "8b4ac3f0-b004-4a30-abf9-29c3d2a5fdbf", - "queued_time": "2025-07-14T19:03:18.2318772Z", - "session_id": "929963e1-609d-4bbe-8728-c790daed3f81", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 12, - "statement_ids": [ - 12 - ] - }, - "text/plain": [ - "StatementMeta(, 929963e1-609d-4bbe-8728-c790daed3f81, 12, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Creating pipeline Pipeline-Worker\n", - "Creating pipeline Environment-Variables\n", - "Environment-Variables created and uploaded.\n", - "Pipeline-Worker created and uploaded.\n", - "Creating pipeline Extract-SQL\n", - "Extract-SQL created and uploaded.\n", - "Creating pipeline Refresh-PBI\n", - "Refresh-PBI created and uploaded.\n", - "Skipping Copy-SFTP-Prodata as requires SFTP Connection. Install Manually if required.\n", - "Creating pipeline Ingest-SQL-CDC\n", - "Ingest-SQL-CDC created and uploaded.\n", - "Creating pipeline Pipeline-Controller\n", - "Pipeline-Controller created and uploaded.\n", - "Creating pipeline Backup_ConfigSchema\n", - "Backup_ConfigSchema created and uploaded.\n", - "Creating pipeline Restore_ConfigSchema\n", - "Restore_ConfigSchema created and uploaded.\n", - "Creating pipeline TR_ConfigBackup\n", - "TR_ConfigBackup created and uploaded.\n", - "Creating pipeline TR_Ops\n", - "TR_Ops created and uploaded.\n", - "Creating pipeline TR_AdventureWorks\n", - "TR_AdventureWorks created and uploaded.\n" - ] - } - ], - "source": [ - "access_token = mssparkutils.credentials.getToken('pbi') #Rebuild access token \n", - "\n", - "headers = {\n", - " \"Authorization\": f\"Bearer {access_token}\",\n", - " \"Content-Type\": \"application/json\"\n", - "}\n", - "\n", - "env_variables_map = {\n", - " \"5941a6c0-8c98-4d79-b065-a3789e9e0960\": WORKSPACE_ID,\n", - " \"fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.database.fabric.microsoft.com\": meta_endpoint,\n", - " \"Meta-fe70c606-af27-4f64-973a-2be877526212\": meta_databasename,\n", - " \"fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com\": dw_endpoint,\n", - " \"DW\": warehouse_name,\n", - " \"d58f4f2d-59d7-406d-ae4c-898354a6a75f\" : lh_artifact_id,\n", - " \"fe70c606-af27-4f64-973a-2be877526212\" : meta_artifact_id\n", - "}\n", - "\n", - "class DataPipeline:\n", - " pass\n", - "\n", - "class DataPipeline:\n", - " name: str\n", - " git_id: str\n", - " real_id: str\n", - " raw_definition: str\n", - " definition: object\n", - " pipeline_created: bool = False\n", - " folder_id: str\n", - "\n", - " def __init__(self, name, git_id, raw_definition, definition, folder_id):\n", - " self.name = name\n", - " self.git_id = git_id\n", - " self.definition = definition\n", - " self.raw_definition = raw_definition\n", - " self.folder_id=folder_id\n", - " def __hash__(self) -> int:\n", - " return hash(self.git_id)\n", - " def __eq__(self, other):\n", - " return self.git_id == other.git_id\n", - "\n", - " def __str__(self):\n", - " return f\"DataPipeline: {self.name}\"\n", - "\n", - " def __repr__(self):\n", - " return f\"DataPipeline: {self.name}\"\n", - "\n", - " def is_created(self) -> bool:\n", - " return self in pipelines_created or self.pipeline_created\n", - "\n", - " def create(self, pipelines):\n", - " global pipelines_created\n", - "\n", - " if self.name in aw_pipelines:\n", - " print(f\"Skipping {self.name} as requires SFTP Connection. Install Manually if required.\")\n", - " return\n", - "\n", - " if self.is_created():\n", - " print(f\"{self.name} already created\")\n", - " return\n", - "\n", - " print(f\"Creating pipeline {self.name}\")\n", - "\n", - " child_pipelines: list[DataPipeline] = self.get_child_pipelines(pipelines)\n", - "\n", - " for child_pipeline in child_pipelines:\n", - " if child_pipeline.is_created():\n", - " continue\n", - " child_pipeline.create(pipelines)\n", - "\n", - " self.update_activity_references()\n", - " self.upload_pipeline()\n", - "\n", - " print(f\"{self.name} created and uploaded.\")\n", - "\n", - " def get_child_pipelines(self, pipelines : list[DataPipeline]) -> list[DataPipeline]:\n", - " self.child_pipelines: set[DataPipeline] = set()\n", - "\n", - " def find_pipelines(activities : list [object]):\n", - " for activity in activities:\n", - " activity_type = activity.get(\"type\")\n", - " type_properties = activity.get(\"typeProperties\", {})\n", - "\n", - " if activity_type == \"ExecutePipeline\":\n", - " reference_name = type_properties.get(\"pipeline\", {}).get(\n", - " \"referenceName\"\n", - " )\n", - " matched_pipeline = next(\n", - " (x for x in pipelines if x.git_id == reference_name), None\n", - " )\n", - " if matched_pipeline:\n", - " self.child_pipelines.add(matched_pipeline)\n", - "\n", - " elif activity_type == \"InvokePipeline\":\n", - " pipeline_id = type_properties.get(\"pipelineId\")\n", - " matched_pipeline = next(\n", - " (x for x in pipelines if x.git_id == pipeline_id), None\n", - " )\n", - " if matched_pipeline:\n", - " self.child_pipelines.add(matched_pipeline)\n", - " \n", - " else:\n", - " for key in [\"activities\", \"ifTrueActivities\", \"ifFalseActivities\"]:\n", - " if key in type_properties:\n", - " find_pipelines(type_properties[key])\n", - " return self.child_pipelines\n", - "\n", - " return find_pipelines(\n", - " self.definition.get(\"properties\", {}).get(\"activities\", [])\n", - " )\n", - "\n", - " def update_activity_references(self):\n", - " global pipelines_created\n", - "\n", - " def fetch_trident_notebooks():\n", - " notebooks_url = f\"{BASE_URL}/workspaces/{WORKSPACE_ID}/notebooks\"\n", - " response = requests.get(notebooks_url, headers=headers)\n", - " response.raise_for_status()\n", - " return {notebook[\"displayName\"]: notebook[\"id\"] for notebook in response.json().get(\"value\", [])}\n", - "\n", - " trident_notebooks = fetch_trident_notebooks()\n", - "\n", - " new_definition = self.raw_definition\n", - " for pipeline in pipelines_created: \n", - " new_definition = new_definition.replace(pipeline.git_id, pipeline.real_id)\n", - " for source_id, target_id in env_variables_map.items(): \n", - " new_definition = new_definition.replace(source_id, target_id)\n", - "\n", - " definition_json = json.loads(new_definition)\n", - "\n", - " def replace_values(obj):\n", - " if isinstance(obj, dict):\n", - " for key, value in obj.items():\n", - " if key == \"type\" and isinstance(value, str):\n", - " if \"Lakehouse\" in value:\n", - " if \"typeProperties\" in obj and isinstance(obj[\"typeProperties\"], dict) :\n", - " if \"artifactId\" in obj[\"typeProperties\"] and not obj[\"typeProperties\"][\"artifactId\"].startswith(\"@\"):\n", - " obj[\"typeProperties\"][\"artifactId\"] = lh_artifact_id \n", - " \n", - " elif \"DataWarehouse\" in value:\n", - " if \"typeProperties\" in obj and isinstance(obj[\"typeProperties\"], dict):\n", - " if \"artifactId\" in obj[\"typeProperties\"] and not obj[\"typeProperties\"][\"artifactId\"].startswith(\"@\"):\n", - " obj[\"typeProperties\"][\"artifactId\"] = dw_artifact_id \n", - " if \"endpoint\" in obj and not obj[\"endpoint\"].startswith(\"@\"):\n", - " obj[\"endpoint\"] = dw_endpoint \n", - "\n", - " elif \"FabricSqlDatabase\" in value:\n", - " if \"typeProperties\" in obj and isinstance(obj[\"typeProperties\"], dict):\n", - " if \"artifactId\" in obj[\"typeProperties\"] and not obj[\"typeProperties\"][\"artifactId\"].startswith(\"@\"):\n", - " obj[\"typeProperties\"][\"artifactId\"] = meta_artifact_id \n", - " if \"endpoint\" in obj and not obj[\"endpoint\"].startswith(\"@\"):\n", - " obj[\"endpoint\"] = meta_endpoint \n", - " if \"externalReferences\" in obj and isinstance(obj[\"externalReferences\"], dict):\n", - " obj[\"externalReferences\"][\"connection\"] = fabricsqldb_connection_id\n", - "\n", - " elif \"InvokePipeline\" in value:\n", - " if \"externalReferences\" in obj and isinstance(obj[\"externalReferences\"], dict):\n", - " obj[\"externalReferences\"][\"connection\"] = fabricdatapipelines_connection_id\n", - "\n", - " elif \"PBISemanticModelRefresh\" in value:\n", - " if \"externalReferences\" in obj and isinstance(obj[\"externalReferences\"], dict):\n", - " obj[\"externalReferences\"][\"connection\"] = pbi_connection_id\n", - " \n", - " elif \"TridentNotebook\" in value:\n", - " notebook_name = obj.get(\"name\")\n", - " if notebook_name in trident_notebooks:\n", - " obj[\"typeProperties\"][\"notebookId\"] = trident_notebooks[notebook_name]\n", - "\n", - " elif key == \"workspaceId\" and isinstance(value, str) and not value.startswith(\"@\"):\n", - " obj[key] = WORKSPACE_ID \n", - "\n", - " replace_values(value)\n", - "\n", - " elif isinstance(obj, list):\n", - " for item in obj:\n", - " replace_values(item)\n", - "\n", - " replace_values(definition_json)\n", - "\n", - " self.raw_definition = json.dumps(definition_json, indent=4)\n", - "\n", - " def upload_pipeline(self):\n", - " global pipelines_created\n", - " pipelines_url = f\"{BASE_URL}/workspaces/{WORKSPACE_ID}/dataPipelines\"\n", - " pipeline_b64 = base64.b64encode(self.raw_definition.encode()).decode()\n", - "\n", - " payload = {\n", - " \"displayName\": self.name,\n", - " \"folderId\" : self.folder_id,\n", - " \"definition\": {\n", - " \"parts\": [\n", - " {\n", - " \"path\": \"pipeline.content.json\",\n", - " \"payload\": pipeline_b64,\n", - " \"payloadType\": \"InlineBase64\",\n", - " }\n", - " ]\n", - " },\n", - " }\n", - " create_pipeline_request = requests.post(pipelines_url, headers=headers, data=json.dumps(payload))\n", - " if not create_pipeline_request.ok:\n", - " print(create_pipeline_request.json())\n", - " raise RuntimeError(f\"Failed to create pipeline {self.name}. ({create_pipeline_request.text})\")\n", - " self.real_id = create_pipeline_request.json().get(\"id\")\n", - " self.pipeline_created = True\n", - " pipelines_created.append(self)\n", - " \n", - "pipelines: list[DataPipeline] = []\n", - "folder_cache = {}\n", - "for path, dirs, files in os.walk(\".\"):\n", - " if not path.endswith(\".DataPipeline\"):\n", - " continue\n", - " folder_id=None\n", - " folder_path = \"/\".join( path.split(\"/\")[3:-1])\n", - " if folder_path in folder_cache:\n", - " folder_id=folder_cache[folder_path]\n", - " else:\n", - " for folder in folder_path.split('/'):\n", - " folder_id=create_folder(folder, folder_id)\n", - " folder_cache[folder_path] = folder_id\n", - " pipeline_name = path.strip(\".\").split(\".\")[0].replace(\"\\\\\", \"/\").split(\"/\")[-1]\n", - " platform = json.load(open(os.path.join(path, \".platform\")))\n", - " raw_definition = open(os.path.join(path, \"pipeline-content.json\")).read()\n", - " definition = json.loads(raw_definition)\n", - " pipeline_id = platform[\"config\"][\"logicalId\"]\n", - "\n", - " data_pipeline = DataPipeline(pipeline_name, pipeline_id, raw_definition, definition, folder_id)\n", - " pipelines.append(data_pipeline)\n", - "pipelines_url = f\"{BASE_URL}/workspaces/{WORKSPACE_ID}/dataPipelines\"\n", - "response = requests.get(pipelines_url, headers=headers)\n", - "response.raise_for_status()\n", - "\n", - "pipelines_created: list[DataPipeline] = []\n", - "\n", - "pipelines_existing: list[(str, str)] = [\n", - " (pipeline[\"displayName\"], pipeline[\"id\"])\n", - " for pipeline in response.json().get(\"value\", [])\n", - "]\n", - "\n", - "for pipeline_name, pipeline_id in pipelines_existing:\n", - " pipeline = next((x for x in pipelines if x.name == pipeline_name), None)\n", - " if not pipeline:\n", - " continue\n", - "\n", - " pipeline.real_id = pipeline_id\n", - " pipeline.pipeline_created = True\n", - " pipelines_created.append(pipeline)\n", - "i=0\n", - "\n", - "for pipeline in pipelines:\n", - " i+=1\n", - " if pipeline in pipelines_created or pipeline.pipeline_created:\n", - " continue\n", - " #if i>1 : break\n", - " pipeline.create(pipelines)\n", - "\n", - " " - ] - }, - { - "cell_type": "markdown", - "id": "acc1cdcb-2625-449d-9a11-c903889ec341", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 7. Create objects for Adventure Works when deploy_aw is set to True" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "350e775b-7c60-4d73-a9d7-d6bd77673d28", - "metadata": { - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-07-14T19:11:34.6981076Z", - "execution_start_time": "2025-07-14T19:09:25.581636Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "c75098f4-7b9a-489f-a2f3-b648f08947ca", - "queued_time": "2025-07-14T19:03:18.3015063Z", - "session_id": "929963e1-609d-4bbe-8728-c790daed3f81", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 13, - "statement_ids": [ - 13 - ] - }, - "text/plain": [ - "StatementMeta(, 929963e1-609d-4bbe-8728-c790daed3f81, 13, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "AdventureWorks Samples Deployed.\n" - ] - } - ], - "source": [ - "if deploy_aw == False:\n", - " mssparkutils.notebook.exit(1)\n", - "\n", - "class CustomTokenCredential:\n", - " def get_token(self, *scopes, **kwargs):\n", - " return AccessToken(notebookutils.credentials.getToken('storage'), expires_on=9999999999)\n", - "\n", - "credential = CustomTokenCredential()\n", - "service_client = DataLakeServiceClient(account_url=f\"https://onelake.dfs.fabric.microsoft.com\", credential=credential)\n", - "fs = service_client.get_file_system_client(WORKSPACE_NAME)\n", - "\n", - "lh_paths = {\n", - " \"tmp/landing/\": f\"{lakehouse_name}.Lakehouse/Files/landing\",\n", - " \"tmp/backup/\": f\"{lakehouse_name}.Lakehouse/Files/backup\",\n", - " \"tmp/Tables/\": f\"{lakehouse_name}.Lakehouse/Tables\"\n", - "}\n", - "\n", - "def unpack_files(Setup_dir, archives):\n", - " for archive, target in archives.items():\n", - " shutil.unpack_archive(os.path.join(Setup_dir, archive), target, \"zip\")\n", - "\n", - "def upload_files(local_path, azure_path):\n", - " for root, _, files in os.walk(local_path):\n", - " if root.endswith(\"/_metadata\"): continue\n", - " for file in files:\n", - " file_path_on_local = os.path.join(root, file)\n", - " relative_path = os.path.relpath(root, local_path)\n", - " file_path_on_azure = os.path.join(azure_path, relative_path, file).replace(\"\\\\\", \"/\")\n", - " file_client = fs.get_file_client(file_path_on_azure)\n", - " with open(file_path_on_local, \"rb\") as data:\n", - " file_client.upload_data(data, overwrite=True)\n", - "\n", - "git_lh_directory = \"Setup/Files/Lakehouse\"\n", - "archives = {\n", - " 'AW_landing.zip': \"tmp/landing\",\n", - " 'AW_Backup.zip': \"tmp/backup\",\n", - " 'AW_tables.zip': \"tmp/Tables\"\n", - "}\n", - "\n", - "unpack_files(git_lh_directory, archives)\n", - "for local, azure in lh_paths.items():\n", - " upload_files(local, azure)\n", - "\n", - "#Deploy AW Warehouse Objects \n", - "time.sleep(120) # 2mins wait for MD Sync of the Lakehouse objects\n", - "process_sql_objects(DBType.Warehouse, \"Tables\")\n", - "process_sql_objects(DBType.Warehouse, \"Views\")\n", - "print ('AdventureWorks Samples Deployed.')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b9b06dba-27d5-41e5-a8c6-3aea4099d953", - "metadata": { - "collapsed": false, - "microsoft": { - "language": "python", - "language_group": "synapse_pyspark" - } - }, - "outputs": [ - { - "data": { - "application/vnd.livy.statement-meta+json": { - "execution_finish_time": "2025-07-14T19:12:01.1615146Z", - "execution_start_time": "2025-07-14T19:11:34.7003662Z", - "livy_statement_state": "available", - "normalized_state": "finished", - "parent_msg_id": "3b82af23-9fca-4ba6-a02a-d9262e156b01", - "queued_time": "2025-07-14T19:03:18.3907112Z", - "session_id": "929963e1-609d-4bbe-8728-c790daed3f81", - "session_start_time": null, - "spark_pool": null, - "state": "finished", - "statement_id": 14, - "statement_ids": [ - 14 - ] - }, - "text/plain": [ - "StatementMeta(, 929963e1-609d-4bbe-8728-c790daed3f81, 14, Finished, Available, Finished)" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "restoring cdcSqlTables...\n", - "restoring Configurations...\n", - "restoring edwTableLineage...\n", - "restoring IdentityMethods...\n", - "restoring PackageGroups...\n", - "restoring Templates...\n", - "restoring Datasets...\n", - "restoring edwTables...\n", - "restoring PackageGroupLinks...\n", - "restoring PipelineGroups...\n", - "restoring edwTableJoins...\n", - "restoring PackageGroupTables...\n", - "restoring Pipelines...\n", - "restoring DatasetLineage...\n", - "restoring PackageGroupPipelines...\n", - "restoring PipelineTables...\n", - "Updating config.Configurations\n", - "\n", - "Sample Metadata restored. Now run TR_Ops Pipeline to complete\n" - ] - } - ], - "source": [ - "#Restore Latest Configuration\n", - "import pandas as pd\n", - "\n", - "if deploy_aw == False:\n", - " mssparkutils.notebook.exit(1)\n", - "\n", - "WORKSPACE_ID = fabric.get_workspace_id()\n", - "WORKSPACE_NAME = fabric.resolve_workspace_name()\n", - "lakehouse_id = fabric.resolve_item_id(lakehouse_name, \"Lakehouse\", WORKSPACE_ID)\n", - "client = fabric.FabricRestClient()\n", - "items = fabric.FabricRestClient().get(f\"/v1/workspaces/{WORKSPACE_ID}/SQLDatabases\").json()[\"value\"]\n", - "sql_database=next((endpoint for endpoint in items if endpoint[\"displayName\"] == metadata_db_name ))\n", - "sql_end_point = sql_database[\"properties\"][\"serverFqdn\"]\n", - "sql_database_name = sql_database[\"properties\"][\"databaseName\"]\n", - "connection_string = f\"Driver={{ODBC Driver 18 for SQL Server}};Server={sql_end_point};database={sql_database_name}\"\n", - "token = mssparkutils.credentials.getToken('https://analysis.windows.net/powerbi/api').encode(\"UTF-16-LE\")\n", - "token_struct = struct.pack(f' str: \n"," display_name = DW_NAME if self == DBType.Warehouse else META_DB_NAME\n"," client = fabric.FabricRestClient()\n"," endpoint = f\"/v1/workspaces/{WORKSPACE_ID}/{self.value}\"\n"," databases = client.get(endpoint).json()\n","\n"," selected_database = next((db for db in databases.get(\"value\", []) if db.get(\"displayName\") == display_name), None)\n"," if not selected_database:\n"," raise ValueError(f\"No {self.value} with displayName '{display_name}' found.\")\n"," \n"," server = selected_database['properties'].get('serverFqdn') or selected_database['properties'].get('connectionString')\n"," database = selected_database['properties'].get('databaseName', warehouse_name)\n"," \n"," return f\"Driver={{ODBC Driver 18 for SQL Server}};Server={server};database={database};LongAsMax=YES\"\n","\n"," def get_engine(self) -> Engine:\n"," if self not in engine_pool:\n"," token = mssparkutils.credentials.getToken('https://analysis.windows.net/powerbi/api').encode(\"UTF-16-LE\")\n"," token_struct = struct.pack(f' list:\n"," if not os.path.exists(file_path):\n"," return []\n"," \n"," with open(file_path, \"r\") as f:\n"," return [line.strip() for line in f if line.strip()]\n","\n","def process_sql_script(db_type: DBType, script_path: str):\n"," if not os.path.exists(script_path):\n"," return\n"," \n"," filename = os.path.basename(script_path)\n"," if \"prodata.ie\" in filename:\n"," print(f\"Skipping {filename} because it ends with 'prodata.ie'\")\n"," return\n","\n"," with open(script_path, \"r\") as file:\n"," script = file.read()\n","\n"," statements = script.split(\"\\nGO\\n\")\n"," \n"," object_name = script_path.split(\"/\")[-1].split(\".\")[0]\n"," db_name = script_path.split(\"/\")[0].replace(\".\", \" \")\n"," schema_name = script_path.split(\"/\")[1]\n","\n"," if \"Tables/\" in script_path:\n"," check_query = f\"SELECT COUNT(*) FROM sys.tables WHERE name = '{object_name}' AND schema_id = SCHEMA_ID('{schema_name}')\"\n"," elif \"Views/\" in script_path:\n"," check_query = f\"SELECT COUNT(*) FROM sys.views WHERE name = '{object_name}' AND schema_id = SCHEMA_ID('{schema_name}')\"\n"," elif \"StoredProcedures/\" in script_path:\n"," check_query = f\"SELECT COUNT(*) FROM sys.procedures WHERE name = '{object_name}' AND schema_id = SCHEMA_ID('{schema_name}')\"\n"," elif \"Security/\" in script_path: \n"," check_query = f\"SELECT COUNT(*) FROM sys.schemas WHERE name = '{object_name}'\"\n"," elif \"Functions\" in script_path:\n"," check_query= f\"SELECT COUNT(*) FROM sys.objects WHERE name = '{object_name}'\"\n"," else:\n"," check_query = f\"SELECT COUNT(*) FROM sys.schemas WHERE name = '{schema_name}'\"\n"," \n"," with db_type.get_engine().connect() as conn:\n"," object_exists = conn.execute(text(check_query)).scalar() > 0\n"," \n"," if not object_exists:\n"," try:\n"," for statement in filter(None, map(str.strip, statements)):\n"," conn.execute(text(statement))\n"," conn.commit()\n"," except Exception as e:\n"," conn.rollback()\n"," print(f\"Failed to create {schema_name}.{object_name}: {e}\")\n"," raise\n"," else:\n"," print(f\"The object {schema_name}.{object_name} already exists in {db_name}. No changes made.\")\n","\n","def iterate_sql_objects(db_type: DBType, object_type: str, objects_created: set):\n"," base_path = \"Meta.SQLDatabase\" if db_type == DBType.SQLDatabase else \"DW.Warehouse\"\n"," visited_dirs = set()\n"," \n"," for root, _, files in os.walk(base_path):\n"," if root in visited_dirs:\n"," continue\n"," visited_dirs.add(root)\n"," \n"," for file in filter(lambda f: f.endswith(\".sql\"), files):\n"," script_path = os.path.join(root, file).lstrip(\"./\")\n"," \n"," if script_path in objects_created:\n"," continue\n","\n"," parts = script_path.split(os.sep)\n"," if object_type == \"Schema\":\n"," if any(folder in parts for folder in [\"Tables\", \"Views\", \"StoredProcedures\"]):\n"," continue \n"," else:\n"," if object_type not in parts:\n"," continue \n","\n"," process_sql_script(db_type, script_path)\n"," objects_created.add(script_path)\n","\n","def process_sql_objects(db_type: DBType, obj_type: str = None):\n"," table_orders_cache = {}\n"," has_changed_directory = False \n"," current_dir = os.path.abspath(os.getcwd())\n","\n"," table_order_files = {\n"," DBType.SQLDatabase: os.path.join(current_dir, \"Setup\", \"Files\", \"SQLDatabase\", \"MetaTableOrder.txt\"),\n"," DBType.Warehouse: os.path.join(current_dir, \"Setup\", \"Files\", \"Warehouse\", \"DWTableOrder.txt\")\n"," }\n"," \n"," for db_type_key, file_path in table_order_files.items(): \n"," if os.path.exists(file_path):\n"," table_orders_cache[db_type_key] = read_table_order(file_path)\n","\n"," table_order = table_orders_cache.get(db_type, [])\n","\n"," if not has_changed_directory:\n"," workspaces_dwa_path = os.path.join(current_dir, \"Workspaces\", \"DWA\")\n"," \n"," if os.path.exists(workspaces_dwa_path):\n"," os.chdir(workspaces_dwa_path)\n"," has_changed_directory = True \n"," \n"," objects_created = set()\n"," base_path = \"Meta.SQLDatabase\" if db_type == DBType.SQLDatabase else \"DW.Warehouse\"\n"," \n"," for table in table_order:\n"," schema = table.split(\".\", maxsplit=1)[0].strip(\"[]\")\n"," table_name = table.split(\".\", maxsplit=1)[1].strip(\"[]\")\n"," script_path = f\"{base_path}/{schema}/{obj_type}/{table_name}.sql\"\n"," \n"," if os.path.exists(script_path) and script_path not in objects_created:\n"," process_sql_script(db_type, script_path)\n"," objects_created.add(script_path)\n"," \n"," iterate_sql_objects(db_type, obj_type, objects_created)\n"," os.chdir(original_dir)\n","\n","#Create meta databse objects\n","process_sql_objects(DBType.SQLDatabase, \"Security\")\n","process_sql_objects(DBType.SQLDatabase, \"Tables\")\n","process_sql_objects(DBType.SQLDatabase, \"Functions\")\n","process_sql_objects(DBType.SQLDatabase, \"Views\")\n","process_sql_objects(DBType.SQLDatabase, \"StoredProcedures\") \n","\n","\n","# #Create warehouse objects\n","process_sql_objects(DBType.Warehouse, \"Schema\")\n","process_sql_objects(DBType.Warehouse, \"StoredProcedures\")"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":14,"statement_ids":[14],"state":"finished","livy_statement_state":"available","session_id":"80e4293b-cfd2-4f4d-bac2-07adf37c5f4b","normalized_state":"finished","queued_time":"2025-11-11T13:59:12.1706018Z","session_start_time":null,"execution_start_time":"2025-11-11T13:59:12.1722391Z","execution_finish_time":"2025-11-11T13:59:13.6257446Z","parent_msg_id":"24d6e375-dd12-4bdc-aeb7-caf06b103246"},"text/plain":"StatementMeta(, 80e4293b-cfd2-4f4d-bac2-07adf37c5f4b, 14, Finished, Available, Finished)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["The object dbo.ufn_BuildChatPayloadJson already exists in Meta SQLDatabase. No changes made.\n"]}],"execution_count":12,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"de3b1a99-895a-4e74-8a0a-32132f617a34"},{"cell_type":"markdown","source":["### 6. Upload Data Pipelines"],"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"id":"da333f14-9428-48c3-aead-65c5d0bf1117"},{"cell_type":"code","source":["# Stores new references for updating old data pipelines connections \n","lh_artifact_id = fabric.resolve_item_id(lakehouse_name, \"Lakehouse\", WORKSPACE_ID) #Lakehouse ID\n","meta_artifact_id = fabric.resolve_item_id(metadata_db_name, \"SqlDatabase\", WORKSPACE_ID) #SQL DB ID and Connection String\n","sqldb_url = f'{BASE_URL}/workspaces/{WORKSPACE_ID}/SQLDatabases/{meta_artifact_id}'\n","response = requests.get(sqldb_url, headers=headers)\n","\n","meta_endpoint = response.json().get(\"properties\", {}).get(\"serverFqdn\")\n","meta_databasename = response.json().get(\"properties\", {}).get(\"databaseName\")\n","\n","dw_artifact_id = fabric.resolve_item_id(warehouse_name, \"Warehouse\", WORKSPACE_ID) #Warehouse ID and Connection String\n","\n","dw_url = f'{BASE_URL}/workspaces/{WORKSPACE_ID}/warehouses/{dw_artifact_id}'\n","response = requests.get(dw_url, headers=headers)\n","\n","dw_endpoint = response.json().get(\"properties\", {}).get(\"connectionString\")\n","\n","def create_folder(folder_name: str, parent_folder_id: str=None, workspace_id: str = None):\n"," if not workspace_id:\n"," workspace_id = sf.get_notebook_workspace_id()\n"," client = fabric.FabricRestClient()\n"," response = client.get( f\"v1/workspaces/{workspace_id}/folders\").json()\n"," folders = response.get(\"value\", [])\n"," folder_id=next((f[\"id\"] for f in folders if f[\"displayName\"].lower() == folder_name.lower()), None)\n"," if folder_id:\n"," return folder_id\n"," else:\n"," client = fabric.FabricRestClient()\n"," if parent_folder_id:\n"," json={ \n"," \"displayName\": folder_name,\n"," \"parentFolderId\": parent_folder_id\n"," \n"," }\n"," else:\n"," json={ \n"," \"displayName\": folder_name,\n"," } \n"," response = client.post(f\"v1/workspaces/{workspace_id}/folders\",json=json).json()\n"," return response['id']\n","\n","def get_connection_id_by_name(connection_name: str) -> str:\n"," url = f\"{BASE_URL}/connections\"\n","\n"," response = requests.get(url, headers=headers)\n"," response.raise_for_status()\n","\n"," connections = response.json().get(\"value\", [])\n","\n"," for conn in connections:\n"," if conn.get(\"displayName\") == connection_name:\n"," return conn.get(\"id\")\n","\n"," raise ValueError(f\"Connection with name '{connection_name}' not found in workspace {WORKSPACE_ID}.\")\n","\n","fabricsqldb_connection_id = get_connection_id_by_name(fabricsqldb_connection_name)\n","fabricdatapipelines_connection_id = get_connection_id_by_name(fabricdatapipelines_connection_name)\n","pbi_connection_id = get_connection_id_by_name(pbi_connection_name)\n","\n","current_dir = os.path.abspath(os.getcwd())\n","file_path = os.path.join(current_dir, \"Setup\", \"Files\", \"Workspace\", \"AWDataPipelines.txt\")\n","if os.path.exists(file_path):\n"," with open(file_path, \"r\") as f:\n"," aw_pipelines = [line.strip() for line in f if line.strip()]\n"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":11,"statement_ids":[11],"state":"finished","livy_statement_state":"available","session_id":"3945c33e-ce3b-4b2e-a201-c2e61c94bbd1","normalized_state":"finished","queued_time":"2025-11-11T10:26:31.2156833Z","session_start_time":null,"execution_start_time":"2025-11-11T10:31:20.7353325Z","execution_finish_time":"2025-11-11T10:31:25.342544Z","parent_msg_id":"48c4d53b-909b-4744-8e36-d3f56d27b9e0"},"text/plain":"StatementMeta(, 3945c33e-ce3b-4b2e-a201-c2e61c94bbd1, 11, Finished, Available, Finished)"},"metadata":{}}],"execution_count":9,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"eee6400d-7f6f-4b5f-a5a0-3b5433065a31"},{"cell_type":"code","source":["access_token = mssparkutils.credentials.getToken('pbi') #Rebuild access token \n","\n","headers = {\n"," \"Authorization\": f\"Bearer {access_token}\",\n"," \"Content-Type\": \"application/json\"\n","}\n","\n","env_variables_map = {\n"," \"5941a6c0-8c98-4d79-b065-a3789e9e0960\": WORKSPACE_ID,\n"," \"fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.database.fabric.microsoft.com\": meta_endpoint,\n"," \"Meta-fe70c606-af27-4f64-973a-2be877526212\": meta_databasename,\n"," \"fkm4vwf6l6zebg4lqrhbtdcmsq-yctecwmyrr4u3mdfun4j5hqjma.datawarehouse.fabric.microsoft.com\": dw_endpoint,\n"," \"DW\": warehouse_name,\n"," \"d58f4f2d-59d7-406d-ae4c-898354a6a75f\" : lh_artifact_id,\n"," \"fe70c606-af27-4f64-973a-2be877526212\" : meta_artifact_id\n","}\n","\n","class DataPipeline:\n"," pass\n","\n","class DataPipeline:\n"," name: str\n"," git_id: str\n"," real_id: str\n"," raw_definition: str\n"," definition: object\n"," pipeline_created: bool = False\n"," folder_id: str\n","\n"," def __init__(self, name, git_id, raw_definition, definition, folder_id):\n"," self.name = name\n"," self.git_id = git_id\n"," self.definition = definition\n"," self.raw_definition = raw_definition\n"," self.folder_id=folder_id\n"," def __hash__(self) -> int:\n"," return hash(self.git_id)\n"," def __eq__(self, other):\n"," return self.git_id == other.git_id\n","\n"," def __str__(self):\n"," return f\"DataPipeline: {self.name}\"\n","\n"," def __repr__(self):\n"," return f\"DataPipeline: {self.name}\"\n","\n"," def is_created(self) -> bool:\n"," return self in pipelines_created or self.pipeline_created\n","\n"," def create(self, pipelines):\n"," global pipelines_created\n","\n"," if self.name in aw_pipelines:\n"," print(f\"Skipping {self.name} as requires SFTP Connection. Install Manually if required.\")\n"," return\n","\n"," if self.is_created():\n"," print(f\"{self.name} already created\")\n"," return\n","\n"," print(f\"Creating pipeline {self.name}\")\n","\n"," child_pipelines: list[DataPipeline] = self.get_child_pipelines(pipelines)\n","\n"," for child_pipeline in child_pipelines:\n"," if child_pipeline.is_created():\n"," continue\n"," child_pipeline.create(pipelines)\n","\n"," self.update_activity_references()\n"," self.upload_pipeline()\n","\n"," print(f\"{self.name} created and uploaded.\")\n","\n"," def get_child_pipelines(self, pipelines : list[DataPipeline]) -> list[DataPipeline]:\n"," self.child_pipelines: set[DataPipeline] = set()\n","\n"," def find_pipelines(activities : list [object]):\n"," for activity in activities:\n"," activity_type = activity.get(\"type\")\n"," type_properties = activity.get(\"typeProperties\", {})\n","\n"," if activity_type == \"ExecutePipeline\":\n"," reference_name = type_properties.get(\"pipeline\", {}).get(\n"," \"referenceName\"\n"," )\n"," matched_pipeline = next(\n"," (x for x in pipelines if x.git_id == reference_name), None\n"," )\n"," if matched_pipeline:\n"," self.child_pipelines.add(matched_pipeline)\n","\n"," elif activity_type == \"InvokePipeline\":\n"," pipeline_id = type_properties.get(\"pipelineId\")\n"," matched_pipeline = next(\n"," (x for x in pipelines if x.git_id == pipeline_id), None\n"," )\n"," if matched_pipeline:\n"," self.child_pipelines.add(matched_pipeline)\n"," \n"," else:\n"," for key in [\"activities\", \"ifTrueActivities\", \"ifFalseActivities\"]:\n"," if key in type_properties:\n"," find_pipelines(type_properties[key])\n"," return self.child_pipelines\n","\n"," return find_pipelines(\n"," self.definition.get(\"properties\", {}).get(\"activities\", [])\n"," )\n","\n"," def update_activity_references(self):\n"," global pipelines_created\n","\n"," def fetch_trident_notebooks():\n"," notebooks_url = f\"{BASE_URL}/workspaces/{WORKSPACE_ID}/notebooks\"\n"," response = requests.get(notebooks_url, headers=headers)\n"," response.raise_for_status()\n"," return {notebook[\"displayName\"]: notebook[\"id\"] for notebook in response.json().get(\"value\", [])}\n","\n"," trident_notebooks = fetch_trident_notebooks()\n","\n"," new_definition = self.raw_definition\n"," for pipeline in pipelines_created: \n"," new_definition = new_definition.replace(pipeline.git_id, pipeline.real_id)\n"," for source_id, target_id in env_variables_map.items(): \n"," new_definition = new_definition.replace(source_id, target_id)\n","\n"," definition_json = json.loads(new_definition)\n","\n"," def replace_values(obj):\n"," if isinstance(obj, dict):\n"," for key, value in obj.items():\n"," if key == \"type\" and isinstance(value, str):\n"," if \"Lakehouse\" in value:\n"," if \"typeProperties\" in obj and isinstance(obj[\"typeProperties\"], dict) :\n"," if \"artifactId\" in obj[\"typeProperties\"] and not obj[\"typeProperties\"][\"artifactId\"].startswith(\"@\"):\n"," obj[\"typeProperties\"][\"artifactId\"] = lh_artifact_id \n"," \n"," elif \"DataWarehouse\" in value:\n"," if \"typeProperties\" in obj and isinstance(obj[\"typeProperties\"], dict):\n"," if \"artifactId\" in obj[\"typeProperties\"] and not obj[\"typeProperties\"][\"artifactId\"].startswith(\"@\"):\n"," obj[\"typeProperties\"][\"artifactId\"] = dw_artifact_id \n"," if \"endpoint\" in obj and not obj[\"endpoint\"].startswith(\"@\"):\n"," obj[\"endpoint\"] = dw_endpoint \n","\n"," elif \"FabricSqlDatabase\" in value:\n"," if \"typeProperties\" in obj and isinstance(obj[\"typeProperties\"], dict):\n"," if \"artifactId\" in obj[\"typeProperties\"] and not obj[\"typeProperties\"][\"artifactId\"].startswith(\"@\"):\n"," obj[\"typeProperties\"][\"artifactId\"] = meta_artifact_id \n"," if \"endpoint\" in obj and not obj[\"endpoint\"].startswith(\"@\"):\n"," obj[\"endpoint\"] = meta_endpoint \n"," if \"externalReferences\" in obj and isinstance(obj[\"externalReferences\"], dict):\n"," obj[\"externalReferences\"][\"connection\"] = fabricsqldb_connection_id\n","\n"," elif \"InvokePipeline\" in value:\n"," if \"externalReferences\" in obj and isinstance(obj[\"externalReferences\"], dict):\n"," obj[\"externalReferences\"][\"connection\"] = fabricdatapipelines_connection_id\n","\n"," elif \"PBISemanticModelRefresh\" in value:\n"," if \"externalReferences\" in obj and isinstance(obj[\"externalReferences\"], dict):\n"," obj[\"externalReferences\"][\"connection\"] = pbi_connection_id\n"," \n"," elif \"TridentNotebook\" in value:\n"," notebook_name = obj.get(\"name\")\n"," if notebook_name in trident_notebooks:\n"," obj[\"typeProperties\"][\"notebookId\"] = trident_notebooks[notebook_name]\n","\n"," elif key == \"workspaceId\" and isinstance(value, str) and not value.startswith(\"@\"):\n"," obj[key] = WORKSPACE_ID \n","\n"," replace_values(value)\n","\n"," elif isinstance(obj, list):\n"," for item in obj:\n"," replace_values(item)\n","\n"," replace_values(definition_json)\n","\n"," self.raw_definition = json.dumps(definition_json, indent=4)\n","\n"," def upload_pipeline(self):\n"," global pipelines_created\n"," pipelines_url = f\"{BASE_URL}/workspaces/{WORKSPACE_ID}/dataPipelines\"\n"," pipeline_b64 = base64.b64encode(self.raw_definition.encode()).decode()\n","\n"," payload = {\n"," \"displayName\": self.name,\n"," \"folderId\" : self.folder_id,\n"," \"definition\": {\n"," \"parts\": [\n"," {\n"," \"path\": \"pipeline.content.json\",\n"," \"payload\": pipeline_b64,\n"," \"payloadType\": \"InlineBase64\",\n"," }\n"," ]\n"," },\n"," }\n"," create_pipeline_request = requests.post(pipelines_url, headers=headers, data=json.dumps(payload))\n"," if not create_pipeline_request.ok:\n"," print(create_pipeline_request.json())\n"," raise RuntimeError(f\"Failed to create pipeline {self.name}. ({create_pipeline_request.text})\")\n"," self.real_id = create_pipeline_request.json().get(\"id\")\n"," self.pipeline_created = True\n"," pipelines_created.append(self)\n"," \n","pipelines: list[DataPipeline] = []\n","folder_cache = {}\n","for path, dirs, files in os.walk(\".\"):\n"," if not path.endswith(\".DataPipeline\"):\n"," continue\n"," folder_id=None\n"," folder_path = \"/\".join( path.split(\"/\")[3:-1])\n"," if folder_path in folder_cache:\n"," folder_id=folder_cache[folder_path]\n"," else:\n"," for folder in folder_path.split('/'):\n"," folder_id=create_folder(folder, folder_id)\n"," folder_cache[folder_path] = folder_id\n"," pipeline_name = path.strip(\".\").split(\".\")[0].replace(\"\\\\\", \"/\").split(\"/\")[-1]\n"," platform = json.load(open(os.path.join(path, \".platform\")))\n"," raw_definition = open(os.path.join(path, \"pipeline-content.json\")).read()\n"," definition = json.loads(raw_definition)\n"," pipeline_id = platform[\"config\"][\"logicalId\"]\n","\n"," data_pipeline = DataPipeline(pipeline_name, pipeline_id, raw_definition, definition, folder_id)\n"," pipelines.append(data_pipeline)\n","pipelines_url = f\"{BASE_URL}/workspaces/{WORKSPACE_ID}/dataPipelines\"\n","response = requests.get(pipelines_url, headers=headers)\n","response.raise_for_status()\n","\n","pipelines_created: list[DataPipeline] = []\n","\n","pipelines_existing: list[(str, str)] = [\n"," (pipeline[\"displayName\"], pipeline[\"id\"])\n"," for pipeline in response.json().get(\"value\", [])\n","]\n","\n","for pipeline_name, pipeline_id in pipelines_existing:\n"," pipeline = next((x for x in pipelines if x.name == pipeline_name), None)\n"," if not pipeline:\n"," continue\n","\n"," pipeline.real_id = pipeline_id\n"," pipeline.pipeline_created = True\n"," pipelines_created.append(pipeline)\n","i=0\n","\n","for pipeline in pipelines:\n"," i+=1\n"," if pipeline in pipelines_created or pipeline.pipeline_created:\n"," continue\n"," #if i>1 : break\n"," pipeline.create(pipelines)\n","\n"," "],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":12,"statement_ids":[12],"state":"finished","livy_statement_state":"available","session_id":"3945c33e-ce3b-4b2e-a201-c2e61c94bbd1","normalized_state":"finished","queued_time":"2025-11-11T10:26:31.2684581Z","session_start_time":null,"execution_start_time":"2025-11-11T10:31:25.3449987Z","execution_finish_time":"2025-11-11T10:32:54.103487Z","parent_msg_id":"ea60ab14-d2a9-4956-ab21-e03b5383b80d"},"text/plain":"StatementMeta(, 3945c33e-ce3b-4b2e-a201-c2e61c94bbd1, 12, Finished, Available, Finished)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["Creating pipeline SCH_AdventureWorks\nCreating pipeline Pipeline-Controller\nCreating pipeline Pipeline-Worker\nCreating pipeline Environment-Variables\nEnvironment-Variables created and uploaded.\nPipeline-Worker created and uploaded.\nPipeline-Controller created and uploaded.\nSCH_AdventureWorks created and uploaded.\nCreating pipeline SCH_ConfigBackup\nCreating pipeline Backup_ConfigSchema\nBackup_ConfigSchema created and uploaded.\nSCH_ConfigBackup created and uploaded.\nCreating pipeline SCH_Ops\nSCH_Ops created and uploaded.\nCreating pipeline Restore_ConfigSchema\nRestore_ConfigSchema created and uploaded.\nCreating pipeline Refresh-PBI\nRefresh-PBI created and uploaded.\nSkipping Copy-SFTP-Prodata as requires SFTP Connection. Install Manually if required.\nCreating pipeline Ingest-SQL-CDC\nIngest-SQL-CDC created and uploaded.\nCreating pipeline Extract-SQL\nExtract-SQL created and uploaded.\n"]}],"execution_count":10,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"19d74a8c-843a-4ba4-bcc8-f55a28ebf92f"},{"cell_type":"markdown","source":["### 7. Create objects for Adventure Works when deploy_aw is set to True"],"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"},"nteract":{"transient":{"deleting":false}}},"id":"acc1cdcb-2625-449d-9a11-c903889ec341"},{"cell_type":"code","source":["if deploy_aw == False:\n"," mssparkutils.notebook.exit(1)\n","\n","class CustomTokenCredential:\n"," def get_token(self, *scopes, **kwargs):\n"," return AccessToken(notebookutils.credentials.getToken('storage'), expires_on=9999999999)\n","\n","credential = CustomTokenCredential()\n","service_client = DataLakeServiceClient(account_url=f\"https://onelake.dfs.fabric.microsoft.com\", credential=credential)\n","fs = service_client.get_file_system_client(WORKSPACE_NAME)\n","\n","lh_paths = {\n"," \"tmp/landing/\": f\"{lakehouse_name}.Lakehouse/Files/landing\",\n"," \"tmp/backup/\": f\"{lakehouse_name}.Lakehouse/Files/backup\",\n"," \"tmp/Tables/\": f\"{lakehouse_name}.Lakehouse/Tables\"\n","}\n","\n","def unpack_files(Setup_dir, archives):\n"," for archive, target in archives.items():\n"," shutil.unpack_archive(os.path.join(Setup_dir, archive), target, \"zip\")\n","\n","def upload_files(local_path, azure_path):\n"," for root, _, files in os.walk(local_path):\n"," if root.endswith(\"/_metadata\"): continue\n"," for file in files:\n"," file_path_on_local = os.path.join(root, file)\n"," relative_path = os.path.relpath(root, local_path)\n"," file_path_on_azure = os.path.join(azure_path, relative_path, file).replace(\"\\\\\", \"/\")\n"," file_client = fs.get_file_client(file_path_on_azure)\n"," with open(file_path_on_local, \"rb\") as data:\n"," file_client.upload_data(data, overwrite=True)\n","\n","git_lh_directory = \"Setup/Files/Lakehouse\"\n","archives = {\n"," 'AW_landing.zip': \"tmp/landing\",\n"," 'AW_Backup.zip': \"tmp/backup\",\n"," 'AW_tables.zip': \"tmp/Tables\"\n","}\n","\n","unpack_files(git_lh_directory, archives)\n","for local, azure in lh_paths.items():\n"," upload_files(local, azure)\n","\n","#Deploy AW Warehouse Objects \n","time.sleep(120) # 2mins wait for MD Sync of the Lakehouse objects\n","process_sql_objects(DBType.Warehouse, \"Tables\")\n","process_sql_objects(DBType.Warehouse, \"Views\")\n","print ('AdventureWorks Samples Deployed.')"],"outputs":[{"output_type":"display_data","data":{"application/vnd.livy.statement-meta+json":{"spark_pool":null,"statement_id":13,"statement_ids":[13],"state":"finished","livy_statement_state":"available","session_id":"3945c33e-ce3b-4b2e-a201-c2e61c94bbd1","normalized_state":"finished","queued_time":"2025-11-11T10:26:31.3321935Z","session_start_time":null,"execution_start_time":"2025-11-11T10:32:54.1054295Z","execution_finish_time":"2025-11-11T10:35:07.8347611Z","parent_msg_id":"f3fa9c6a-077e-4922-83a6-cad45193706a"},"text/plain":"StatementMeta(, 3945c33e-ce3b-4b2e-a201-c2e61c94bbd1, 13, Finished, Available, Finished)"},"metadata":{}},{"output_type":"stream","name":"stdout","text":["AdventureWorks Samples Deployed.\n"]}],"execution_count":11,"metadata":{"microsoft":{"language":"python","language_group":"synapse_pyspark"}},"id":"350e775b-7c60-4d73-a9d7-d6bd77673d28"},{"cell_type":"code","source":["#Restore Latest Configuration\n","import pandas as pd\n","\n","if deploy_aw == False:\n"," mssparkutils.notebook.exit(1)\n","\n","WORKSPACE_ID = fabric.get_workspace_id()\n","WORKSPACE_NAME = fabric.resolve_workspace_name()\n","lakehouse_id = fabric.resolve_item_id(lakehouse_name, \"Lakehouse\", WORKSPACE_ID)\n","client = fabric.FabricRestClient()\n","items = fabric.FabricRestClient().get(f\"/v1/workspaces/{WORKSPACE_ID}/SQLDatabases\").json()[\"value\"]\n","sql_database=next((endpoint for endpoint in items if endpoint[\"displayName\"] == metadata_db_name ))\n","sql_end_point = sql_database[\"properties\"][\"serverFqdn\"]\n","sql_database_name = sql_database[\"properties\"][\"databaseName\"]\n","connection_string = f\"Driver={{ODBC Driver 18 for SQL Server}};Server={sql_end_point};database={sql_database_name}\"\n","token = mssparkutils.credentials.getToken('https://analysis.windows.net/powerbi/api').encode(\"UTF-16-LE\")\n","token_struct = struct.pack(f' re.Pattern: + p = phrase.strip() + if p.endswith(":"): + p = p[:-1].rstrip() + # Escape each token, join with flexible whitespace + tokens = re.split(r"\s+", p) + pattern = r"\b" + r"\s+".join(re.escape(t) for t in tokens if t) + r"\s*:?" + return re.compile(pattern, re.IGNORECASE) + +def sections_from_text(text: str, marker_phrase: str): + marker_re = make_marker_regex(marker_phrase) + pages = [p.strip() for p in text.split(PAGE_BREAK)] + starts = [i for i, pg in enumerate(pages) if marker_re.search(pg)] + if not starts: + return [] + + sections = [] + for idx, start in enumerate(starts): + end = (starts[idx + 1] - 1) if idx + 1 < len(starts) else (len(pages) - 1) + body = ("\n\n" + PAGE_BREAK + "\n\n").join(pages[start : end + 1]) + sections.append({ + "section_id": idx + 1, + "start_page": start + 1, + "end_page": end + 1, + "num_pages": end - start + 1, + "content": body, + }) + return sections + + +#This is to fix a bug in Reading PDF Headings with Azure Document Intelligence +def format_document_headings(doc): + lines = doc.split('\n') + out = [] + i = 0 + + while i < len(lines): + line = lines[i].strip() + # Fix escaped dot (e.g. '10\.') to '10.' + line = re.sub(r'(\d+)\\\.', r'\1.', line) + + # Check if line is only numbering e.g. '10.', '10.1', etc. + if re.match(r'^\d+(\.\d+)*\.?$', line): + next_line = lines[i+1].strip() if i + 1 < len(lines) else '' + + if next_line: + # For level-1 headings (single number + dot), add '## ' prefix + if re.match(r'^\d+\.$', line): + combined = f'## {line} {next_line}' + else: + combined = f'{line} {next_line}' + + out.append(combined) + i += 2 + continue + + out.append(line) + i += 1 + + return '\n'.join(out) + + +class CustomTokenCredential: + def get_token(self, *scopes, **kwargs): + return AccessToken(notebookutils.credentials.getToken('storage'), expires_on=int(time.time()) + 3000) + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# CELL ******************** + +import json +import os +import time +from operator import itemgetter +import sempy.fabric as fabric +import fsspec +from pathlib import PurePath +from azure.core.credentials import AzureKeyCredential,AccessToken +from azure.ai.documentintelligence import DocumentIntelligenceClient +from azure.ai.documentintelligence.models import AnalyzeOutputOption + +#Settings +source_connection_settings = json.loads(SourceConnectionSettings or "{}") +keyvault,di_secret,di_endpoint, = itemgetter("KeyVault", "diSecret","diEndpoint")(json.loads(SourceConnectionSettings or "{}")) +source_lakehouse_id = source_connection_settings.get("lakehouseId",fabric.get_lakehouse_id()) +source_workspace_id = source_connection_settings.get("workspaceId",fabric.get_workspace_id()) +source_lakehouse_name = source_connection_settings.get("lakehouse",fabric.resolve_item_name(item_id=source_lakehouse_id, workspace=source_workspace_id)) +source_workspace_name = fabric.resolve_workspace_name(source_workspace_id) + +target_connection_settings = json.loads(TargetConnectionSettings or '{}') +target_lakehouse_id = target_connection_settings.get("lakehouseId",fabric.get_lakehouse_id()) +target_workspace_id = target_connection_settings.get("workspaceId",fabric.get_workspace_id()) +target_lakehouse_name = target_connection_settings.get("lakehouse",fabric.resolve_item_name(item_id=target_lakehouse_id, workspace=target_workspace_id)) +target_workspace_name = fabric.resolve_workspace_name(target_workspace_id) + +source_settings = json.loads(SourceSettings or "{}") +delete = bool(source_settings.get("delete", False)) +source_directory = source_settings["directory"] +source = f"abfss://{source_workspace_id}@onelake.dfs.fabric.microsoft.com/{source_lakehouse_id}/Files" +source_path = os.path.join(source, source_directory) + +target_settings = json.loads(TargetSettings or "{}") +processed_directory = target_settings["directory"] +processed_path = os.path.join(source, processed_directory ) +activity_settings = json.loads(ActivitySettings or "{}") +split_marker= activity_settings.get("split_marker", None) + + +if not mssparkutils.fs.exists(source_path) or len(mssparkutils.fs.ls(source_path)) == 0: + mssparkutils.notebook.exit(0) # no folder to process, quit + +files = mssparkutils.fs.ls(source_path) +files = [fi for fi in files if getattr(fi, "size", 0) and int(getattr(fi, "size", 0)) > 0] #Fiter out Folders and zero length files + +credential = AzureKeyCredential(notebookutils.credentials.getSecret(keyvault, di_secret)) +document_intelligence_client = DocumentIntelligenceClient(di_endpoint, credential) + +start_time=time.time() +for file in files: + file_path = file.path + file_name = PurePath(file_path).name + print(f"Processing: {file_name}") + + fs = fsspec.filesystem("abfss", account_name="onelake" , account_host="onelake.dfs.fabric.microsoft.com", credential=CustomTokenCredential()) + + #with open(file_path, "rb") as f: + with fs.open(file_path, "rb") as f: + poller = document_intelligence_client.begin_analyze_document( + "prebuilt-layout", + body=f.read(), + output=[AnalyzeOutputOption.FIGURES], + output_content_format="MARKDOWN" + ) + + result = poller.result() + markdown_contents = format_document_headings(result.content)#Fix for Bug in document intelligence + + + if split_marker: + for section in sections_from_text( markdown_contents, split_marker): + file= os.path.join(f"abfss://{target_workspace_id}@onelake.dfs.fabric.microsoft.com/{target_lakehouse_id}/Files", processed_directory, f"{file_name}_section_{section['section_id']}.md") + notebookutils.fs.put( file, section['content'],overwrite=True) + print (f" -Wrote {file_name}_section_{section['section_id']}.md") + else: + file= os.path.join(f"abfss://{target_workspace_id}@onelake.dfs.fabric.microsoft.com/{target_lakehouse_id}/Files", processed_directory, f"{file_name}.md") + notebookutils.fs.put(file, markdown_contents,overwrite=True) + print (f" -Wrote {file_name}.md") + if delete: + notebookutils.fs.rm(file_path) + +elapsed = time.time() - start_time +print(f"Processed {len(files)} Documents in {elapsed:.0f} secs") + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } diff --git a/Workspaces/DWA/DWA/Library/AI/Extract-AI-LH.Notebook/.platform b/Workspaces/DWA/DWA/Library/AI/Extract-AI-LH.Notebook/.platform new file mode 100644 index 0000000..963070e --- /dev/null +++ b/Workspaces/DWA/DWA/Library/AI/Extract-AI-LH.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "Extract-AI-LH", + "description": "New notebook" + }, + "config": { + "version": "2.0", + "logicalId": "75c9fbd5-b62e-9367-4e6c-0a1c0c860360" + } +} \ No newline at end of file diff --git a/Workspaces/DWA/DWA/Library/AI/Extract-AI-LH.Notebook/notebook-content.py b/Workspaces/DWA/DWA/Library/AI/Extract-AI-LH.Notebook/notebook-content.py new file mode 100644 index 0000000..2a6f9b5 --- /dev/null +++ b/Workspaces/DWA/DWA/Library/AI/Extract-AI-LH.Notebook/notebook-content.py @@ -0,0 +1,439 @@ +# Fabric notebook source + +# METADATA ******************** + +# META { +# META "kernel_info": { +# META "name": "synapse_pyspark" +# META }, +# META "dependencies": { +# META "lakehouse": { +# META "default_lakehouse": "f2f9c5fa-ca0c-41b2-b0e1-3028165b4f6c", +# META "default_lakehouse_name": "FabricLH", +# META "default_lakehouse_workspace_id": "9b8a6500-5ccb-49a9-885b-b5b081efed75", +# META "known_lakehouses": [ +# META { +# META "id": "f2f9c5fa-ca0c-41b2-b0e1-3028165b4f6c" +# META } +# META ] +# META }, +# META "environment": {} +# META } +# META } + +# MARKDOWN ******************** + +# ### Extract-AI-LH +# Run a GPT Prompt on one or more files, or without any files
+# Files can be markdown files, txt or PDF +# Output is written to LH tables +# +# #### Parameters +# ##### SourceConnectionSettings +# - api_provider [openai or azure_openai]. We recommend azure_openai if you wnat to use stronger entra authentication +# - api_family [chat or responses]. We recocommend responses if your region supports it +# - ai_endpoint = URL of AI end point. Supports Azure Open AI, Azure AI Foundry, Open AI, Perplexity and others +# - ai_secret. The secret name in KeyVault for API Key +# - keyvault. The URL of the KeyVault +# - api_version. The API version (Azure Open AI only). Eg 2025-04-01-preview +# - max_retries. The maximum number of retries is AI compute too busy (HTTP 429) +# +# ##### ActivitySettings +# The body of request to by sent AI client as per +# Responses and Completions API Documentation
+# https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/responses?tabs=python-key +# https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/chatgpt +# +# +# +# +# +# +# + + +# CELL ******************** + +!pip install openai +import importlib,sys +sys.modules.pop("typing_extensions", None) +import typing_extensions +importlib.reload(typing_extensions) + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# PARAMETERS CELL ******************** + +import json +SourceSettings = '{"Directory": "landing/ai/invoices/markdown", "File":"*.md", "delete":"False"}' +TargetSettings = '{ "SchemaName":"ai","TableName": "invoice1", "Directory":"landing/ai/invoices//processed/"}' +SourceConnectionSettings = '{"api_family":"responses", "api_provider":"azure_openai","client_id":"dwa-training-app-id","client_secret":"dwa-training-app-secret","ai_endpoint":"https://ai-foundry-training-swe.cognitiveservices.azure.com/","KeyVault":"https://dw-training-kv.vault.azure.net/"}' +SinkConnectionSettings = '{}' +ActivitySettings='{}' +LineageKey : str = '00000000-0000-0000-0000-000000000000' #Unique Identifier. Will be Random if Zero or None and not passed +RunId : str = '00000000-0000-0000-0000-000000000000' + +#Unit Test for Sample Extract, Categorisation and basic reasoning on Invoices +ActivitySettings={"model":"gpt-4.1","instructions":"You are a specialized data extraction assistant for invoices","input":"Extract These Fields and return a single json object:Supplier : Customer, InvoiceNo, OrderNo, InvoiceDate, DueDate, Items: {Json Array of items on Invoice with attributes: description, Qty, Price, SubTotal}, Tax, Total,PaymentTerms,Valid: Return yes if the sum of the SubTotals equals the total on the invoice,Sector: Return the Industry Sector for the Supplier or Services","temperature":0.3} + + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# CELL ******************** + +import json +import os +import re +import ast +from operator import itemgetter + +source_connection_settings = json.loads(SourceConnectionSettings or '{}') +source_settings = json.loads(SourceSettings or '{}') +target_settings = json.loads(TargetSettings or '{}') +activity_settings = dict(ActivitySettings) if isinstance(ActivitySettings, dict) else json.loads(ActivitySettings or '{}') +if isinstance(activity_settings.get("temperature"), str): activity_settings["temperature"] = float(activity_settings["temperature"]) + +keyvault= source_connection_settings.get("KeyVault",None) +kv_client_id= source_connection_settings.get("client_id",None) +kv_client_secret= source_connection_settings.get("client_secret",None) +open_ai_secret_name = source_connection_settings.get("ai_secret",None) +open_ai_max_retries = source_connection_settings.get("max_retries",1) +open_ai_endpoint = source_connection_settings.get("ai_endpoint",None) +open_ai_api_version = source_connection_settings.get("api_version","2025-04-01-preview") +api_family = source_connection_settings.get("api_family","responses") +api_family = "chat_completions" if "chat" in str(api_family).lower() else "responses" +api_provider = source_connection_settings.get("api_provider","azure_openai") +api_provider = "azure_openai" if "azure" in str(api_provider) else "openai" + +model = activity_settings.pop("model","gpt-4.1-mini") +delete = bool(ast.literal_eval(source_settings.get("Delete", "False"))) +response_schema=activity_settings.get("schema",None) +response_schema = json.loads (response_schema) if isinstance(response_schema, str) else (response_schema or {}) + +table_name = target_settings.get("TableName","content") +schema_name = target_settings.get("SchemaName","dbo") +table_layout = target_settings.get("table_layout","content") +source_directory = source_settings.get("Directory",None) +source_file = source_settings.get("File", "*.md") +processed_directory = target_settings.get("Directory",None) + +if source_directory and source_directory.startswith("Files"): + source_directory = os.path.join("Files",source_directory) +if processed_directory and not processed_directory.startswith("Files"): + processed_directory = os.path.join("Files",processed_directory) + +post_extract_spark_sql = activity_settings.pop('PostExtractSparkSQL', None) + +if isinstance(model, str): + models= model.split(",") + model = [model.strip(" ") for model in models] +elif isinstance(model, list): + assert(all([isinstance(model,str) for model in models])) + models = model +else: + raise ValueError("model must be either a string or a list of strings. Eg gpt-'4.1-mini', 'gpt-4.1'") + + +LAKEHOUSE_DEFAULT_PREFIX = "/lakehouse/default/" +FILES_DEFAULT_PREFIX = "Files" +if source_directory: + if not source_directory.startswith(FILES_DEFAULT_PREFIX): + source_directory = os.path.join(FILES_DEFAULT_PREFIX,source_directory) + if not source_directory.startswith(LAKEHOUSE_DEFAULT_PREFIX): + source_directory = os.path.join(LAKEHOUSE_DEFAULT_PREFIX,source_directory) +if processed_directory: + if not processed_directory.startswith(FILES_DEFAULT_PREFIX): + processed_directory = os.path.join(FILES_DEFAULT_PREFIX,processed_directory) + if not processed_directory.startswith(LAKEHOUSE_DEFAULT_PREFIX): + processed_directory = os.path.join(LAKEHOUSE_DEFAULT_PREFIX,processed_directory) + +def clean_cols(df, names=None, pat=r'[ ,;{}()\n\t=]', replace=None): + cols = list(names or df.columns) + new = [re.sub('_+','_', re.sub(pat,'_', c)).strip('_') for c in cols] + if replace: # optional dict of renames + new = [replace.get(orig, replace.get(n, n)) for orig, n in zip(cols, new)] + return df.toDF(*new) + + +def build_structured_input(prompt: str, file_ids=None, role: str = "user"): + content = [{"type": "input_text", "text": prompt}] + if file_ids: + for fid in (file_ids if isinstance(file_ids, (list, tuple)) else [file_ids]): + content.append({"type": "input_file", "file_id": fid}) + + return [{ + "role": role, + "content": content + }] + +def azure_ad_token_provider(): + return credential.get_token("https://cognitiveservices.azure.com/.default").token + +def normalize_json_types(obj): + """Recursively convert ints -> floats and ensure lists/dicts are consistent. Need this to avoid data type mismatch.""" + if isinstance(obj, dict): + return {k: normalize_json_types(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [normalize_json_types(v) for v in obj] + elif isinstance(obj, int): # promote all ints to floats + return float(obj) + else: + return obj + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# CELL ******************** + +import time +from openai import AzureOpenAI +from openai import OpenAI +from azure.identity import ClientSecretCredential +import re +import sempy.fabric as fabric +import pandas as pd +from fnmatch import filter as fnfilter +from typing import Dict, Any, Iterable, Tuple, List +from io import BytesIO +from urllib.parse import urlparse, urlunparse +from datetime import datetime, timezone +import base64, os + +rows = [] +open_ai_max_retries = int(open_ai_max_retries) if open_ai_max_retries is not None else 2 +if kv_client_id: + tenant_id=spark.conf.get("trident.tenant.id") + client_id = notebookutils.credentials.getSecret(keyvault, kv_client_id) + client_secret = notebookutils.credentials.getSecret(keyvault, kv_client_secret) + credential = ClientSecretCredential(tenant_id, client_id, client_secret) + def azure_ad_token_provider(): + # Return a fresh token each time (auto-renew via azure-identity) + return credential.get_token("https://cognitiveservices.azure.com/.default").token + auth_kw= {"azure_ad_token_provider": azure_ad_token_provider} +else: + api_key = notebookutils.credentials.getSecret(keyvault, open_ai_secret_name) + auth_kw = {"api_key": api_key} + +if api_provider== "azure_openai": + client = AzureOpenAI( + **auth_kw, + api_version = open_ai_api_version, + azure_endpoint = open_ai_endpoint, + max_retries=open_ai_max_retries + ) +else: + if open_ai_endpoint: + open_ai_endpoint = (lambda s: urlunparse((p:=urlparse(s))._replace(path=((p.path.rstrip('/') + '/openai/v1') if ('azure.com' in s.lower() and not p.path.rstrip('/').endswith('/openai/v1')) else p.path))))(open_ai_endpoint) + client = OpenAI( + api_key = notebookutils.credentials.getSecret(keyvault, open_ai_secret_name), + base_url= open_ai_endpoint, + max_retries=open_ai_max_retries + ) + else: + client = OpenAI( + api_key = api_key, + max_retries=open_ai_max_retries + ) + +if not source_directory: + files_df = pd.DataFrame([{"FileName": "", "SourceDirectory": ""}]) +else: + files = [file for file in os.listdir(source_directory) if os.path.isfile(os.path.join(source_directory, file))] + files = fnfilter(files, source_file) + files_df = pd.DataFrame({"FileName": files}) + files_df["SourceDirectory"] = source_directory[len(LAKEHOUSE_DEFAULT_PREFIX):] + +if api_family=="responses": + activity_settings.setdefault("text",{"format": {"type": "json_object"}}) + prompt = activity_settings["input"] if isinstance(activity_settings.get("input"), str) else next((p["text"] for m in activity_settings["input"] for p in m.get("content", []) if p.get("type") == "input_text"), None) +else: + prompt = next((m["content"] for m in activity_settings.get("messages", []) if m.get("role") == "user"), None) + +if "response_format" in activity_settings: + is_json = activity_settings.get("response_format", {}).get("type").lower().startswith("json") +elif "text" in activity_settings: + is_json = activity_settings.get("text", {}).get("format", {}).get("type").lower().startswith("json") +else: + is_json=False + + +parent_df: Dict[str, pd.DataFrame] = {} +child_df: Dict[str, pd.DataFrame] = {} + +start_time=time.time() +doc_count=0 +file_user_prompt=None + +for index, row in files_df.iterrows(): + doc_count+=1 + file_path = os.path.join(source_directory, row["FileName"]) if source_directory else None + file_name = row["FileName"] + print(f"Processing: {file_name if source_directory else 'Live Web Search or Reasoning'}") + + file_activity_settings=activity_settings + if file_path: + if file_path.lower().endswith(".pdf") and api_family=="responses": + if api_provider=="openai": + file = client.files.create( + file=open(file_path, "rb"), + purpose="assistants" + ) + file_user_prompt= build_structured_input(prompt, file_ids=file.id) + else: #No Azure Open AI Support for file upload yet, so using base 64 encoding + with open(file_path, "rb") as f: + b64_pdf =base64.b64encode(f.read()).decode("utf-8") + content = [{"type": "input_text", "text": prompt}] + content.append({"type": "input_file", "mime_type": "application/pdf", "data": b64_pdf}) + file_user_prompt =[{"role": "user", "content": content }] + else: + with open(file_path, "rb") as f: + file_user_prompt = f"{prompt}{f.read()}" + if api_family=="responses": + file_activity_settings['input']=file_user_prompt + else: + next(m for m in file_activity_settings["messages"] if m.get("role") == "user")["content"] = file_user_prompt + + for model in models: + model=model.strip() + gpt_start_time=time.time() + activity_settings['model']=model + + if api_family=="responses": + response = client.responses.create(**file_activity_settings) + content = response.output_text + + usage=json.dumps(response.usage.to_dict(), indent=2) + else: + response = response = client.chat.completions.create(**file_activity_settings) + content = response.choices[0].message.content + usage=json.dumps(response.usage.to_dict(), indent=2) + + if is_json and table_layout !="content": + content_json=json.loads(content) + else: + content_json={"content":content} + content_json['id']=response.id + content_json['usage']=usage + if file_name!="": + content_json['file_name']=file_name + gpt_elapsed= time.time() -gpt_start_time + content_json['duration_secs']= int(round(gpt_elapsed)) + content_json['model']=model + content_json['created_date']=datetime.now(timezone.utc) + rows.append(content_json) + print(f" - Prompt with Model [{model}] took {gpt_elapsed:.0f} secs") + +rows = [normalize_json_types(r) for r in rows] +df = spark.createDataFrame(rows) +display(df.limit(10)) + + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# CELL ******************** + +from pyspark.sql import functions as F +from pyspark.sql.types import NumericType, FloatType +from pyspark.sql import types as T + +if table_layout == "multiple": + parent_df=df + child_obj={} + array_cols = [f.name for f in df.schema.fields if 'array' in f.dataType.simpleString()] + for item in array_cols: + keys = ( + parent_df + .select(F.explode_outer(item).alias("m")) + .select(F.explode_outer(F.map_keys("m")).alias("k")) + .distinct() + .rdd.map(lambda r: r[0]) + .collect() + ) + item_df = ( + parent_df + .select("id", F.explode_outer(item).alias("item")) + .select( + "id", + *[F.element_at("item", k).alias(k) for k in keys] + ) + ) + child_obj[item]=item_df +else: + parent_df=df + child_obj=None + + +#Write Top Level Table. Eg ai.invoice +doc_start_time=time.time() +parent_df = parent_df.select([ + F.col(c).cast(FloatType()) if isinstance(df.schema[c].dataType, NumericType) else F.col(c) + for c in parent_df.columns +]) +full_table_name = f"{schema_name}.{table_name}" +parent_df=clean_cols(parent_df) +parent_df.write.mode("append").saveAsTable(full_table_name) +elapsed = time.time() - doc_start_time +print(f" - Wrote {parent_df.count()} rows to {full_table_name} in {elapsed:.0f} secs") + + +# Save to LH and Archive FIles if needed +#Write Nested Tables. Eg ai.invoice_items +if child_obj: + doc_start_time=time.time() + for k, v in child_obj.items(): + nested_table_name = f"{schema_name}.{table_name}_{k}" + print (nested_table_name) + if v: + v=clean_cols(v) + v.write.mode("append").saveAsTable(nested_table_name) + elapsed = time.time() - doc_start_time + print(f" - Wrote {v.count()} rows to {nested_table_name} in {elapsed:.0f} secs") + else: + elapsed = time.time() - doc_start_time + +#Archive Files if needed +if source_directory and processed_directory: + for index, row in files_df.iterrows(): + file_name = row["FileName"] + file_path = os.path.join(source_directory[len(LAKEHOUSE_DEFAULT_PREFIX):], file_name) + processed_file_path = os.path.join(processed_directory[len(LAKEHOUSE_DEFAULT_PREFIX):], file_name) + if notebookutils.fs.exists(file_path) and processed_directory: + processed_file_path = os.path.join(processed_directory[len(LAKEHOUSE_DEFAULT_PREFIX):], file_name) + if notebookutils.fs.exists(processed_file_path): + notebookutils.fs.rm(processed_file_path) + if delete: + notebookutils.fs.mv(file_path, processed_file_path, True, True) + else: + notebookutils.fs.cp(file_path, processed_file_path, True) + +print ('') +elapsed = time.time() - start_time +print(f"Completed. Extracted {doc_count} documents in {elapsed:.0f} secs") + + + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } diff --git a/Workspaces/DWA/DWA/Library/AI/Extract-Markdown-EH.Notebook/.platform b/Workspaces/DWA/DWA/Library/AI/Extract-Markdown-EH.Notebook/.platform new file mode 100644 index 0000000..d7c8c1e --- /dev/null +++ b/Workspaces/DWA/DWA/Library/AI/Extract-Markdown-EH.Notebook/.platform @@ -0,0 +1,12 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "Notebook", + "displayName": "Extract-Markdown-EH", + "description": "New notebook" + }, + "config": { + "version": "2.0", + "logicalId": "65753566-9d24-9cd0-4920-6c4058a2fd1c" + } +} \ No newline at end of file diff --git a/Workspaces/DWA/DWA/Library/AI/Extract-Markdown-EH.Notebook/notebook-content.py b/Workspaces/DWA/DWA/Library/AI/Extract-Markdown-EH.Notebook/notebook-content.py new file mode 100644 index 0000000..9f4fb55 --- /dev/null +++ b/Workspaces/DWA/DWA/Library/AI/Extract-Markdown-EH.Notebook/notebook-content.py @@ -0,0 +1,608 @@ +# Fabric notebook source + +# METADATA ******************** + +# META { +# META "kernel_info": { +# META "name": "synapse_pyspark" +# META }, +# META "dependencies": { +# META "lakehouse": { +# META "default_lakehouse": "f2f9c5fa-ca0c-41b2-b0e1-3028165b4f6c", +# META "default_lakehouse_name": "FabricLH", +# META "default_lakehouse_workspace_id": "9b8a6500-5ccb-49a9-885b-b5b081efed75", +# META "known_lakehouses": [ +# META { +# META "id": "f2f9c5fa-ca0c-41b2-b0e1-3028165b4f6c" +# META } +# META ] +# META } +# META } +# META } + +# CELL ******************** + +%pip install openai + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# PARAMETERS CELL ******************** + +SourceSettings = '{"directory": "landing/ai/contracts/markdown"}' +TargetSettings = '{"directory": "landing/ai/contracts/archive", "kustoDatabase":"FabricEH", "kustoTable":"contract", "chunkChars":"32768" }' +SourceConnectionSettings = '{"openAIEndpoint": "https://open-ai-we.openai.azure.com/", "openAISecret" : "open-ai-we-key", "kustoUri":"https://trd-gvbqgkgfycghtwm0z3.z6.kusto.fabric.microsoft.com", "KeyVault": "https://kv-fabric-dev.vault.azure.net/", "model":"text-embedding-3-large"}' +SinkConnectionSettings = '{}' +ActivitySettings='{}' # "ArchiveDirectory": "landing/ai/contracts/Archive/Markdown/" +LineageKey : str = '00000000-0000-0000-0000-000000000000' +RunId : str = '00000000-0000-0000-0000-000000000000' + + + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# CELL ******************** + +import json +from operator import itemgetter +import sempy.fabric as fabric + +#Settings +source_connection_settings = json.loads(SourceConnectionSettings or "{}") +keyvault,openai_endpoint, openai_secret,kustoUri = itemgetter("KeyVault", "openAIEndpoint", "openAISecret","kustoUri")(json.loads(SourceConnectionSettings or "{}")) +source_lakehouse_id = source_connection_settings.get("lakehouseId",fabric.get_lakehouse_id()) +source_workspace_id = source_connection_settings.get("workspaceId",fabric.get_workspace_id()) +source_lakehouse_name = source_connection_settings.get("lakehouse",fabric.resolve_item_name(item_id=source_lakehouse_id, workspace=source_workspace_id)) +source_workspace_name = fabric.resolve_workspace_name(source_workspace_id) +source_model = source_connection_settings.get("workspaceId","text-embedding-3-large") + +source_settings = json.loads(SourceSettings or "{}") +delete = bool(source_settings.get("delete", False)) +source_directory = source_settings["directory"] + +target_settings = json.loads(TargetSettings or "{}") +processed_directory = target_settings.get("directory", "") +table_name = target_settings['kustoTable'] +kustoDatabase= target_settings['kustoDatabase'] +CHUNK_CHARS = int(target_settings.get("chunkChars",32768)) + + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# CELL ******************** + +import json +import time +import uuid +import os +import re +import fsspec +from typing import List, Tuple, Dict, Optional +import pandas as pd +from openai import AzureOpenAI +import requests +from pathlib import PurePath +from azure.core.credentials import AzureKeyCredential,AccessToken + +DIM = 3072 # 1536 for ada-002, 3072 for text-embedding-3-large +#CHUNK_CHARS = 32768 #Parameterisd into TargetSettings Set this for smaller chunks. +CHUNK_OVERLAP = 100 +DO_EMBED = True +EMBED_MODEL = source_model +EMBED_DIM = 3072 +EMBED_BATCH = 32 +TABLE_DOCS = f"{table_name}_docs" +TABLE_CHUNKS = f"{table_name}_chunks" + + +if DO_EMBED: + from openai import AzureOpenAI + client = AzureOpenAI( + api_key = notebookutils.credentials.getSecret(keyvault, openai_secret), + api_version = "2024-02-01", + azure_endpoint = openai_endpoint + ) + + +class CustomTokenCredential: + def get_token(self, *scopes, **kwargs): + return AccessToken(notebookutils.credentials.getToken('storage'), expires_on=int(time.time()) + 3000) + +def _mgmt(url): + return f"{url.rstrip('/')}/v1/rest/mgmt" + +def _query(url): + return f"{url.rstrip('/')}/v1/rest/query" + +def get_existing_doc_ids() -> Dict[str, str]: + token = notebookutils.credentials.getToken("kusto") + url = _query(kustoUri) + csl = f"{TABLE_DOCS} | project file_name, doc_id" + r = requests.post( + url, + headers={"Authorization": f"Bearer {token}", "Accept": "application/json"}, + json={"db": kustoDatabase, "csl": csl}, + timeout=60, + ) + r.raise_for_status() + result = r.json() + doc_id_map = {} + if "Tables" in result and result["Tables"]: + table = result["Tables"][0] + rows = table.get("Rows", []) + for row in rows: + if len(row) >= 2: + file_name, doc_id = row[0], row[1] + if file_name and doc_id: + doc_id_map[file_name] = doc_id + + print(f"Found {len(doc_id_map)} existing doc_id mappings") + return doc_id_map + + +def soft_delete_by_file_name(table: str, file_name: str, whatif: bool = False): + token = notebookutils.credentials.getToken("kusto") + url = _mgmt(kustoUri) + safe = file_name.replace('"', '\\"') + opts = "with (whatif=true) " if whatif else "" + + csl = f'''.delete table {table} records {opts}<| {table} | where file_name == "{safe}"''' + r = requests.post( + url, + headers={"Authorization": f"Bearer {token}", "Accept": "application/json"}, + json={"db": kustoDatabase, "csl": csl}, + timeout=60, + ) + r.raise_for_status() + return r.json() + +H_ATX = re.compile(r'^(?P#{1,6})\s+(?P.+?)(?:\s+#+\s*)?$', re.MULTILINE) + +def find_setext_headings(text: str) -> List[Tuple[int, str, int]]: + results = [] + lines = text.splitlines(keepends=True) + i, pos = 0, 0 + while i < len(lines) - 1: + title = lines[i].rstrip("\r\n") + underline = lines[i+1].rstrip("\r\n") + if title.strip() and underline and set(underline) <= {"="}: + results.append((1, title.strip(), pos)) + i += 2; pos += len(lines[i-2]) + len(lines[i-1]) + elif title.strip() and underline and set(underline) <= {"-"}: + results.append((2, title.strip(), pos)) + i += 2; pos += len(lines[i-2]) + len(lines[i-1]) + else: + pos += len(lines[i]); i += 1 + return results + +H_ATX = re.compile(r'^(?P<hashes>#{1,6})\s+(?P<title>.+?)(?:\s+#+\s*)?$', re.MULTILINE) + +def parse_headings(md: str) -> List[Tuple[int, str, int]]: + atx = [(len(m.group("hashes")), m.group("title").strip(), m.start()) + for m in H_ATX.finditer(md)] + setext = find_setext_headings(md) + all_h = atx + setext + all_h.sort(key=lambda x: x[2]) + return all_h + +# Regex to detect appendix/exhibit patterns +APPENDIX_EXHIBIT_RE = re.compile(r'^\s*(APPENDIX|EXHIBIT|Appendix|Exhibit)\s+([A-Za-z0-9]+|[IVXLCDMivxlcdm]+)\b', re.IGNORECASE) + +def is_appendix_or_exhibit(title: str) -> bool: + """Check if a title indicates an appendix or exhibit section.""" + match_result = bool(APPENDIX_EXHIBIT_RE.match(title.strip())) + return match_result + +def classify_section_type(title: str) -> str: + t = (title or "").strip() + tl = t.lower() + if tl.startswith("schedule "): return "schedule" + if tl.startswith("annex "): return "annex" + if "table of contents" in tl or tl == "contents": return "toc" + if is_appendix_or_exhibit(title): return "appendix" + return "clause" + +NUM_PREFIX_RE = re.compile(r'^(?P<num>(?:\d+(?:\.\d+)*|[A-Z]\.|[a-z]\.)|(?:Schedule|Annex)\s+\w+)', re.IGNORECASE) + +def split_section_number_and_title(title: str) -> Tuple[str, str]: + t = (title or "").strip() + m = NUM_PREFIX_RE.match(t) + if m: + num = m.group("num").rstrip(" .:-") + rest = t[m.end():].lstrip(" .:-") + return num, rest or t + return "", t + +def find_page_breaks(text: str) -> List[int]: + page_break_pattern = re.compile(r'<!--\s*PageBreak\s*-->', re.IGNORECASE) + positions = [match.start() for match in page_break_pattern.finditer(text)] + return positions + +def get_page_number_from_position(char_position: int, page_break_positions: List[int]) -> int: + if not page_break_positions: + return 1 + + page_num = 1 + for break_pos in page_break_positions: + if char_position >= break_pos: + page_num += 1 + else: + break + return page_num + +def get_total_pages_from_breaks(page_break_positions: List[int]) -> int: + return len(page_break_positions) + 1 + +def sections_from_markdown(md: str, page_break_positions: List[int]) -> List[Dict]: + heads = parse_headings(md) # [(level, title, start_index), ...] + total_pages = get_total_pages_from_breaks(page_break_positions) + + if not heads: + return [{ + "section_id": 0, + "level": 0, + "title": "Document", + "path": "Document", + "type": "other", + "start": 0, + "end": len(md), + "text": md, + "sec_number": "", + "page_start": 1, + "page_end": total_pages + }] + + sections = [] + section_id_counter = 0 + + # Check if there's content before the first heading + if heads and heads[0][2] > 0: # If first heading doesn't start at position 0 + # Extract content before first heading + first_heading_pos = heads[0][2] + pre_heading_text = md[:first_heading_pos].strip() + + # Only create a section if there's meaningful content (not just whitespace/comments) + if pre_heading_text and len(pre_heading_text.replace('<!--', '').replace('-->', '').strip()) > 0: + page_start = get_page_number_from_position(0, page_break_positions) + page_end = get_page_number_from_position(first_heading_pos, page_break_positions) + + sections.append({ + "section_id": section_id_counter, + "level": 1, + "title": "Agreement", + "sec_number": "", + "start": 0, + "end": first_heading_pos, + "text": md[:first_heading_pos], + "type": "clause", + "page_start": page_start, + "page_end": page_end + }) + section_id_counter += 1 + + # Process regular headings with appendix consolidation + i = 0 + while i < len(heads): + lvl, title, start = heads[i] + + # Find the end of this section + end = len(md) # Default to end of document + + # Check if this is an appendix/exhibit + section_type = classify_section_type(title) + + if section_type == "appendix": + # For appendices, include all subsections until we hit the next same-level or higher section + # or another appendix + j = i + 1 + while j < len(heads): + next_lvl, next_title, next_start = heads[j] + next_type = classify_section_type(next_title) + + # Stop if we hit another appendix, or a section at same/higher level + if next_type == "appendix" or next_lvl <= lvl: + end = next_start + break + j += 1 + + if j == len(heads): # We reached the end + end = len(md) + else: + # Regular section - find next section at same or higher level + j = i + 1 + while j < len(heads): + next_lvl, next_title, next_start = heads[j] + if next_lvl <= lvl: + end = next_start + break + j += 1 + + if j == len(heads): # We reached the end + end = len(md) + + # Extract the full text for this section (including any subsections for appendices) + text = md[start:end] + sec_num, clean_title = split_section_number_and_title(title) + + page_start = get_page_number_from_position(start, page_break_positions) + page_end = get_page_number_from_position(end, page_break_positions) + + sections.append({ + "section_id": section_id_counter, + "level": lvl, + "title": clean_title, + "sec_number": sec_num, + "start": start, + "end": end, + "text": text, + "type": section_type, + "page_start": page_start, + "page_end": page_end + }) + section_id_counter += 1 + + # Skip the subsections we've already included in appendices + if section_type == "appendix": + # Skip all the headers we consumed + while i + 1 < len(heads) and heads[i + 1][2] < end: + i += 1 + + i += 1 + + # Build hierarchical paths + stack = [] + for s in sections: + while stack and stack[-1]["level"] >= s["level"]: + stack.pop() + stack.append({"level": s["level"], "title": s["title"]}) + s["path"] = " › ".join(x["title"] for x in stack) + + return sections + +def chunk_within_section(text: str, size: int, overlap: int, section_start_char: int, page_break_positions: List[int], section_type: str = "clause") -> List[Dict]: + + # For appendices/exhibits, return the entire section as one chunk + if section_type == "appendix": + page_num = get_page_number_from_position(section_start_char, page_break_positions) + result = [{"text": text, "page_number": page_num}] + + return result + + # Regular chunking for non-appendix sections + if size <= 0: + page_num = get_page_number_from_position(section_start_char, page_break_positions) + return [{"text": text, "page_number": page_num}] + + chunks = [] + i = 0 + step = max(size - overlap, 1) + + while i < len(text): + chunk_text = text[i:i+size] + # Get page number for this chunk based on its position in the overall document + chunk_char_position = section_start_char + i + page_number = get_page_number_from_position(chunk_char_position, page_break_positions) + + chunks.append({ + "text": chunk_text, + "page_number": page_number + }) + i += step + + return chunks + +def approx_tokens(s: str) -> int: + # quick proxy if you don't run a tokenizer: ~4 chars per token + return max(1, int(len(s) / 4)) + +def embed_batch(texts: List[str]) -> List[List[float]]: + resp = client.embeddings.create(model=EMBED_MODEL, input=texts) + return [d.embedding for d in resp.data] + +def batched(seq, n): + for i in range(0, len(seq), n): + yield seq[i:i+n] + + +source_abfss = f"abfss://{source_workspace_id}@onelake.dfs.fabric.microsoft.com/{source_lakehouse_id}/Files" +source_path = os.path.join(source_abfss, source_directory) +if not notebookutils.fs.exists(source_path) : + notebookutils.notebook.exit(0) # no folder to process, quit + +files = notebookutils.fs.ls(source_path) +files = [ + fi for fi in files + if int(getattr(fi, "size", 0) or 0) > 0 + and str(getattr(fi, "name", getattr(fi, "path", getattr(fi, "__str__", lambda: str(fi))()))).lower().endswith(".md") +] + +existing_doc_ids = get_existing_doc_ids() +fs = fsspec.filesystem("abfss", account_name="onelake" , account_host="onelake.dfs.fabric.microsoft.com", credential=CustomTokenCredential()) +docs_rows = [] +chunks_rows = [] +chunk_id=1 +for file in files: + file_path = file.path + file_name = PurePath(file_path).name + print(f"Processing: \"{file_name}\"") + start_time = time.perf_counter() + + with fs.open(file_path, "rb") as f: + md = f.read() + if isinstance(md, bytes): + md = md.decode("utf-8", errors="ignore") + + doc_id = existing_doc_ids.get(file_name, str(uuid.uuid4())) + if file_name in existing_doc_ids: + print(f" - Reusing existing doc_id: {doc_id}") + else: + print(f" - Generated new doc_id: {doc_id}") + + title = os.path.splitext(file_name)[0] + + page_break_positions = find_page_breaks(md) + total_pages = get_total_pages_from_breaks(page_break_positions) + + sections = sections_from_markdown(md,page_break_positions) + + doc_chunks = [] + appendix_sections = [] + for s in sections: + sec_text = s["text"] + + sec_chunks = chunk_within_section( + sec_text, + CHUNK_CHARS, + CHUNK_OVERLAP, + s["start"], + page_break_positions, + s["type"] # Pass section type to chunking function + ) + + + for idx, chunk_info in enumerate(sec_chunks): + chunk_row = { + "doc_id": doc_id, + "file_name": file_name, + "section_number": s["sec_number"], + "section_title": s["title"], + "section_level": s["level"], + "section_path": s["path"], + "section_type": s["type"], + "page_start": s.get("page_start", 1), + "page_end": s.get("page_end", 1), + "chunk_id": chunk_id, + "chunk_text": chunk_info["text"], + "chunk_tokens": approx_tokens(chunk_info["text"]), + "embedding": None + } + doc_chunks.append(chunk_row) + chunk_id+=1 + + if DO_EMBED and doc_chunks: + print(f" - Generating embeddings for {len(doc_chunks)} chunks...") + for batch in batched(doc_chunks, EMBED_BATCH): + embs = embed_batch([c["chunk_text"] for c in batch]) + for c, e in zip(batch, embs): + c["embedding"] = e + print(f"\n=== SUMMARY for {file_name} ===") + appendix_chunks = [c for c in doc_chunks if c["section_type"] == "appendix"] + regular_chunks = [c for c in doc_chunks if c["section_type"] != "appendix"] + + print(f"Total sections: {len(sections)}") + print(f"Regular chunks: {len(regular_chunks)}") + print(f"Appendix chunks: {len(appendix_chunks)}") + chunks_rows.extend(doc_chunks) + + docs_rows.append({ + "doc_id": doc_id, + "file_name": file_name, + "title": title, + "pages": str(total_pages) + }) + + elapsed = time.perf_counter() - start_time + print(f" - Processed in {elapsed:.2f} secs, {len(doc_chunks)} chunks created") + + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } + +# CELL ******************** + +from pyspark.sql.types import ( + StructType, StructField, + StringType, IntegerType, LongType +) +from pyspark.sql import functions as F + + +for row in chunks_rows: + v = row.get("embedding") + if v is not None and not isinstance(v, str): + row["embedding"] = json.dumps(v, ensure_ascii=False) + +docs_schema = StructType([ + StructField("doc_id", StringType(), True), + StructField("file_name", StringType(), True), + StructField("title", StringType(), True), + StructField("pages", StringType(), True), +]) + +chunks_schema = StructType([ + StructField("doc_id", StringType(), True), + StructField("file_name", StringType(), True), + StructField("section_number", StringType(), True), + StructField("section_title", StringType(), True), + StructField("section_level", IntegerType(), True), + StructField("section_path", StringType(), True), + StructField("section_type", StringType(), True), + StructField("page_start", LongType(), True), + StructField("page_end", LongType(), True), + StructField("chunk_id", StringType(), True), + StructField("chunk_text", StringType(), True), + StructField("chunk_tokens", IntegerType(), True), + StructField("embedding", StringType(), True), # JSON text for ADX dynamic +]) + +docs_df = spark.createDataFrame(docs_rows, schema=docs_schema) +chunks_df = spark.createDataFrame(chunks_rows, schema=chunks_schema) +print("Cleaning up existing records...") +for row in docs_df.select("file_name").distinct().toLocalIterator(): + fn = row["file_name"] + print(f" - Deleting existing records for: {fn}") + soft_delete_by_file_name(TABLE_CHUNKS, fn) + soft_delete_by_file_name(TABLE_DOCS, fn) + + +if "embedding" in chunks_df.columns and not isinstance(chunks_df.schema["embedding"].dataType, StringType): + chunks_df = chunks_df.withColumn("embedding", F.to_json(F.col("embedding"))) + +# 3) Write to ADX/Eventhouse using Current Identity (best for Fabric) +print("Writing to Kusto...") +accessToken = notebookutils.credentials.getToken('kusto') + +(docs_df.write +.format("com.microsoft.kusto.spark.datasource") +.option("kustoCluster", kustoUri) +.option("kustoDatabase", kustoDatabase) +.option("kustoTable", TABLE_DOCS) +.option("accessToken", accessToken ) +.mode("Append") +.save()) + +(chunks_df.write +.format("com.microsoft.kusto.spark.datasource") +.option("kustoCluster", kustoUri) +.option("kustoDatabase", kustoDatabase) +.option("kustoTable", TABLE_CHUNKS) +.option("accessToken", accessToken ) +.mode("Append") +.save()) + +print(f"Successfully wrote docs={docs_df.count()} rows, chunks={chunks_df.count()} rows to Eventhouse.") + +# METADATA ******************** + +# META { +# META "language": "python", +# META "language_group": "synapse_pyspark" +# META } diff --git a/Workspaces/DWA/DWA/Library/Extract/Extract-SQL.DataPipeline/pipeline-content.json b/Workspaces/DWA/DWA/Library/Extract/Extract-SQL.DataPipeline/pipeline-content.json index 1af31bb..2033fc0 100644 --- a/Workspaces/DWA/DWA/Library/Extract/Extract-SQL.DataPipeline/pipeline-content.json +++ b/Workspaces/DWA/DWA/Library/Extract/Extract-SQL.DataPipeline/pipeline-content.json @@ -42,9 +42,9 @@ "value": "@if(\n contains(\n pipeline().parameters.TargetSettings, 'tableActionOption'\n )\n ,json(string(pipeline().parameters.TargetSettings)).tableActionOption\n ,'OverwriteSchema'\n )", "type": "Expression" }, + "applyVOrder": true, "upsertSettings": {}, "partitionOption": "None", - "applyVOrder": true, "datasetSettings": { "type": "LakehouseTable", "typeProperties": { @@ -59,7 +59,7 @@ }, "schema": [], "linkedService": { - "name": "75d4ebf8_bfa8_472b_8c5d_36bcb00f3e23", + "name": "3076a8b4_b5cf_47f7_b8c9_02f486b6230a", "properties": { "type": "Lakehouse", "typeProperties": { @@ -124,7 +124,7 @@ "workspaceId": "00000000-0000-0000-0000-000000000000" }, "externalReferences": { - "connection": "935d976f-470e-42dc-9ee1-97ef97a281a4" + "connection": "51ba5474-a810-44e4-9a9e-fbcc7892954f" }, "annotations": [] } diff --git a/Workspaces/DWA/DWA/Ops/Extract-Artefacts.Notebook/notebook-content.py b/Workspaces/DWA/DWA/Ops/Extract-Artefacts.Notebook/notebook-content.py index 2e8a1f8..1a030fa 100644 --- a/Workspaces/DWA/DWA/Ops/Extract-Artefacts.Notebook/notebook-content.py +++ b/Workspaces/DWA/DWA/Ops/Extract-Artefacts.Notebook/notebook-content.py @@ -82,18 +82,6 @@ df=df.rename(columns=dict(zip(df.columns, [re.sub(pattern, '_', col.strip(pattern).lower()) for col in df.columns]))) -# METADATA ******************** - -# META { -# META "language": "python", -# META "language_group": "synapse_pyspark" -# META } - -# CELL ******************** - -#Write to Lakehouse -spark.createDataFrame(df).write.mode("overwrite").saveAsTable("dict_artefacts") - # METADATA ******************** # META { diff --git a/Workspaces/DWA/Meta.SQLDatabase/Meta.sqlproj b/Workspaces/DWA/Meta.SQLDatabase/Meta.sqlproj index d4e0a3c..232f7ea 100644 --- a/Workspaces/DWA/Meta.SQLDatabase/Meta.sqlproj +++ b/Workspaces/DWA/Meta.SQLDatabase/Meta.sqlproj @@ -1,12 +1,19 @@ <?xml version="1.0" encoding="utf-8"?> <Project DefaultTargets="Build"> - <Sdk Name="Microsoft.Build.Sql" Version="2.0.0-preview.2" /> + <Sdk Name="Microsoft.Build.Sql" Version="2.0.0-preview.5" /> <PropertyGroup> <Name>Meta</Name> <ProjectGuid>{00000000-0000-0000-0000-000000000000}</ProjectGuid> <DSP>Microsoft.Data.Tools.Schema.Sql.SqlDbFabricDatabaseSchemaProvider</DSP> <ModelCollation>1033, CI</ModelCollation> </PropertyGroup> + <ItemGroup> + <PackageReference Include="Microsoft.SqlServer.Dacpacs.DbFabric"> + <SuppressMissingDependenciesErrors>False</SuppressMissingDependenciesErrors> + <DatabaseVariableLiteralValue>master</DatabaseVariableLiteralValue> + <Version>170.0.0</Version> + </PackageReference> + </ItemGroup> <Target Name="BeforeBuild"> <Delete Files="$(BaseIntermediateOutputPath)\project.assets.json" /> </Target> diff --git a/Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_PipelineBuildMetaData.sql b/Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_PipelineBuildMetaData.sql index 7e281e9..138d895 100644 --- a/Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_PipelineBuildMetaData.sql +++ b/Workspaces/DWA/Meta.SQLDatabase/config/StoredProcedures/usp_PipelineBuildMetaData.sql @@ -9,6 +9,7 @@ History: 10/03/2024 Kristan, added SessionTag 25/04/2025 Kristan, added LakehouseConnectionSettings 29/06/2025 Bob, Added support for multiple workspaces for Templates + 09/11/2025 Bob, Added suport for ai.Prompts */ CREATE PROC [config].[usp_PipelineBuildMetaData] @PipelineID int =null @@ -107,7 +108,7 @@ BEGIN ,@Enabled = CASE WHEN p.Enabled = 0 OR pg.Enabled = 0 THEN 0 ELSE 1 END ,@Stage = COALESCE(pg.Stage, p.Stage) ,@StartDateTime = GETDATE() - ,@ActivityType = CASE WHEN LEFT(LTRIM(COALESCE(p.ActivitySettings, pg.ActivitySettings)),1) = '{' THEN 'JSON' WHEN LEFT(LTRIM(COALESCE(p.ActivitySettings, pg.ActivitySettings)),1) = '<' THEN 'XML' ELSE 'OTHER' END + ,@ActivityType = CASE WHEN LEFT(LTRIM(COALESCE(p.ActivitySettings, pg.ActivitySettings)),1) = '{' and p.PromptID is null THEN 'JSON' WHEN LEFT(LTRIM(COALESCE(p.ActivitySettings, pg.ActivitySettings)),1) = '<' THEN 'XML' ELSE 'OTHER' END ,@TemplateType= t.ArtefactType ,@ID=a.artefact_id ,@TableID = p.TableID diff --git a/Workspaces/DWA/Meta.SQLDatabase/config/Tables/Configurations.sql b/Workspaces/DWA/Meta.SQLDatabase/config/Tables/Configurations.sql index 748c0f3..1735c74 100644 --- a/Workspaces/DWA/Meta.SQLDatabase/config/Tables/Configurations.sql +++ b/Workspaces/DWA/Meta.SQLDatabase/config/Tables/Configurations.sql @@ -2,7 +2,7 @@ CREATE TABLE [config].[Configurations] ( [ConfigurationID] INT NOT NULL, [ConfigurationName] VARCHAR (8000) NULL, [ConnectionSettings] VARCHAR (8000) NULL, - [Enabled] INT NULL, + [Enabled] INT CONSTRAINT [DF_Configurations_Enabled] DEFAULT ((1)) NOT NULL, CONSTRAINT [PK_Configurations] PRIMARY KEY CLUSTERED ([ConfigurationID] ASC) ); diff --git a/Workspaces/DWA/Meta.SQLDatabase/config/Tables/Pipelines.sql b/Workspaces/DWA/Meta.SQLDatabase/config/Tables/Pipelines.sql index 50e947e..32ca0ec 100644 --- a/Workspaces/DWA/Meta.SQLDatabase/config/Tables/Pipelines.sql +++ b/Workspaces/DWA/Meta.SQLDatabase/config/Tables/Pipelines.sql @@ -6,7 +6,7 @@ CREATE TABLE [config].[Pipelines] ( [SourceSettings] VARCHAR (8000) NULL, [TargetSettings] VARCHAR (8000) NULL, [ActivitySettings] VARCHAR (8000) NULL, - [Enabled] INT NULL, + [Enabled] INT CONSTRAINT [DF_Pipelines_Enabled] DEFAULT ((1)) NOT NULL, [PackageGroup] VARCHAR (50) NULL, [TableID] SMALLINT NULL, [Stage] VARCHAR (50) NULL, @@ -21,6 +21,7 @@ CREATE TABLE [config].[Pipelines] ( [Comments] VARCHAR (200) NULL, [SessionTag] VARCHAR (50) NULL, [LakehouseConfigurationID] INT NULL, + [PromptID] INT NULL, CONSTRAINT [PK_Pipelines] PRIMARY KEY CLUSTERED ([PipelineID] ASC), CONSTRAINT [FK_Pipelines_Configurations] FOREIGN KEY ([SourceConfigurationID]) REFERENCES [config].[Configurations] ([ConfigurationID]), CONSTRAINT [FK_Pipelines_PackageGroups] FOREIGN KEY ([PackageGroup]) REFERENCES [config].[PackageGroups] ([PackageGroup]), @@ -31,14 +32,19 @@ CREATE TABLE [config].[Pipelines] ( GO - - -CREATE TRIGGER [config].TR_Pipelines +CREATE TRIGGER [config].[TR_Pipelines] ON [config].[Pipelines] AFTER INSERT,DELETE,UPDATE AS BEGIN SET NOCOUNT ON; + + UPDATE p + SET p.ActivitySettings = ai.ActivitySettings + FROM config.Pipelines p + INNER JOIN inserted i on i.PipelineID=p.PipelineID + INNER JOIN config.aiPrompts ai on ai.PromptID = p.PromptID + exec [config].[usp_PipelineBuildMetaData] END diff --git a/Workspaces/DWA/Meta.SQLDatabase/config/Tables/aiPrompts.sql b/Workspaces/DWA/Meta.SQLDatabase/config/Tables/aiPrompts.sql new file mode 100644 index 0000000..a52fe82 --- /dev/null +++ b/Workspaces/DWA/Meta.SQLDatabase/config/Tables/aiPrompts.sql @@ -0,0 +1,52 @@ +CREATE TABLE [config].[aiPrompts] ( + [PromptID] INT NOT NULL, + [prompt_name] VARCHAR (8000) NULL, + [description] VARCHAR (8000) NULL, + [instructions] VARCHAR (MAX) NULL, + [input] VARCHAR (MAX) NULL, + [model] VARCHAR (8000) NULL, + [json_schema] VARCHAR (MAX) NULL, + [ai_endpoint] VARCHAR (8000) NULL, + [api_family] VARCHAR (8000) CONSTRAINT [DF_aiPrompts_api_family] DEFAULT ('responses') NOT NULL, + [api_provider] VARCHAR (8000) CONSTRAINT [DF_aiPrompts_api_provider] DEFAULT ('azure_openai') NOT NULL, + [temperature] NUMERIC (3, 2) CONSTRAINT [DF_aiPrompts_temperature] DEFAULT ((0.3)) NULL, + [SourceConnnectionID] INT NULL, + [ExtraSettings] VARCHAR (MAX) NULL, + [ActivitySettings] VARCHAR (MAX) NULL, + CONSTRAINT [PK_aiPrompts] PRIMARY KEY CLUSTERED ([PromptID] ASC) +); + + +GO + +/* + Update Activiy Settings for AI Prompts + +*/ +CREATE TRIGGER [config].[TR_aiPrompt_Update] + ON [config].[aiPrompts] + AFTER UPDATE, INSERT +AS +BEGIN + SET NOCOUNT ON; + + UPDATE p + SET p.ActivitySettings = dbo.ufn_BuildChatPayloadJson ( + i.model, i.api_family, i.instructions, i.input, + i.json_schema, i.temperature, i.ExtraSettings + ) + FROM config.aiPrompts AS p + JOIN inserted AS i + ON i.PromptID = p.PromptID; + + UPDATE p + set p.ActivitySettings= ai.ActivitySettings + FROM config.Pipelines p + INNER JOIN inserted i on i.PromptID = p.PromptID + INNER JOIN config.aiPrompts ai on ai.PromptID=i.PromptiD + + +END + +GO + diff --git a/Workspaces/DWA/Meta.SQLDatabase/dbo/Functions/ufn_BuildChatPayloadJson.sql b/Workspaces/DWA/Meta.SQLDatabase/dbo/Functions/ufn_BuildChatPayloadJson.sql new file mode 100644 index 0000000..34eb120 --- /dev/null +++ b/Workspaces/DWA/Meta.SQLDatabase/dbo/Functions/ufn_BuildChatPayloadJson.sql @@ -0,0 +1,83 @@ +/* + Retutn the JSon Payload for Both the v1 completions API or the V2 Responses API for AI LLM Calls as per: +https://platform.openai.com/docs/api-reference/completions +https://platform.openai.com/docs/api-reference/responses + + +History: 06/11/2025, Bob, Completed + +*/ +CREATE FUNCTION [dbo].[ufn_BuildChatPayloadJson]( + @model varchar(4000)=null + ,@api_family varchar (4000) = 'chat' /* chat | responses */ + ,@instructions varchar(max)=null + ,@input varchar(max)=null + ,@json_schema varchar(max)=null + ,@temperature float=null + ,@ExtraSettings varchar(max)=null +) +RETURNS varchar(max) +WITH SCHEMABINDING +AS +BEGIN + DECLARE @Json varchar(max) + +;WITH keys AS ( + SELECT 'model' AS [key], @model AS [value], '1' AS [type] WHERE @model IS NOT NULL + UNION ALL + SELECT 'messages' AS [key], + '[ ' + + COALESCE('{"role": "system", "content": "' + @instructions + '"},', '') + + '{"role": "user", "content": "' + @input + '"}' + + ']' AS [value], + '4' AS [type] + WHERE @input IS NOT NULL AND @api_family='chat' + UNION ALL + SELECT 'response_format' AS [key], + '{"type": "json_schema","json_schema":' + @json_schema + '}' AS [value], + '4' AS [type] + WHERE @json_schema IS NOT NULL and @api_family='chat' + UNION ALL + SELECT 'text' AS [key], '{"format": {"type": "json_schema","name": "text_schema","schema": ' + @json_schema+ ',"strict": true}}' + , '4' AS [type] + WHERE @json_schema IS NOT NULL AND @api_family<>'chat' + UNION ALL + SELECT 'instructions' AS [key], @instructions, '1' as [type] + WHERE @instructions IS NOT NULL and @api_family<>'chat' + UNION ALL + SELECT 'input' AS [key], @input, '1' as [type] + WHERE @input IS NOT NULL and @api_family<>'chat' + UNION ALL + SELECT 'temperature' AS [key], CONVERT(varchar, @temperature) AS [value], '2' AS [type] + WHERE @temperature IS NOT NULL + +) +SELECT @Json= ( + SELECT '{' + + STRING_AGG( + '"' + s.[key] + '":' + + CASE + WHEN s.[type] = '1' THEN '"' + s.[value] + '"' COLLATE SQL_Latin1_General_CP1_CI_AS + WHEN s.[type] = '0' THEN 'null' + ELSE s.[value] COLLATE SQL_Latin1_General_CP1_CI_AS + END + , ',' + ) + + '}' as b + FROM ( + SELECT k.[key], k.[value], k.[type] + FROM keys k + + UNION ALL + + SELECT j.[key], j.[value], j.[type] + FROM OPENJSON(@ExtraSettings) AS j + LEFT JOIN keys k ON k.[key] = j.[key] + WHERE k.[key] IS NULL + ) s +) +RETURN @Json +END + +GO + diff --git a/Workspaces/DWA/Meta.SQLDatabase/dbo/Tables/dict_artefacts.sql b/Workspaces/DWA/Meta.SQLDatabase/dbo/Tables/dict_artefacts.sql index 9ad2513..81f7bff 100644 --- a/Workspaces/DWA/Meta.SQLDatabase/dbo/Tables/dict_artefacts.sql +++ b/Workspaces/DWA/Meta.SQLDatabase/dbo/Tables/dict_artefacts.sql @@ -3,7 +3,8 @@ CREATE TABLE [dbo].[dict_artefacts] ( [display_name] VARCHAR (MAX) NULL, [description] VARCHAR (MAX) NULL, [type] VARCHAR (MAX) NULL, - [workspace_id] VARCHAR (MAX) NULL + [workspace_id] VARCHAR (MAX) NULL, + [folder_id] VARCHAR (MAX) NULL );