From bfd48514fc4631bce64dd2cfaeac44350ce482e2 Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Mon, 14 Jul 2025 17:15:46 -0700 Subject: [PATCH 01/22] Add naming conventions table and anchors to guideline blocks --- .../Views/Home/Guidelines.cshtml | 40 +- EssentialCSharp.Web/wwwroot/css/styles.css | 22 ++ EssentialCSharp.Web/wwwroot/js/site.js | 358 +++++++++--------- 3 files changed, 246 insertions(+), 174 deletions(-) diff --git a/EssentialCSharp.Web/Views/Home/Guidelines.cshtml b/EssentialCSharp.Web/Views/Home/Guidelines.cshtml index 87a5387a..a9de4307 100644 --- a/EssentialCSharp.Web/Views/Home/Guidelines.cshtml +++ b/EssentialCSharp.Web/Views/Home/Guidelines.cshtml @@ -8,10 +8,48 @@

@ViewData["Title"]

+
+

+ C# Naming Conventions - Quick Reference Table + +

+
+ + + + + + + + + + + + + + + + + + + + + +
KindNaming ConventionExample
ClassesPascalCaseclass Car
Types and NamespacesPascalCasenamespace VehicleManufacturer;
ParameterscamelCasepublic Car(int odometerMileage, string manufacturer)
MethodsPascalCasepublic void StartEngine()
PropertiesPascalCasepublic double FuelLevel { get; set; }
Local VariablescamelCaseint yearManufactured;
Local FunctionsPascalCasestring CalculateMilesUntilEmpty(double fuelLevel)
Fields_PascalCaseprivate string _Day;
Enum MembersPascalCaseenum Status { Unknown, Operational, Broken, InShop }
Type ParametersTPascalCasepublic TOutput Convert<TInput, TOutput>(TInput from)
InterfacesIPascalCaseinterface ISampleInterface
+
+
+
@foreach (var group in guidelines.GroupBy(g => g.SanitizedSubsection).OrderBy(g => g.Key)) { -

@group.Key

+

+ @group.Key + +

foreach (var guideline in group) {
diff --git a/EssentialCSharp.Web/wwwroot/css/styles.css b/EssentialCSharp.Web/wwwroot/css/styles.css index f0b46c6c..fe346fb6 100644 --- a/EssentialCSharp.Web/wwwroot/css/styles.css +++ b/EssentialCSharp.Web/wwwroot/css/styles.css @@ -788,6 +788,28 @@ details > summary::-webkit-details-marker { border-color: var(--grey-lighten-2) transparent transparent transparent; } +/* Anchor Styling */ +.heading-wrapper:not(:hover) .anchor-link:not(:focus-visible) { + opacity: 0; +} + +.anchor-link { + border: none; + color: var(--link-color); + text-decoration: none; + position: absolute; + font-size: 14px; + margin: 4px 2px; + transition-duration: 0.4s; + cursor: pointer; + background-color: transparent; +} + + .anchor-link:hover { + color: var(--link-color-hover); + } + + /* The snackbar - position it at the bottom and in the middle of the screen */ #snackbar { visibility: hidden; /* Hidden by default. Visible on click */ diff --git a/EssentialCSharp.Web/wwwroot/js/site.js b/EssentialCSharp.Web/wwwroot/js/site.js index 5d57eb98..ee6313c4 100644 --- a/EssentialCSharp.Web/wwwroot/js/site.js +++ b/EssentialCSharp.Web/wwwroot/js/site.js @@ -1,118 +1,118 @@ -import { - createApp, - ref, - reactive, - onMounted, - markRaw, - watch, - computed, -} from "vue"; -import { useWindowSize } from "vue-window-size"; - -/** - * @typedef {Object} TocItem - * @prop {number} [level] - * @prop {string} [key] - * @prop {string} [href] - * @prop {string} [title] - * @prop {TocItem[]} [items] - */ -/** @type {TocItem} */ -const tocData = markRaw(TOC_DATA); - -//Add new content or features here: - -const featuresComingSoonList = [ - { - title: "Client-side Compiler", - text: "Write, compile, and run code snippets right from your browser. Enjoy hands-on experience with the code as you go through the site.", - }, - { - title: "Interactive Code Listings", - text: "Edit, compile, and run the code listings found throughout Essential C#.", - }, - { - title: "Hyperlinking", - text: "Easily navigate to interesting and relevant sites as well as related sections in Essential C#.", - }, - { - title: "Table of Contents Filtering", - text: "The Table of Contents filter will let you narrow down the list of topics to help you quickly and easily find your destination.", - }, -]; - -const contentComingSoonList = [ - { - title: "Experimental attribute", - text: "New feature from C# 12.0.", - }, - { - title: "Source Generators", - text: "A newer .NET feature.", - }, - { - title: "C# 13.0 Features", - text: "Various new features coming in .C# 13.0", - }, -]; - -const completedFeaturesList = [ - { - title: "Copying Header Hyperlinks", - text: "Easily copy a header URL to link to a book section.", - }, - { - title: "Home Page", - text: "Add a home page that features a short description of the book and a high level mindmap.", - }, - { - title: "Keyboard Shortcuts", - text: "Quickly navigate through the book via keyboard shortcuts (right/left arrows, 'n', 'p').", - }, -]; - -/** - * Find the path of TOC entries that lead to the current page. - * @param {TocItem[]} path - * @param {TocItem[]} items - * @returns {TocItem[] | undefined} path of items to the current page - * */ -function findCurrentPage(path, items) { - for (const item of items) { - const itemPath = [item, ...path]; - if ( - window.location.href.endsWith("/" + item.href) || - window.location.href.endsWith("/" + item.key) - ) { - return itemPath; - } - - const recursivePath = findCurrentPage(itemPath, item.items); - if (recursivePath) return recursivePath; - } -} - -function openSearch() { - const el = document - .getElementById("docsearch") - .querySelector(".DocSearch-Button"); - el.click(); -} - -const smallScreenSize = 768; - -const removeHashPath = (path) => { - if (!path) { - return null; - } - let index = path.indexOf("#"); - index = index > 0 ? index : path.length; - return path.substring(0, index); -}; -// v-bind dont like the # in the url -const nextPagePath = removeHashPath(NEXT_PAGE); -const previousPagePath = removeHashPath(PREVIOUS_PAGE); - +import { + createApp, + ref, + reactive, + onMounted, + markRaw, + watch, + computed, +} from "vue"; +import { useWindowSize } from "vue-window-size"; + +/** + * @typedef {Object} TocItem + * @prop {number} [level] + * @prop {string} [key] + * @prop {string} [href] + * @prop {string} [title] + * @prop {TocItem[]} [items] + */ +/** @type {TocItem} */ +const tocData = markRaw(TOC_DATA); + +//Add new content or features here: + +const featuresComingSoonList = [ + { + title: "Client-side Compiler", + text: "Write, compile, and run code snippets right from your browser. Enjoy hands-on experience with the code as you go through the site.", + }, + { + title: "Interactive Code Listings", + text: "Edit, compile, and run the code listings found throughout Essential C#.", + }, + { + title: "Hyperlinking", + text: "Easily navigate to interesting and relevant sites as well as related sections in Essential C#.", + }, + { + title: "Table of Contents Filtering", + text: "The Table of Contents filter will let you narrow down the list of topics to help you quickly and easily find your destination.", + }, +]; + +const contentComingSoonList = [ + { + title: "Experimental attribute", + text: "New feature from C# 12.0.", + }, + { + title: "Source Generators", + text: "A newer .NET feature.", + }, + { + title: "C# 13.0 Features", + text: "Various new features coming in .C# 13.0", + }, +]; + +const completedFeaturesList = [ + { + title: "Copying Header Hyperlinks", + text: "Easily copy a header URL to link to a book section.", + }, + { + title: "Home Page", + text: "Add a home page that features a short description of the book and a high level mindmap.", + }, + { + title: "Keyboard Shortcuts", + text: "Quickly navigate through the book via keyboard shortcuts (right/left arrows, 'n', 'p').", + }, +]; + +/** + * Find the path of TOC entries that lead to the current page. + * @param {TocItem[]} path + * @param {TocItem[]} items + * @returns {TocItem[] | undefined} path of items to the current page + * */ +function findCurrentPage(path, items) { + for (const item of items) { + const itemPath = [item, ...path]; + if ( + window.location.href.endsWith("/" + item.href) || + window.location.href.endsWith("/" + item.key) + ) { + return itemPath; + } + + const recursivePath = findCurrentPage(itemPath, item.items); + if (recursivePath) return recursivePath; + } +} + +function openSearch() { + const el = document + .getElementById("docsearch") + .querySelector(".DocSearch-Button"); + el.click(); +} + +const smallScreenSize = 768; + +const removeHashPath = (path) => { + if (!path) { + return null; + } + let index = path.indexOf("#"); + index = index > 0 ? index : path.length; + return path.substring(0, index); +}; +// v-bind dont like the # in the url +const nextPagePath = removeHashPath(NEXT_PAGE); +const previousPagePath = removeHashPath(PREVIOUS_PAGE); + const app = createApp({ setup() { const { width: windowWidth } = useWindowSize(); @@ -125,10 +125,22 @@ const app = createApp({ const snackbarColor = ref(); function copyToClipboard(copyText) { - let url = window.location.origin + "/" + copyText; + let url; + + // If copyText contains a #, it's a full path with anchor (e.g., 'page#anchor') + // If copyText doesn't contain a #, it's just an anchor for the current page + if (copyText.includes('#')) { + // Full path case: construct URL with origin + path + url = window.location.origin + "/" + copyText; + } else { + // Anchor only case: use current page URL + anchor + const currentUrl = window.location.href.split('#')[0]; // Remove any existing anchor + url = currentUrl + "#" + copyText; + } + let referralId = REFERRAL_ID; - if (referralId && referralId.trim()) { - url = addQueryParam(url, 'rid', referralId); + if (referralId && referralId.trim()) { + url = addQueryParam(url, 'rid', referralId); } navigator.clipboard .writeText(url) @@ -156,10 +168,10 @@ const app = createApp({ ); } - function addQueryParam(url, key, value) { - let urlObj = new URL(url, window.location.origin); - urlObj.searchParams.set(key, value); - return urlObj.toString(); + function addQueryParam(url, key, value) { + let urlObj = new URL(url, window.location.origin); + urlObj.searchParams.set(key, value); + return urlObj.toString(); } function goToPrevious() { @@ -252,9 +264,9 @@ const app = createApp({ onMounted(() => { // If a setting is set in storage already, follow that - if (sidebarShown.value === null) { - if (windowWidth.value > smallScreenSize) { - sidebarShown.value = true; + if (sidebarShown.value === null) { + if (windowWidth.value > smallScreenSize) { + sidebarShown.value = true; } } @@ -266,8 +278,8 @@ const app = createApp({ block: "center", inline: "center", }); - }); - + }); + const enableTocFilter = ref('none'); const searchQuery = ref(''); @@ -291,8 +303,8 @@ const app = createApp({ matches = matches || childMatches; } return matches; - } - + } + watch(searchQuery, (newQuery) => { if (!newQuery) { expandedTocs.clear(); @@ -310,51 +322,51 @@ const app = createApp({ } }); } - }); - + }); + function normalizeString(str) { return str.replace(/[^\w\s]|_/g, "").replace(/\s+/g, " ").toLowerCase(); - } - - return { - previousPageUrl, - nextPageUrl, - goToPrevious, - goToNext, - openSearch, - - snackbarMessage, - snackbarColor, - copyToClipboard, - - contentComingSoonList, - featuresComingSoonList, - completedFeaturesList, - - sidebarShown, - sidebarTab, - toggleSidebar, - - smallScreen, - - sectionTitle, - tocData, - expandedTocs, - currentPage, - percentComplete, - chapterParentPage, + } + + return { + previousPageUrl, + nextPageUrl, + goToPrevious, + goToNext, + openSearch, + + snackbarMessage, + snackbarColor, + copyToClipboard, + + contentComingSoonList, + featuresComingSoonList, + completedFeaturesList, + + sidebarShown, + sidebarTab, + toggleSidebar, + + smallScreen, + + sectionTitle, + tocData, + expandedTocs, + currentPage, + percentComplete, + chapterParentPage, searchQuery, - filteredTocData, - enableTocFilter, - isContentPage - }; - }, -}); - -app.component("toc-tree", { - props: ["item", "expandedTocs", "currentPage"], - template: "#toc-tree", -}); - -app.mount("#app"); + filteredTocData, + enableTocFilter, + isContentPage + }; + }, +}); + +app.component("toc-tree", { + props: ["item", "expandedTocs", "currentPage"], + template: "#toc-tree", +}); + +app.mount("#app"); From d9dc06ed26a707a17e1c8362ed957ab88b58e13c Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Mon, 14 Jul 2025 20:03:15 -0700 Subject: [PATCH 02/22] PR Feedback --- .../Views/Home/Guidelines.cshtml | 8 +- EssentialCSharp.Web/wwwroot/js/site.js | 344 +++++++++--------- 2 files changed, 176 insertions(+), 176 deletions(-) diff --git a/EssentialCSharp.Web/Views/Home/Guidelines.cshtml b/EssentialCSharp.Web/Views/Home/Guidelines.cshtml index a9de4307..cb2a4842 100644 --- a/EssentialCSharp.Web/Views/Home/Guidelines.cshtml +++ b/EssentialCSharp.Web/Views/Home/Guidelines.cshtml @@ -11,9 +11,9 @@

C# Naming Conventions - Quick Reference Table - +

@@ -46,9 +46,9 @@ {

@group.Key - +

foreach (var guideline in group) { diff --git a/EssentialCSharp.Web/wwwroot/js/site.js b/EssentialCSharp.Web/wwwroot/js/site.js index ee6313c4..ca70cbfb 100644 --- a/EssentialCSharp.Web/wwwroot/js/site.js +++ b/EssentialCSharp.Web/wwwroot/js/site.js @@ -1,118 +1,118 @@ -import { - createApp, - ref, - reactive, - onMounted, - markRaw, - watch, - computed, -} from "vue"; -import { useWindowSize } from "vue-window-size"; - -/** - * @typedef {Object} TocItem - * @prop {number} [level] - * @prop {string} [key] - * @prop {string} [href] - * @prop {string} [title] - * @prop {TocItem[]} [items] - */ -/** @type {TocItem} */ -const tocData = markRaw(TOC_DATA); - -//Add new content or features here: - -const featuresComingSoonList = [ - { - title: "Client-side Compiler", - text: "Write, compile, and run code snippets right from your browser. Enjoy hands-on experience with the code as you go through the site.", - }, - { - title: "Interactive Code Listings", - text: "Edit, compile, and run the code listings found throughout Essential C#.", - }, - { - title: "Hyperlinking", - text: "Easily navigate to interesting and relevant sites as well as related sections in Essential C#.", - }, - { - title: "Table of Contents Filtering", - text: "The Table of Contents filter will let you narrow down the list of topics to help you quickly and easily find your destination.", - }, -]; - -const contentComingSoonList = [ - { - title: "Experimental attribute", - text: "New feature from C# 12.0.", - }, - { - title: "Source Generators", - text: "A newer .NET feature.", - }, - { - title: "C# 13.0 Features", - text: "Various new features coming in .C# 13.0", - }, -]; - -const completedFeaturesList = [ - { - title: "Copying Header Hyperlinks", - text: "Easily copy a header URL to link to a book section.", - }, - { - title: "Home Page", - text: "Add a home page that features a short description of the book and a high level mindmap.", - }, - { - title: "Keyboard Shortcuts", - text: "Quickly navigate through the book via keyboard shortcuts (right/left arrows, 'n', 'p').", - }, -]; - -/** - * Find the path of TOC entries that lead to the current page. - * @param {TocItem[]} path - * @param {TocItem[]} items - * @returns {TocItem[] | undefined} path of items to the current page - * */ -function findCurrentPage(path, items) { - for (const item of items) { - const itemPath = [item, ...path]; - if ( - window.location.href.endsWith("/" + item.href) || - window.location.href.endsWith("/" + item.key) - ) { - return itemPath; - } - - const recursivePath = findCurrentPage(itemPath, item.items); - if (recursivePath) return recursivePath; - } -} - -function openSearch() { - const el = document - .getElementById("docsearch") - .querySelector(".DocSearch-Button"); - el.click(); -} - -const smallScreenSize = 768; - -const removeHashPath = (path) => { - if (!path) { - return null; - } - let index = path.indexOf("#"); - index = index > 0 ? index : path.length; - return path.substring(0, index); -}; -// v-bind dont like the # in the url -const nextPagePath = removeHashPath(NEXT_PAGE); -const previousPagePath = removeHashPath(PREVIOUS_PAGE); - +import { + createApp, + ref, + reactive, + onMounted, + markRaw, + watch, + computed, +} from "vue"; +import { useWindowSize } from "vue-window-size"; + +/** + * @typedef {Object} TocItem + * @prop {number} [level] + * @prop {string} [key] + * @prop {string} [href] + * @prop {string} [title] + * @prop {TocItem[]} [items] + */ +/** @type {TocItem} */ +const tocData = markRaw(TOC_DATA); + +//Add new content or features here: + +const featuresComingSoonList = [ + { + title: "Client-side Compiler", + text: "Write, compile, and run code snippets right from your browser. Enjoy hands-on experience with the code as you go through the site.", + }, + { + title: "Interactive Code Listings", + text: "Edit, compile, and run the code listings found throughout Essential C#.", + }, + { + title: "Hyperlinking", + text: "Easily navigate to interesting and relevant sites as well as related sections in Essential C#.", + }, + { + title: "Table of Contents Filtering", + text: "The Table of Contents filter will let you narrow down the list of topics to help you quickly and easily find your destination.", + }, +]; + +const contentComingSoonList = [ + { + title: "Experimental attribute", + text: "New feature from C# 12.0.", + }, + { + title: "Source Generators", + text: "A newer .NET feature.", + }, + { + title: "C# 13.0 Features", + text: "Various new features coming in .C# 13.0", + }, +]; + +const completedFeaturesList = [ + { + title: "Copying Header Hyperlinks", + text: "Easily copy a header URL to link to a book section.", + }, + { + title: "Home Page", + text: "Add a home page that features a short description of the book and a high level mindmap.", + }, + { + title: "Keyboard Shortcuts", + text: "Quickly navigate through the book via keyboard shortcuts (right/left arrows, 'n', 'p').", + }, +]; + +/** + * Find the path of TOC entries that lead to the current page. + * @param {TocItem[]} path + * @param {TocItem[]} items + * @returns {TocItem[] | undefined} path of items to the current page + * */ +function findCurrentPage(path, items) { + for (const item of items) { + const itemPath = [item, ...path]; + if ( + window.location.href.endsWith("/" + item.href) || + window.location.href.endsWith("/" + item.key) + ) { + return itemPath; + } + + const recursivePath = findCurrentPage(itemPath, item.items); + if (recursivePath) return recursivePath; + } +} + +function openSearch() { + const el = document + .getElementById("docsearch") + .querySelector(".DocSearch-Button"); + el.click(); +} + +const smallScreenSize = 768; + +const removeHashPath = (path) => { + if (!path) { + return null; + } + let index = path.indexOf("#"); + index = index > 0 ? index : path.length; + return path.substring(0, index); +}; +// v-bind dont like the # in the url +const nextPagePath = removeHashPath(NEXT_PAGE); +const previousPagePath = removeHashPath(PREVIOUS_PAGE); + const app = createApp({ setup() { const { width: windowWidth } = useWindowSize(); @@ -139,8 +139,8 @@ const app = createApp({ } let referralId = REFERRAL_ID; - if (referralId && referralId.trim()) { - url = addQueryParam(url, 'rid', referralId); + if (referralId && referralId.trim()) { + url = addQueryParam(url, 'rid', referralId); } navigator.clipboard .writeText(url) @@ -168,10 +168,10 @@ const app = createApp({ ); } - function addQueryParam(url, key, value) { - let urlObj = new URL(url, window.location.origin); - urlObj.searchParams.set(key, value); - return urlObj.toString(); + function addQueryParam(url, key, value) { + let urlObj = new URL(url, window.location.origin); + urlObj.searchParams.set(key, value); + return urlObj.toString(); } function goToPrevious() { @@ -264,9 +264,9 @@ const app = createApp({ onMounted(() => { // If a setting is set in storage already, follow that - if (sidebarShown.value === null) { - if (windowWidth.value > smallScreenSize) { - sidebarShown.value = true; + if (sidebarShown.value === null) { + if (windowWidth.value > smallScreenSize) { + sidebarShown.value = true; } } @@ -278,8 +278,8 @@ const app = createApp({ block: "center", inline: "center", }); - }); - + }); + const enableTocFilter = ref('none'); const searchQuery = ref(''); @@ -303,8 +303,8 @@ const app = createApp({ matches = matches || childMatches; } return matches; - } - + } + watch(searchQuery, (newQuery) => { if (!newQuery) { expandedTocs.clear(); @@ -322,51 +322,51 @@ const app = createApp({ } }); } - }); - + }); + function normalizeString(str) { return str.replace(/[^\w\s]|_/g, "").replace(/\s+/g, " ").toLowerCase(); - } - - return { - previousPageUrl, - nextPageUrl, - goToPrevious, - goToNext, - openSearch, - - snackbarMessage, - snackbarColor, - copyToClipboard, - - contentComingSoonList, - featuresComingSoonList, - completedFeaturesList, - - sidebarShown, - sidebarTab, - toggleSidebar, - - smallScreen, - - sectionTitle, - tocData, - expandedTocs, - currentPage, - percentComplete, - chapterParentPage, + } + + return { + previousPageUrl, + nextPageUrl, + goToPrevious, + goToNext, + openSearch, + + snackbarMessage, + snackbarColor, + copyToClipboard, + + contentComingSoonList, + featuresComingSoonList, + completedFeaturesList, + + sidebarShown, + sidebarTab, + toggleSidebar, + + smallScreen, + + sectionTitle, + tocData, + expandedTocs, + currentPage, + percentComplete, + chapterParentPage, searchQuery, - filteredTocData, - enableTocFilter, - isContentPage - }; - }, -}); - -app.component("toc-tree", { - props: ["item", "expandedTocs", "currentPage"], - template: "#toc-tree", -}); - -app.mount("#app"); + filteredTocData, + enableTocFilter, + isContentPage + }; + }, +}); + +app.component("toc-tree", { + props: ["item", "expandedTocs", "currentPage"], + template: "#toc-tree", +}); + +app.mount("#app"); From 9a1adf0419d004082ab2dbf6f7843e29fa4ef810 Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Tue, 15 Jul 2025 21:32:55 -0700 Subject: [PATCH 03/22] WIP --- Directory.Packages.props | 21 ++- .../EssentialCSharp.Chat.Common.csproj | 20 +++ .../EssentialCSharp.Chat.csproj | 36 +++++ EssentialCSharp.Chat/Program.cs | 148 ++++++++++++++++++ .../EssentialCSharp.VectorDbBuilder.csproj | 33 ++++ EssentialCSharp.VectorDbBuilder/Program.cs | 148 ++++++++++++++++++ EssentialCSharp.Web.sln | 21 ++- 7 files changed, 423 insertions(+), 4 deletions(-) create mode 100644 EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj create mode 100644 EssentialCSharp.Chat/EssentialCSharp.Chat.csproj create mode 100644 EssentialCSharp.Chat/Program.cs create mode 100644 EssentialCSharp.VectorDbBuilder/EssentialCSharp.VectorDbBuilder.csproj create mode 100644 EssentialCSharp.VectorDbBuilder/Program.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 182f7e19..06bd4a94 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,7 +3,7 @@ true false 1.1.1.5540 - false + true https://api.nuget.org/v3/index.json; @@ -14,6 +14,7 @@ + @@ -36,11 +37,27 @@ + + + + + + + + + + + + + + + + - + \ No newline at end of file diff --git a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj new file mode 100644 index 00000000..08ffef79 --- /dev/null +++ b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj @@ -0,0 +1,20 @@ + + + + net9.0 + enable + enable + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + diff --git a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj new file mode 100644 index 00000000..eebf48b4 --- /dev/null +++ b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj @@ -0,0 +1,36 @@ + + + + Exe + net9.0 + 0.0.1 + + + + + + EssentialCSharp.Chat + true + essentialcsharpchat + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + diff --git a/EssentialCSharp.Chat/Program.cs b/EssentialCSharp.Chat/Program.cs new file mode 100644 index 00000000..a9d796b2 --- /dev/null +++ b/EssentialCSharp.Chat/Program.cs @@ -0,0 +1,148 @@ +using System.CommandLine; + +namespace EssentialCSharp.Chat; + +public class Program +{ + static int Main(string[] args) + { + Option fileOption = new("--file") + { + Description = "An option whose argument is parsed as a FileInfo", + Required = true, + DefaultValueFactory = result => + { + if (result.Tokens.Count == 0) + { + return new FileInfo("sampleQuotes.txt"); + + } + string filePath = result.Tokens.Single().Value; + if (!File.Exists(filePath)) + { + result.AddError("File does not exist"); + return new FileInfo("sampleQuotes.txt"); // Return a default FileInfo instead of null + } + else + { + return new FileInfo(filePath); + } + } + }; + + Option delayOption = new("--delay") + { + Description = "Delay between lines, specified as milliseconds per character in a line.", + DefaultValueFactory = parseResult => 42 + }; + Option fgcolorOption = new("--fgcolor") + { + Description = "Foreground color of text displayed on the console.", + DefaultValueFactory = parseResult => ConsoleColor.White + }; + Option lightModeOption = new("--light-mode") + { + Description = "Background color of text displayed on the console: default is black, light mode is white." + }; + + Option searchTermsOption = new("--search-terms") + { + Description = "Strings to search for when deleting entries.", + Required = true, + AllowMultipleArgumentsPerToken = true + }; + Argument quoteArgument = new("quote") + { + Description = "Text of quote." + }; + Argument bylineArgument = new("byline") + { + Description = "Byline of quote." + }; + + RootCommand rootCommand = new("Sample app for System.CommandLine"); + fileOption.Recursive = true; + rootCommand.Options.Add(fileOption); + + Command quotesCommand = new("quotes", "Work with a file that contains quotes."); + rootCommand.Subcommands.Add(quotesCommand); + + Command readCommand = new("read", "Read and display the file.") +{ + delayOption, + fgcolorOption, + lightModeOption +}; + quotesCommand.Subcommands.Add(readCommand); + + Command deleteCommand = new("delete", "Delete lines from the file."); + deleteCommand.Options.Add(searchTermsOption); + quotesCommand.Subcommands.Add(deleteCommand); + + Command addCommand = new("add", "Add an entry to the file."); + addCommand.Arguments.Add(quoteArgument); + addCommand.Arguments.Add(bylineArgument); + addCommand.Aliases.Add("insert"); + quotesCommand.Subcommands.Add(addCommand); + + readCommand.SetAction(parseResult => ReadFile( + parseResult.GetValue(fileOption), + parseResult.GetValue(delayOption), + parseResult.GetValue(fgcolorOption), + parseResult.GetValue(lightModeOption))); + + deleteCommand.SetAction(parseResult => DeleteFromFile( + parseResult.GetValue(fileOption), + parseResult.GetValue(searchTermsOption))); + + addCommand.SetAction(parseResult => AddToFile( + parseResult.GetValue(fileOption), + parseResult.GetValue(quoteArgument), + parseResult.GetValue(bylineArgument)) + ); + + return rootCommand.Parse(args).Invoke(); + } + + internal static void ReadFile(FileInfo? file, int? delay, ConsoleColor? fgColor, bool lightMode) + { + if (file == null || delay == null || fgColor == null) + { + Console.WriteLine("Invalid parameters."); + return; + } + + Console.BackgroundColor = lightMode ? ConsoleColor.White : ConsoleColor.Black; + Console.ForegroundColor = fgColor.Value; + foreach (string line in File.ReadLines(file.FullName)) + { + Console.WriteLine(line); + Thread.Sleep(TimeSpan.FromMilliseconds(delay.Value * line.Length)); + } + } + internal static void DeleteFromFile(FileInfo? file, string[]? searchTerms) + { + if (file == null || searchTerms == null || searchTerms.Length == 0) + { + Console.WriteLine("Invalid parameters."); + return; + } + Console.WriteLine("Deleting from file"); + + var lines = File.ReadLines(file.FullName).Where(line => searchTerms.All(s => !line.Contains(s))); + File.WriteAllLines(file.FullName, lines); + } + internal static void AddToFile(FileInfo? file, string? quote, string? byline) + { + if (file == null || string.IsNullOrWhiteSpace(quote) || string.IsNullOrWhiteSpace(byline)) + { + Console.WriteLine("Invalid parameters."); + return; + } + Console.WriteLine("Adding to file"); + + using StreamWriter writer = file.AppendText(); + writer.WriteLine($"{Environment.NewLine}{Environment.NewLine}{quote}"); + writer.WriteLine($"{Environment.NewLine}-{byline}"); + } +} diff --git a/EssentialCSharp.VectorDbBuilder/EssentialCSharp.VectorDbBuilder.csproj b/EssentialCSharp.VectorDbBuilder/EssentialCSharp.VectorDbBuilder.csproj new file mode 100644 index 00000000..116905c1 --- /dev/null +++ b/EssentialCSharp.VectorDbBuilder/EssentialCSharp.VectorDbBuilder.csproj @@ -0,0 +1,33 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + + diff --git a/EssentialCSharp.VectorDbBuilder/Program.cs b/EssentialCSharp.VectorDbBuilder/Program.cs new file mode 100644 index 00000000..74ef36c2 --- /dev/null +++ b/EssentialCSharp.VectorDbBuilder/Program.cs @@ -0,0 +1,148 @@ +using System.CommandLine; + +namespace EssentialCSharp.VectorDbBuilder; + +public class Program +{ + static int Main(string[] args) + { + Option fileOption = new("--file") + { + Description = "An option whose argument is parsed as a FileInfo", + Required = true, + DefaultValueFactory = result => + { + if (result.Tokens.Count == 0) + { + return new FileInfo("sampleQuotes.txt"); + + } + string filePath = result.Tokens.Single().Value; + if (!File.Exists(filePath)) + { + result.AddError("File does not exist"); + return new FileInfo("sampleQuotes.txt"); // Return a default FileInfo instead of null + } + else + { + return new FileInfo(filePath); + } + } + }; + + Option delayOption = new("--delay") + { + Description = "Delay between lines, specified as milliseconds per character in a line.", + DefaultValueFactory = parseResult => 42 + }; + Option fgcolorOption = new("--fgcolor") + { + Description = "Foreground color of text displayed on the console.", + DefaultValueFactory = parseResult => ConsoleColor.White + }; + Option lightModeOption = new("--light-mode") + { + Description = "Background color of text displayed on the console: default is black, light mode is white." + }; + + Option searchTermsOption = new("--search-terms") + { + Description = "Strings to search for when deleting entries.", + Required = true, + AllowMultipleArgumentsPerToken = true + }; + Argument quoteArgument = new("quote") + { + Description = "Text of quote." + }; + Argument bylineArgument = new("byline") + { + Description = "Byline of quote." + }; + + RootCommand rootCommand = new("Sample app for System.CommandLine"); + fileOption.Recursive = true; + rootCommand.Options.Add(fileOption); + + Command quotesCommand = new("quotes", "Work with a file that contains quotes."); + rootCommand.Subcommands.Add(quotesCommand); + + Command readCommand = new("read", "Read and display the file.") + { + delayOption, + fgcolorOption, + lightModeOption + }; + quotesCommand.Subcommands.Add(readCommand); + + Command deleteCommand = new("delete", "Delete lines from the file."); + deleteCommand.Options.Add(searchTermsOption); + quotesCommand.Subcommands.Add(deleteCommand); + + Command addCommand = new("add", "Add an entry to the file."); + addCommand.Arguments.Add(quoteArgument); + addCommand.Arguments.Add(bylineArgument); + addCommand.Aliases.Add("insert"); + quotesCommand.Subcommands.Add(addCommand); + + readCommand.SetAction(parseResult => ReadFile( + parseResult.GetValue(fileOption), + parseResult.GetValue(delayOption), + parseResult.GetValue(fgcolorOption), + parseResult.GetValue(lightModeOption))); + + deleteCommand.SetAction(parseResult => DeleteFromFile( + parseResult.GetValue(fileOption), + parseResult.GetValue(searchTermsOption))); + + addCommand.SetAction(parseResult => AddToFile( + parseResult.GetValue(fileOption), + parseResult.GetValue(quoteArgument), + parseResult.GetValue(bylineArgument)) + ); + + return rootCommand.Parse(args).Invoke(); + } + + internal static void ReadFile(FileInfo? file, int? delay, ConsoleColor? fgColor, bool lightMode) + { + if (file == null || delay == null || fgColor == null) + { + Console.WriteLine("Invalid parameters."); + return; + } + + Console.BackgroundColor = lightMode ? ConsoleColor.White : ConsoleColor.Black; + Console.ForegroundColor = fgColor.Value; + foreach (string line in File.ReadLines(file.FullName)) + { + Console.WriteLine(line); + Thread.Sleep(TimeSpan.FromMilliseconds(delay.Value * line.Length)); + } + } + internal static void DeleteFromFile(FileInfo? file, string[]? searchTerms) + { + if (file == null || searchTerms == null || searchTerms.Length == 0) + { + Console.WriteLine("Invalid parameters."); + return; + } + Console.WriteLine("Deleting from file"); + + var lines = File.ReadLines(file.FullName).Where(line => searchTerms.All(s => !line.Contains(s))); + File.WriteAllLines(file.FullName, lines); + } + internal static void AddToFile(FileInfo? file, string? quote, string? byline) + { + if (file == null || string.IsNullOrWhiteSpace(quote) || string.IsNullOrWhiteSpace(byline)) + { + Console.WriteLine("Invalid parameters."); + return; + } + Console.WriteLine("Adding to file"); + + using StreamWriter writer = file.AppendText(); + writer.WriteLine($"{Environment.NewLine}{Environment.NewLine}{quote}"); + writer.WriteLine($"{Environment.NewLine}-{byline}"); + } +} diff --git a/EssentialCSharp.Web.sln b/EssentialCSharp.Web.sln index 37de4f0c..66ae444a 100644 --- a/EssentialCSharp.Web.sln +++ b/EssentialCSharp.Web.sln @@ -1,5 +1,4 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 +Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.32014.148 MinimumVisualStudioVersion = 10.0.40219.1 @@ -21,6 +20,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EssentialCSharp.Web", "Esse EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EssentialCSharp.Web.Tests", "EssentialCSharp.Web.Tests\EssentialCSharp.Web.Tests.csproj", "{5717B439-2CFF-4BC5-A1DC-48BBF0FBE50F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.Chat", "EssentialCSharp.Chat\EssentialCSharp.Chat.csproj", "{5D3487A4-F414-1A54-17CE-866AE6298BBD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.Chat.Common", "EssentialCSharp.Chat.Shared\EssentialCSharp.Chat.Common.csproj", "{1B9082D5-D325-42DB-9EC3-03A3953EA8EE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.VectorDbBuilder", "EssentialCSharp.VectorDbBuilder\EssentialCSharp.VectorDbBuilder.csproj", "{8F7A5E12-B234-4C8A-9D45-12F456789ABC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -35,6 +40,18 @@ Global {5717B439-2CFF-4BC5-A1DC-48BBF0FBE50F}.Debug|Any CPU.Build.0 = Debug|Any CPU {5717B439-2CFF-4BC5-A1DC-48BBF0FBE50F}.Release|Any CPU.ActiveCfg = Release|Any CPU {5717B439-2CFF-4BC5-A1DC-48BBF0FBE50F}.Release|Any CPU.Build.0 = Release|Any CPU + {5D3487A4-F414-1A54-17CE-866AE6298BBD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5D3487A4-F414-1A54-17CE-866AE6298BBD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5D3487A4-F414-1A54-17CE-866AE6298BBD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5D3487A4-F414-1A54-17CE-866AE6298BBD}.Release|Any CPU.Build.0 = Release|Any CPU + {1B9082D5-D325-42DB-9EC3-03A3953EA8EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1B9082D5-D325-42DB-9EC3-03A3953EA8EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1B9082D5-D325-42DB-9EC3-03A3953EA8EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1B9082D5-D325-42DB-9EC3-03A3953EA8EE}.Release|Any CPU.Build.0 = Release|Any CPU + {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 1294edc2912845cd8871da606381e9621a3a785c Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Tue, 15 Jul 2025 22:03:48 -0700 Subject: [PATCH 04/22] Basic POC of chunking --- EssentialCSharp.Chat/FileChunkingResult.cs | 14 + EssentialCSharp.Chat/Program.cs | 308 ++++++++++++++------- 2 files changed, 220 insertions(+), 102 deletions(-) create mode 100644 EssentialCSharp.Chat/FileChunkingResult.cs diff --git a/EssentialCSharp.Chat/FileChunkingResult.cs b/EssentialCSharp.Chat/FileChunkingResult.cs new file mode 100644 index 00000000..ce8f4b76 --- /dev/null +++ b/EssentialCSharp.Chat/FileChunkingResult.cs @@ -0,0 +1,14 @@ +namespace EssentialCSharp.Chat; + +/// +/// Data structure to hold chunking results for a single file +/// +public class FileChunkingResult +{ + public string FileName { get; set; } = string.Empty; + public string FilePath { get; set; } = string.Empty; + public int OriginalCharCount { get; set; } + public int ChunkCount { get; set; } + public List Chunks { get; set; } = new(); + public int TotalChunkCharacters { get; set; } +} diff --git a/EssentialCSharp.Chat/Program.cs b/EssentialCSharp.Chat/Program.cs index a9d796b2..12a6aa62 100644 --- a/EssentialCSharp.Chat/Program.cs +++ b/EssentialCSharp.Chat/Program.cs @@ -1,148 +1,252 @@ using System.CommandLine; +using Microsoft.SemanticKernel.Text; namespace EssentialCSharp.Chat; public class Program { + private static readonly char[] _LineSeparators = ['\r', '\n']; + static int Main(string[] args) { - Option fileOption = new("--file") + // Configure command-line options following System.CommandLine patterns + var directoryOption = new Option("--directory") { - Description = "An option whose argument is parsed as a FileInfo", - Required = true, - DefaultValueFactory = result => - { - if (result.Tokens.Count == 0) - { - return new FileInfo("sampleQuotes.txt"); - - } - string filePath = result.Tokens.Single().Value; - if (!File.Exists(filePath)) - { - result.AddError("File does not exist"); - return new FileInfo("sampleQuotes.txt"); // Return a default FileInfo instead of null - } - else - { - return new FileInfo(filePath); - } - } + Description = "Directory containing markdown files to chunk", + DefaultValueFactory = _ => new DirectoryInfo(@"D:\EssentialCSharp.Web\EssentialCSharp.Web\Markdown\") }; - Option delayOption = new("--delay") + var maxTokensOption = new Option("--max-tokens") { - Description = "Delay between lines, specified as milliseconds per character in a line.", - DefaultValueFactory = parseResult => 42 + Description = "Maximum tokens per chunk", + DefaultValueFactory = _ => 500 }; - Option fgcolorOption = new("--fgcolor") + + var overlapTokensOption = new Option("--overlap") { - Description = "Foreground color of text displayed on the console.", - DefaultValueFactory = parseResult => ConsoleColor.White + Description = "Number of tokens to overlap between chunks", + DefaultValueFactory = _ => 50 }; - Option lightModeOption = new("--light-mode") + + var chunkHeaderOption = new Option("--header") { - Description = "Background color of text displayed on the console: default is black, light mode is white." + Description = "Optional header to prepend to each chunk" }; - Option searchTermsOption = new("--search-terms") + var filePatternOption = new Option("--pattern") { - Description = "Strings to search for when deleting entries.", - Required = true, - AllowMultipleArgumentsPerToken = true + Description = "File pattern to match", + DefaultValueFactory = _ => "*.md" }; - Argument quoteArgument = new("quote") + + var outputFormatOption = new Option("--format") { - Description = "Text of quote." + Description = "Output format: summary, detailed, or json", + DefaultValueFactory = _ => "summary" }; - Argument bylineArgument = new("byline") + + // Create root command + var rootCommand = new RootCommand("Semantic Kernel TextChunker - Extract and Chunk Markdown Files") { - Description = "Byline of quote." + directoryOption, + maxTokensOption, + overlapTokensOption, + chunkHeaderOption, + filePatternOption, + outputFormatOption }; - RootCommand rootCommand = new("Sample app for System.CommandLine"); - fileOption.Recursive = true; - rootCommand.Options.Add(fileOption); - - Command quotesCommand = new("quotes", "Work with a file that contains quotes."); - rootCommand.Subcommands.Add(quotesCommand); + // Set the action for the root command + rootCommand.SetAction(parseResult => + { + var directory = parseResult.GetValue(directoryOption); + var maxTokens = parseResult.GetValue(maxTokensOption); + var overlapTokens = parseResult.GetValue(overlapTokensOption); + var chunkHeader = parseResult.GetValue(chunkHeaderOption); + var filePattern = parseResult.GetValue(filePatternOption); + var outputFormat = parseResult.GetValue(outputFormatOption); - Command readCommand = new("read", "Read and display the file.") -{ - delayOption, - fgcolorOption, - lightModeOption -}; - quotesCommand.Subcommands.Add(readCommand); - - Command deleteCommand = new("delete", "Delete lines from the file."); - deleteCommand.Options.Add(searchTermsOption); - quotesCommand.Subcommands.Add(deleteCommand); - - Command addCommand = new("add", "Add an entry to the file."); - addCommand.Arguments.Add(quoteArgument); - addCommand.Arguments.Add(bylineArgument); - addCommand.Aliases.Add("insert"); - quotesCommand.Subcommands.Add(addCommand); - - readCommand.SetAction(parseResult => ReadFile( - parseResult.GetValue(fileOption), - parseResult.GetValue(delayOption), - parseResult.GetValue(fgcolorOption), - parseResult.GetValue(lightModeOption))); - - deleteCommand.SetAction(parseResult => DeleteFromFile( - parseResult.GetValue(fileOption), - parseResult.GetValue(searchTermsOption))); - - addCommand.SetAction(parseResult => AddToFile( - parseResult.GetValue(fileOption), - parseResult.GetValue(quoteArgument), - parseResult.GetValue(bylineArgument)) - ); + return ProcessMarkdownFiles(directory!, maxTokens, overlapTokens, chunkHeader, filePattern!, outputFormat!); + }); return rootCommand.Parse(args).Invoke(); } - internal static void ReadFile(FileInfo? file, int? delay, ConsoleColor? fgColor, bool lightMode) + /// + /// Process markdown files in the specified directory using Semantic Kernel's TextChunker + /// Following Microsoft Learn documentation for proper implementation + /// + internal static int ProcessMarkdownFiles( + DirectoryInfo directory, + int maxTokensPerParagraph, + int overlapTokens, + string? chunkHeader, + string filePattern, + string outputFormat) { - if (file == null || delay == null || fgColor == null) + try { - Console.WriteLine("Invalid parameters."); - return; - } + // Validate input parameters + if (!directory.Exists) + { + Console.Error.WriteLine($"Error: Directory '{directory.FullName}' does not exist."); + return 1; + } + + if (maxTokensPerParagraph <= 0) + { + Console.Error.WriteLine("Error: max-tokens must be a positive number."); + return 1; + } - Console.BackgroundColor = lightMode ? ConsoleColor.White : ConsoleColor.Black; - Console.ForegroundColor = fgColor.Value; - foreach (string line in File.ReadLines(file.FullName)) + if (overlapTokens < 0 || overlapTokens >= maxTokensPerParagraph) + { + Console.Error.WriteLine("Error: overlap-tokens must be between 0 and max-tokens."); + return 1; + } + + // Find markdown files + var markdownFiles = directory.GetFiles(filePattern, SearchOption.TopDirectoryOnly); + + if (markdownFiles.Length == 0) + { + Console.WriteLine($"No files matching pattern '{filePattern}' found in '{directory.FullName}'"); + return 0; + } + + Console.WriteLine($"Processing {markdownFiles.Length} markdown files..."); + Console.WriteLine($"Max tokens per chunk: {maxTokensPerParagraph}"); + Console.WriteLine($"Overlap tokens: {overlapTokens} ({(double)overlapTokens / maxTokensPerParagraph * 100:F1}%)"); + Console.WriteLine($"Chunk header: {(string.IsNullOrEmpty(chunkHeader) ? "None" : $"'{chunkHeader}'")}"); + Console.WriteLine(); + + int totalChunks = 0; + var results = new List(); + + foreach (var file in markdownFiles) + { + var result = ProcessSingleMarkdownFile(file, maxTokensPerParagraph, overlapTokens, chunkHeader); + results.Add(result); + totalChunks += result.ChunkCount; + + // Output per-file summary + Console.WriteLine($"File: {file.Name}"); + Console.WriteLine($" Original size: {result.OriginalCharCount:N0} characters"); + Console.WriteLine($" Chunks created: {result.ChunkCount}"); + Console.WriteLine($" Average chunk size: {(result.ChunkCount > 0 ? result.TotalChunkCharacters / result.ChunkCount : 0):N0} characters"); + Console.WriteLine(); + + // Output detailed chunks if requested + if (outputFormat.Equals("detailed", StringComparison.OrdinalIgnoreCase)) + { + OutputDetailedChunks(result); + } + } + + // Output summary + Console.WriteLine("=== SUMMARY ==="); + Console.WriteLine($"Total files processed: {markdownFiles.Length}"); + Console.WriteLine($"Total chunks created: {totalChunks}"); + Console.WriteLine($"Average chunks per file: {(markdownFiles.Length > 0 ? (double)totalChunks / markdownFiles.Length : 0):F1}"); + + // Output JSON if requested + if (outputFormat.Equals("json", StringComparison.OrdinalIgnoreCase)) + { + OutputJsonResults(results); + } + + return 0; + } + catch (Exception ex) { - Console.WriteLine(line); - Thread.Sleep(TimeSpan.FromMilliseconds(delay.Value * line.Length)); + Console.Error.WriteLine($"Error: {ex.Message}"); + return 1; } } - internal static void DeleteFromFile(FileInfo? file, string[]? searchTerms) + + /// + /// Process a single markdown file using Semantic Kernel's SplitMarkdownParagraphs method + /// Implementation follows Microsoft Learn documentation exactly + /// + internal static FileChunkingResult ProcessSingleMarkdownFile( + FileInfo file, + int maxTokensPerParagraph, + int overlapTokens, + string? chunkHeader) { - if (file == null || searchTerms == null || searchTerms.Length == 0) + // Read the markdown content + var content = File.ReadAllText(file.FullName); + + // Prepare lines for chunking - following Microsoft examples + var lines = content.Split(_LineSeparators, StringSplitOptions.RemoveEmptyEntries).ToList(); + + // Apply Semantic Kernel TextChunker.SplitMarkdownParagraphs + // Following the exact API signature from Microsoft Learn documentation + // Suppress the experimental warning as this is the intended usage per documentation +#pragma warning disable SKEXP0050 + var chunks = TextChunker.SplitMarkdownParagraphs( + lines, + maxTokensPerParagraph, + overlapTokens, + chunkHeader); +#pragma warning restore SKEXP0050 + + // Calculate statistics + var result = new FileChunkingResult { - Console.WriteLine("Invalid parameters."); - return; - } - Console.WriteLine("Deleting from file"); + FileName = file.Name, + FilePath = file.FullName, + OriginalCharCount = content.Length, + ChunkCount = chunks.Count, + Chunks = chunks, + TotalChunkCharacters = chunks.Sum(c => c.Length) + }; - var lines = File.ReadLines(file.FullName).Where(line => searchTerms.All(s => !line.Contains(s))); - File.WriteAllLines(file.FullName, lines); + return result; } - internal static void AddToFile(FileInfo? file, string? quote, string? byline) + + /// + /// Output detailed chunk information for inspection + /// + internal static void OutputDetailedChunks(FileChunkingResult result) { - if (file == null || string.IsNullOrWhiteSpace(quote) || string.IsNullOrWhiteSpace(byline)) + Console.WriteLine($"=== DETAILED CHUNKS for {result.FileName} ==="); + + for (int i = 0; i < result.Chunks.Count; i++) { - Console.WriteLine("Invalid parameters."); - return; + var chunk = result.Chunks[i]; + Console.WriteLine($"Chunk {i + 1}/{result.Chunks.Count}:"); + Console.WriteLine($" Length: {chunk.Length} characters"); + Console.WriteLine($" Preview: {chunk.Substring(0, Math.Min(100, chunk.Length)).Replace('\n', ' ').Replace('\r', ' ')}..."); + Console.WriteLine(" ---"); } - Console.WriteLine("Adding to file"); + Console.WriteLine(); + } - using StreamWriter writer = file.AppendText(); - writer.WriteLine($"{Environment.NewLine}{Environment.NewLine}{quote}"); - writer.WriteLine($"{Environment.NewLine}-{byline}"); + /// + /// Output results in JSON format for programmatic consumption + /// + internal static void OutputJsonResults(List results) + { + Console.WriteLine(); + Console.WriteLine("=== JSON OUTPUT ==="); + Console.WriteLine("{"); + Console.WriteLine(" \"results\": ["); + + for (int i = 0; i < results.Count; i++) + { + var result = results[i]; + Console.WriteLine(" {"); + Console.WriteLine($" \"fileName\": \"{result.FileName}\","); + Console.WriteLine($" \"filePath\": \"{result.FilePath.Replace("\\", "\\\\")}\","); + Console.WriteLine($" \"originalCharCount\": {result.OriginalCharCount},"); + Console.WriteLine($" \"chunkCount\": {result.ChunkCount},"); + Console.WriteLine($" \"totalChunkCharacters\": {result.TotalChunkCharacters},"); + Console.WriteLine($" \"averageChunkSize\": {(result.ChunkCount > 0 ? result.TotalChunkCharacters / result.ChunkCount : 0)}"); + Console.WriteLine(i < results.Count - 1 ? " }," : " }"); + } + + Console.WriteLine(" ]"); + Console.WriteLine("}"); } } From 04e3e3aaf102a782a3e19e9cbfce58a2576ac485 Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Wed, 16 Jul 2025 22:17:24 -0700 Subject: [PATCH 05/22] Initial Markdown Chunking --- Directory.Packages.props | 11 +- .../EssentialCSharp.Chat.Common.csproj | 4 + .../Models/BookContentChunk.cs | 53 +++ .../Services/EmbeddingService.cs | 130 ++++++++ .../Services}/FileChunkingResult.cs | 2 +- .../Services/MarkdownChunkingService.cs | 180 +++++++++++ .../Services/VectorStoreService.cs | 238 ++++++++++++++ .../EssentialCSharp.Chat.Tests.csproj | 26 ++ .../MarkdownChunkingServiceTests.cs | 192 +++++++++++ .../EssentialCSharp.Chat.csproj | 9 +- EssentialCSharp.Chat/Program.cs | 303 +++++------------- EssentialCSharp.Chat/packages.config | 5 - EssentialCSharp.Chat/requirements.txt | 2 - EssentialCSharp.Web.sln | 9 +- 14 files changed, 932 insertions(+), 232 deletions(-) create mode 100644 EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs create mode 100644 EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs rename {EssentialCSharp.Chat => EssentialCSharp.Chat.Shared/Services}/FileChunkingResult.cs (90%) create mode 100644 EssentialCSharp.Chat.Shared/Services/MarkdownChunkingService.cs create mode 100644 EssentialCSharp.Chat.Shared/Services/VectorStoreService.cs create mode 100644 EssentialCSharp.Chat.Tests/EssentialCSharp.Chat.Tests.csproj create mode 100644 EssentialCSharp.Chat.Tests/MarkdownChunkingServiceTests.cs delete mode 100644 EssentialCSharp.Chat/packages.config delete mode 100644 EssentialCSharp.Chat/requirements.txt diff --git a/Directory.Packages.props b/Directory.Packages.props index 06bd4a94..e94603b5 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -14,7 +14,9 @@ + + @@ -41,19 +43,20 @@ + - + + - - - + + diff --git a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj index 08ffef79..fb19d698 100644 --- a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj +++ b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj @@ -7,9 +7,13 @@ + + + + all diff --git a/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs b/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs new file mode 100644 index 00000000..959dc482 --- /dev/null +++ b/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs @@ -0,0 +1,53 @@ +using Microsoft.Extensions.VectorData; + +namespace EssentialCSharp.Chat.Common.Models; + +/// +/// Represents a chunk of markdown content for vector search +/// Following Semantic Kernel Vector Store best practices from Microsoft docs +/// +public sealed class BookContentChunk +{ + /// + /// Unique identifier for the chunk - serves as the vector store key + /// + [VectorStoreKey] + public string Id { get; set; } = string.Empty; + + /// + /// Original source file name + /// + [VectorStoreData] + public string FileName { get; set; } = string.Empty; + + /// + /// Heading or title of the markdown chunk + /// + [VectorStoreData] + public string Heading { get; set; } = string.Empty; + + /// + /// The actual markdown content text for this chunk + /// + [VectorStoreData] + public string ChunkText { get; set; } = string.Empty; + + /// + /// Chapter number extracted from filename (e.g., "Chapter01.md" -> 1) + /// + [VectorStoreData] + public int? ChapterNumber { get; set; } + + /// + /// SHA256 hash of the chunk content for change detection + /// + [VectorStoreData] + public string ContentHash { get; set; } = string.Empty; + + /// + /// Vector embedding for the chunk text - will be generated by embedding service + /// Using 1536 dimensions for Azure OpenAI text-embedding-ada-002 + /// + [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Hnsw)] + public ReadOnlyMemory? TextEmbedding { get; set; } +} diff --git a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs new file mode 100644 index 00000000..42079c26 --- /dev/null +++ b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs @@ -0,0 +1,130 @@ +namespace EssentialCSharp.Chat.Common.Services; + +///// +///// Service for generating embeddings for markdown chunks using Azure OpenAI +///// Following Microsoft best practices for Semantic Kernel embedding generation +///// +//public class EmbeddingService +//{ +// private readonly IEmbeddingGenerator> _EmbeddingGenerator; + +// public EmbeddingService(IEmbeddingGenerator> embeddingGenerator) +// { +// _EmbeddingGenerator = embeddingGenerator ?? throw new ArgumentNullException(nameof(embeddingGenerator)); +// } + +// /// +// /// Generate embeddings for a collection of markdown chunks +// /// +// /// The chunks to generate embeddings for +// /// Cancellation token +// /// The chunks with embeddings populated +// public async Task> GenerateEmbeddingsAsync( +// IList chunks, +// CancellationToken cancellationToken = default) +// { +// if (!chunks.Any()) +// return chunks; + +// Console.WriteLine($"Generating embeddings for {chunks.Count} chunks..."); + +// try +// { +// // Generate embeddings one by one to avoid rate limits +// for (int i = 0; i < chunks.Count; i++) +// { +// var chunk = chunks[i]; +// if (string.IsNullOrWhiteSpace(chunk.ChunkText)) +// continue; + +// var embedding = await _EmbeddingGenerator.GenerateEmbeddingAsync( +// chunk.ChunkText, +// options: null, +// cancellationToken); +// chunk.TextEmbedding = new ReadOnlyMemory(embedding.Vector.ToArray()); + +// // Show progress +// if ((i + 1) % 10 == 0) +// { +// Console.WriteLine($" Generated embeddings for {i + 1}/{chunks.Count} chunks"); +// } +// } + +// var successfulEmbeddings = chunks.Count(c => c.TextEmbedding.HasValue); +// Console.WriteLine($"✅ Successfully generated embeddings for {successfulEmbeddings}/{chunks.Count} chunks"); + +// if (successfulEmbeddings > 0) +// { +// Console.WriteLine($" Vector dimensions: {chunks.First(c => c.TextEmbedding.HasValue).TextEmbedding!.Value.Length}"); +// } +// } +// catch (Exception ex) +// { +// Console.WriteLine($"❌ Error generating embeddings: {ex.Message}"); +// throw; +// } + +// return chunks; +// } + +// /// +// /// Generate embedding for a single text +// /// +// /// The text to generate embedding for +// /// Cancellation token +// /// The generated embedding +// public async Task> GenerateEmbeddingAsync( +// string text, +// CancellationToken cancellationToken = default) +// { +// if (string.IsNullOrWhiteSpace(text)) +// throw new ArgumentException("Text cannot be null or empty", nameof(text)); + +// try +// { +// var embedding = await _EmbeddingGenerator.GenerateEmbeddingAsync( +// text, +// options: null, +// cancellationToken); +// return new ReadOnlyMemory(embedding.Vector.ToArray()); +// } +// catch (Exception ex) +// { +// Console.WriteLine($"❌ Error generating embedding for text: {ex.Message}"); +// throw; +// } +// } + +// /// +// /// Get embedding generation statistics +// /// +// /// The chunks that have embeddings +// /// Statistics about the embeddings +// public static EmbeddingStatistics GetEmbeddingStatistics(IList chunks) +// { +// var chunksWithEmbeddings = chunks.Where(c => c.TextEmbedding.HasValue).ToList(); + +// return new EmbeddingStatistics +// { +// TotalChunks = chunks.Count, +// ChunksWithEmbeddings = chunksWithEmbeddings.Count, +// EmbeddingDimensions = chunksWithEmbeddings.FirstOrDefault()?.TextEmbedding?.Length ?? 0, +// AverageTextLength = chunks.Any() ? (int)chunks.Average(c => c.ChunkText.Length) : 0, +// TotalTextCharacters = chunks.Sum(c => c.ChunkText.Length) +// }; +// } +//} + +///// +///// Statistics about embedding generation +///// +//public record EmbeddingStatistics +//{ +// public int TotalChunks { get; init; } +// public int ChunksWithEmbeddings { get; init; } +// public int EmbeddingDimensions { get; init; } +// public int AverageTextLength { get; init; } +// public int TotalTextCharacters { get; init; } + +// public double EmbeddingCoverage => TotalChunks > 0 ? (double)ChunksWithEmbeddings / TotalChunks * 100 : 0; +//} diff --git a/EssentialCSharp.Chat/FileChunkingResult.cs b/EssentialCSharp.Chat.Shared/Services/FileChunkingResult.cs similarity index 90% rename from EssentialCSharp.Chat/FileChunkingResult.cs rename to EssentialCSharp.Chat.Shared/Services/FileChunkingResult.cs index ce8f4b76..cc28c95c 100644 --- a/EssentialCSharp.Chat/FileChunkingResult.cs +++ b/EssentialCSharp.Chat.Shared/Services/FileChunkingResult.cs @@ -1,4 +1,4 @@ -namespace EssentialCSharp.Chat; +namespace EssentialCSharp.Chat.Common.Services; /// /// Data structure to hold chunking results for a single file diff --git a/EssentialCSharp.Chat.Shared/Services/MarkdownChunkingService.cs b/EssentialCSharp.Chat.Shared/Services/MarkdownChunkingService.cs new file mode 100644 index 00000000..ee66e7ef --- /dev/null +++ b/EssentialCSharp.Chat.Shared/Services/MarkdownChunkingService.cs @@ -0,0 +1,180 @@ +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.Text; + +namespace EssentialCSharp.Chat.Common.Services; + +/// +/// Markdown chunking service using Semantic Kernel's TextChunker +/// +public partial class MarkdownChunkingService( + ILogger logger, + int maxTokensPerChunk = 256, + int overlapTokens = 25) +{ + private static readonly string[] _NewLineSeparators = new[] { "\r\n", "\n", "\r" }; + private readonly int _MaxTokensPerChunk = maxTokensPerChunk; + private readonly int _OverlapTokens = overlapTokens; + + /// + /// Process markdown files in the specified directory using Semantic Kernel's TextChunker + /// + public async Task> ProcessMarkdownFilesAsync( + DirectoryInfo directory, + string filePattern) + { + // Validate input parameters + if (!directory.Exists) + { + logger.LogError("Error: Directory {DirectoryName} does not exist.", directory.FullName); + throw new InvalidOperationException($"Error: Directory '{directory.FullName}' does not exist."); + } + + // Find markdown files + var markdownFiles = directory.GetFiles(filePattern, SearchOption.TopDirectoryOnly); + + if (markdownFiles.Length == 0) + { + throw new InvalidOperationException($"No files matching pattern '{filePattern}' found in '{directory.FullName}'"); + } + + Console.WriteLine($"Processing {markdownFiles.Length} markdown files..."); + + int totalChunks = 0; + var results = new List(); + + foreach (var file in markdownFiles) + { + string[] fileContent = await File.ReadAllLinesAsync(file.FullName); + var result = ProcessSingleMarkdownFile(fileContent, file.Name, file.FullName); + results.Add(result); + totalChunks += result.ChunkCount; + } + Console.WriteLine($"Processed {markdownFiles.Length} markdown files with a total of {totalChunks} chunks."); + + return results; + } + + /// + /// Process a single markdown file using Semantic Kernel's SplitMarkdownParagraphs method + /// + public FileChunkingResult ProcessSingleMarkdownFile( + string[] fileContent, string fileName, string filePath) + { + // Remove all multiple empty lines so there is no more than one empty line between paragraphs + string[] lines = [.. fileContent + .Select(line => line.Trim()) + .Where(line => !string.IsNullOrWhiteSpace(line))]; + + string content = string.Join(Environment.NewLine, lines); + + var sections = MarkdownContentToHeadersAndSection(content); + var allChunks = new List(); + int totalChunkCharacters = 0; + int chunkCount = 0; + + foreach (var section in sections) + { +#pragma warning disable SKEXP0050 + var chunks = TextChunker.SplitMarkdownParagraphs( + lines: section.Content, + maxTokensPerParagraph: _MaxTokensPerChunk, + overlapTokens: _OverlapTokens, + chunkHeader: section.Header + " - " + ); +#pragma warning restore SKEXP0050 + allChunks.AddRange(chunks); + chunkCount += chunks.Count; + totalChunkCharacters += chunks.Sum(c => c.Length); + } + + return new FileChunkingResult + { + FileName = fileName, + FilePath = filePath, + OriginalCharCount = content.Length, + ChunkCount = chunkCount, + Chunks = allChunks, + TotalChunkCharacters = totalChunkCharacters + }; + } + + /// + /// Convert markdown content into a list of headers and their associated content sections. + /// + /// + /// + public static List<(string Header, List Content)> MarkdownContentToHeadersAndSection(string content) + { + var lines = content.Split(_NewLineSeparators, StringSplitOptions.None); + var sections = new List<(string Header, List Content)>(); + var headerRegex = HeadingRegex(); + var listingPattern = ListingRegex(); + var headerStack = new List<(int Level, string Text)>(); + int i = 0; + while (i < lines.Length) + { + // Find next header + while (i < lines.Length && !headerRegex.IsMatch(lines[i])) + i++; + if (i >= lines.Length) break; + + var match = headerRegex.Match(lines[i]); + int level = match.Groups[1].Value.Length; + string headerText = match.Groups[2].Value.Trim(); + bool isListing = headerText.StartsWith("Listing", StringComparison.OrdinalIgnoreCase) && listingPattern.IsMatch(headerText); + + // If this is a listing header, append its content to the previous section + if (isListing && sections.Count > 0) + { + i++; // skip the listing header + var listingContent = new List(); + while (i < lines.Length && !headerRegex.IsMatch(lines[i])) + { + if (!string.IsNullOrWhiteSpace(lines[i])) + listingContent.Add(lines[i]); + i++; + } + // Append to previous section's content + var prev = sections[^1]; + prev.Content.AddRange(listingContent); + sections[^1] = prev; + continue; + } + + // Update header stack for non-listing headers + if (headerStack.Count == 0 || level > headerStack.Last().Level) + { + headerStack.Add((level, headerText)); + } + else + { + while (headerStack.Count > 0 && headerStack.Last().Level >= level) + headerStack.RemoveAt(headerStack.Count - 1); + headerStack.Add((level, headerText)); + } + i++; + + // Collect content until next header + var contentLines = new List(); + while (i < lines.Length && !headerRegex.IsMatch(lines[i])) + { + if (!string.IsNullOrWhiteSpace(lines[i])) + contentLines.Add(lines[i]); + i++; + } + + // Compose full header context + var fullHeader = string.Join(": ", headerStack.Select(h => h.Text)); + if (contentLines.Count > 0) + sections.Add((fullHeader, contentLines)); + } + return sections; + } + + [GeneratedRegex(@"^Listing \d+\.\d+(:.*)?$")] + private static partial Regex ListingRegex(); + + [GeneratedRegex(@"^(#{1,6}) +(.+)$")] + private static partial Regex HeadingRegex(); +} diff --git a/EssentialCSharp.Chat.Shared/Services/VectorStoreService.cs b/EssentialCSharp.Chat.Shared/Services/VectorStoreService.cs new file mode 100644 index 00000000..e21e6da1 --- /dev/null +++ b/EssentialCSharp.Chat.Shared/Services/VectorStoreService.cs @@ -0,0 +1,238 @@ +namespace EssentialCSharp.Chat.Common.Services; + +///// +///// Service for storing and retrieving markdown chunks in PostgreSQL with pgvector +///// Following Microsoft best practices for Semantic Kernel Vector Store +///// +//public class VectorStoreService +//{ +// private readonly PostgresVectorStore _vectorStore; +// private readonly string _collectionName; + +// public VectorStoreService(string connectionString, string collectionName = "markdown_chunks") +// { +// if (string.IsNullOrWhiteSpace(connectionString)) +// throw new ArgumentException("Connection string cannot be null or empty", nameof(connectionString)); + +// _collectionName = collectionName; + +// // Create PostgreSQL vector store using the connection string +// _vectorStore = new PostgresVectorStore(connectionString); +// } + +// public VectorStoreService(PostgresVectorStore vectorStore, string collectionName = "markdown_chunks") +// { +// _vectorStore = vectorStore ?? throw new ArgumentNullException(nameof(vectorStore)); +// _collectionName = collectionName; +// } + +// /// +// /// Initialize the vector store collection +// /// +// /// Cancellation token +// public async Task InitializeAsync(CancellationToken cancellationToken = default) +// { +// Console.WriteLine($"Initializing vector store collection '{_collectionName}'..."); + +// try +// { +// var collection = GetCollection(); + +// // Create the collection if it doesn't exist +// await collection.CreateCollectionIfNotExistsAsync(cancellationToken); + +// Console.WriteLine($"✅ Vector store collection '{_collectionName}' is ready"); +// } +// catch (Exception ex) +// { +// Console.WriteLine($"❌ Error initializing vector store: {ex.Message}"); +// throw; +// } +// } + +// /// +// /// Store markdown chunks in the vector store +// /// +// /// The chunks to store +// /// Cancellation token +// public async Task StoreChunksAsync(IList chunks, CancellationToken cancellationToken = default) +// { +// if (!chunks.Any()) +// { +// Console.WriteLine("No chunks to store"); +// return; +// } + +// Console.WriteLine($"Storing {chunks.Count} chunks in vector store..."); + +// try +// { +// var collection = GetCollection(); + +// // Store chunks in batches to avoid overwhelming the database +// const int batchSize = 100; +// int stored = 0; + +// for (int i = 0; i < chunks.Count; i += batchSize) +// { +// var batch = chunks.Skip(i).Take(batchSize).ToList(); + +// // Upsert chunks (insert or update if exists) +// await collection.UpsertBatchAsync(batch, cancellationToken); + +// stored += batch.Count; +// Console.WriteLine($" Stored {stored}/{chunks.Count} chunks"); +// } + +// Console.WriteLine($"✅ Successfully stored {chunks.Count} chunks in vector store"); +// } +// catch (Exception ex) +// { +// Console.WriteLine($"❌ Error storing chunks: {ex.Message}"); +// throw; +// } +// } + +// /// +// /// Search for similar chunks using vector similarity +// /// +// /// The query embedding to search with +// /// Maximum number of results to return +// /// Minimum relevance score (0.0 to 1.0) +// /// Cancellation token +// /// Similar chunks ordered by relevance +// public async Task>> SearchSimilarChunksAsync( +// ReadOnlyMemory queryEmbedding, +// int limit = 10, +// double minRelevanceScore = 0.0, +// CancellationToken cancellationToken = default) +// { +// Console.WriteLine($"Searching for similar chunks (limit: {limit}, min score: {minRelevanceScore:F2})..."); + +// try +// { +// var collection = GetCollection(); + +// // Perform vector search +// var searchResults = await collection.VectorizedSearchAsync( +// queryEmbedding, +// new() +// { +// Top = limit, +// Filter = null // Could add filters for chapter, file, etc. +// }, +// cancellationToken); + +// var results = await searchResults +// .Where(r => r.Score >= minRelevanceScore) +// .ToListAsync(cancellationToken); + +// Console.WriteLine($"✅ Found {results.Count} similar chunks"); + +// return results; +// } +// catch (Exception ex) +// { +// Console.WriteLine($"❌ Error searching chunks: {ex.Message}"); +// throw; +// } +// } + +// /// +// /// Get chunks by file name +// /// +// /// The file name to search for +// /// Cancellation token +// /// Chunks from the specified file +// public async Task> GetChunksByFileAsync( +// string fileName, +// CancellationToken cancellationToken = default) +// { +// try +// { +// var collection = GetCollection(); + +// // Use VectorSearchOptions.Filter instead of VectorSearchFilter +// var options = new VectorSearchOptions +// { +// Filter = new VectorSearchFilter().EqualTo(nameof(BookContentChunk.FileName), fileName), +// Top = 1000 // Large number to get all chunks +// }; + +// var results = await collection.VectorizedSearchAsync( +// new ReadOnlyMemory(new float[1536]), // Dummy vector, we're filtering not searching +// options, +// cancellationToken); + +// var chunks = await results.Select(r => r.Record).ToListAsync(cancellationToken); + +// return chunks; +// } +// catch (Exception ex) +// { +// Console.WriteLine($"❌ Error getting chunks by file: {ex.Message}"); +// throw; +// } +// } + +// /// +// /// Get collection statistics +// /// +// /// Cancellation token +// /// Statistics about the collection +// public async Task GetStatisticsAsync(CancellationToken cancellationToken = default) +// { +// try +// { +// var collection = GetCollection(); + +// // Get all chunks to calculate statistics +// var searchResults = await collection.VectorizedSearchAsync( +// new ReadOnlyMemory(new float[1536]), // Dummy vector +// new() { Top = 10000 }, // Large number to get all +// cancellationToken); + +// var allChunks = await searchResults.Select(r => r.Record).ToListAsync(cancellationToken); + +// var fileGroups = allChunks.GroupBy(c => c.FileName).ToList(); + +// return new VectorStoreStatistics +// { +// TotalChunks = allChunks.Count, +// TotalFiles = fileGroups.Count, +// AverageChunksPerFile = fileGroups.Any() ? fileGroups.Average(g => g.Count()) : 0, +// ChunksWithEmbeddings = allChunks.Count(c => c.TextEmbedding.HasValue), +// TotalTokens = allChunks.Sum(c => c.TokenCount) +// }; +// } +// catch (Exception ex) +// { +// Console.WriteLine($"❌ Error getting statistics: {ex.Message}"); +// throw; +// } +// } + +// private VectorStoreCollection GetCollection() +// { +// return _vectorStore.GetCollection(_collectionName); +// } + +// public void Dispose() +// { +// _vectorStore?.Dispose(); +// } +//} + +///// +///// Statistics about the vector store +///// +//public record VectorStoreStatistics +//{ +// public int TotalChunks { get; init; } +// public int TotalFiles { get; init; } +// public double AverageChunksPerFile { get; init; } +// public int ChunksWithEmbeddings { get; init; } +// public int TotalTokens { get; init; } + +// public double EmbeddingCoverage => TotalChunks > 0 ? (double)ChunksWithEmbeddings / TotalChunks * 100 : 0; +//} diff --git a/EssentialCSharp.Chat.Tests/EssentialCSharp.Chat.Tests.csproj b/EssentialCSharp.Chat.Tests/EssentialCSharp.Chat.Tests.csproj new file mode 100644 index 00000000..11683e18 --- /dev/null +++ b/EssentialCSharp.Chat.Tests/EssentialCSharp.Chat.Tests.csproj @@ -0,0 +1,26 @@ + + + + net9.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + diff --git a/EssentialCSharp.Chat.Tests/MarkdownChunkingServiceTests.cs b/EssentialCSharp.Chat.Tests/MarkdownChunkingServiceTests.cs new file mode 100644 index 00000000..8aab8cb6 --- /dev/null +++ b/EssentialCSharp.Chat.Tests/MarkdownChunkingServiceTests.cs @@ -0,0 +1,192 @@ +using EssentialCSharp.Chat.Common.Services; +using Moq; + +namespace EssentialCSharp.Chat.Tests; +// TODO: Move to editorconfig later, just moving quick +#pragma warning disable CA1707 // Identifiers should not contain underscores +public class MarkdownChunkingServiceTests +{ + #region MarkdownContentToHeadersAndSection + [Fact] + public void MarkdownContentToHeadersAndSection_ParsesSampleMarkdown_CorrectlyCombinesHeadersAndExtractsContent() + { + string markdown = """ +### Beginner Topic +#### What Is a Method? + +Syntactically, a **method** in C# is a named block of code introduced by a method declaration (e.g., `static void Main()`) and (usually) followed by zero or more statements within curly braces. Methods perform computations and/or actions. Like paragraphs in written languages, methods provide a means of structuring and organizing code so that it is more readable. More important, methods can be reused and called from multiple places and so avoid the need to duplicate code. The method declaration introduces the method and defines the method name along with the data passed to and from the method. In Listing 1.8, `Main()` followed by `{ ... }` is an example of a C# method. + +## Main Method + +The location where C# programs begin execution is the **Main method**, which begins with `static void Main()`. When you execute the program by typing `dotnet run` on the terminal, the program starts with the Main method and begins executing the first statement, as identified in Listing 1.8. + + + +### Listing 1.8: Breaking Apart `HelloWorld` +publicclass Program // BEGIN Class definition +{ +publicstaticvoid Main() // Method declaration + { // BEGIN method implementation + Console.WriteLine( // This statement spans 2 lines +"Hello, My name is Inigo Montoya"); + } // END method implementation +} // END class definition +Although the Main method declaration can vary to some degree, `static` and the method name, `Main`, are always required for a program (see “Advanced Topic: Declaration of the Main Method”). + +The **comments**, text that begins with `//` in Listing 1.8, are explained later in the chapter. They are included to identify the various constructs in the listing. + +### Advanced Topic +#### Declaration of the Main Method + +C# requires that the Main method return either `void` or `int` and that it take either no parameters or a single array of strings. Listing 1.9 shows the full declaration of the Main method. The `args` parameter is an array of strings corresponding to the command-line arguments. The executable name is not included in the `args` array (unlike in C and C++). To retrieve the full command used to execute the program, including the program name, use `Environment.CommandLine`. +"""; + + var sections = MarkdownChunkingService.MarkdownContentToHeadersAndSection(markdown); + + Assert.Equal(3, sections.Count); + Assert.Contains(sections, s => s.Header == "Beginner Topic: What Is a Method?" && string.Join("\n", s.Content).Contains("Syntactically, a **method** in C# is a named block of code")); + Assert.Contains(sections, s => s.Header == "Main Method" && string.Join("\n", s.Content).Contains("The location where C# programs begin execution is the **Main method**, which begins with `static void Main()`") + && string.Join("\n", s.Content).Contains("publicclass Program")); + Assert.Contains(sections, s => s.Header == "Main Method: Advanced Topic: Declaration of the Main Method" && string.Join("\n", s.Content).Contains("C# requires that the Main method return either `void` or `int`")); + } + + [Fact] + public void MarkdownContentToHeadersAndSection_AppendsCodeListingToPriorSection() + { + string markdown = """ +## Working with Variables + +Now that you’ve been introduced to the most basic C# program, it’s time to declare a local variable. Once a variable is declared, you can assign it a value, replace that value with a new value, and use it in calculations, output, and so on. However, you cannot change the data type of the variable. In Listing 1.12, `string max` is a variable declaration. + + + +### Listing 1.12: Declaring and Assigning a Variable + +publicclass MiracleMax +{ +publicstaticvoid Main() + { +string max; // "string" identifies the data type +// "max" is the variable + max = "Have fun storming the castle!"; + Console.WriteLine(max); + } +} + +### Beginner Topic +#### Local Variables + +A **variable** is a name that refers to a value that can change over time. Local indicates that the programmer **declared** the variable within a method. + +To declare a variable is to define it, which you do by + +* Specifying the type of data which the variable will contain +* Assigning it an identifier (name) +"""; + + var sections = MarkdownChunkingService.MarkdownContentToHeadersAndSection(markdown); + + Assert.Equal(2, sections.Count); + // The code listing should be appended to the Working with Variables section, not as its own section + var workingWithVariablesSection = sections.FirstOrDefault(s => s.Header == "Working with Variables"); + Assert.True(!string.IsNullOrEmpty(workingWithVariablesSection.Header)); + Assert.Contains("publicclass MiracleMax", string.Join("\n", workingWithVariablesSection.Content)); + Assert.DoesNotContain(sections, s => s.Header == "Listing 1.12: Declaring and Assigning a Variable"); + } + + [Fact] + public void MarkdownContentToHeadersAndSection_KeepsPriorHeadersAppended() + { + string markdown = """ +### Beginner Topic +#### What Is a Data Type? + +The type of data that a variable declaration specifies is called a **data type** (or object type). A data type, or simply **type**, is a classification of things that share similar characteristics and behavior. For example, animal is a type. It classifies all things (monkeys, warthogs, and platypuses) that have animal characteristics (multicellular, capacity for locomotion, and so on). Similarly, in programming languages, a type is a definition for several items endowed with similar qualities. + +## Declaring a Variable + +In Listing 1.12, `string max` is a variable declaration of a string type whose name is `max`. It is possible to declare multiple variables within the same statement by specifying the data type once and separating each identifier with a comma. Listing 1.13 demonstrates such a declaration. + +### Listing 1.13: Declaring Two Variables within One Statement +string message1, message2; + +### Declaring another thing + +Because a multivariable declaration statement allows developers to provide the data type only once within a declaration, all variables will be of the same type. + +In C#, the name of the variable may begin with any letter or an underscore (`_`), followed by any number of letters, numbers, and/or underscores. By convention, however, local variable names are camelCased (the first letter in each word is capitalized, except for the first word) and do not include underscores. + +## Assigning a Variable + +After declaring a local variable, you must assign it a value before reading from it. One way to do this is to use the `=` **operator**, also known as the **simple assignment operator**. Operators are symbols used to identify the function the code is to perform. Listing 1.14 demonstrates how to use the assignment operator to designate the string values to which the variables `miracleMax` and `valerie` will point. + +### Listing 1.14: Changing the Value of a Variable +publicclass StormingTheCastle +{ +publicstaticvoid Main() + { +string valerie; +string miracleMax = "Have fun storming the castle!"; + + valerie = "Think it will work?"; + + Console.WriteLine(miracleMax); + Console.WriteLine(valerie); + + miracleMax = "It would take a miracle."; + Console.WriteLine(miracleMax); + } +} + +### Continued Learning +From this listing, observe that it is possible to assign a variable as part of the variable declaration (as it was for `miracleMax`) or afterward in a separate statement (as with the variable `valerie`). The value assigned must always be on the right side of the declaration. +"""; + + var sections = MarkdownChunkingService.MarkdownContentToHeadersAndSection(markdown); + Assert.Equal(5, sections.Count); + + Assert.Contains(sections, s => s.Header == "Beginner Topic: What Is a Data Type?" && string.Join("\n", s.Content).Contains("The type of data that a variable declaration specifies is called a **data type**")); + Assert.Contains(sections, s => s.Header == "Declaring a Variable" && string.Join("\n", s.Content).Contains("In Listing 1.12, `string max` is a variable declaration")); + Assert.Contains(sections, s => s.Header == "Declaring a Variable: Declaring another thing" && string.Join("\n", s.Content).Contains("Because a multivariable declaration statement allows developers to provide the data type only once")); + Assert.Contains(sections, s => s.Header == "Assigning a Variable" && string.Join("\n", s.Content).Contains("After declaring a local variable, you must assign it a value before reading from it.")); + Assert.Contains(sections, s => s.Header == "Assigning a Variable: Continued Learning" && string.Join("\n", s.Content).Contains("From this listing, observe that it is possible to assign a variable as part of the variable declaration")); + } + #endregion MarkdownContentToHeadersAndSection + + #region ProcessSingleMarkdownFile + [Fact] + public void ProcessSingleMarkdownFile_ProducesExpectedChunksAndHeaders() + { + // Arrange + var logger = new Mock>().Object; + var service = new MarkdownChunkingService(logger); + string[] fileContent = new[] + { + "## Section 1", + "This is the first section.", + "", + "### Listing 1.1: Example Listing", + "Console.WriteLine(\"Hello World\");", + "", + "## Section 2", + "This is the second section." + }; + string fileName = "TestFile.md"; + string filePath = "/path/to/TestFile.md"; + + // Act + var result = service.ProcessSingleMarkdownFile(fileContent, fileName, filePath); + + // Assert + Assert.NotNull(result); + Assert.Equal(fileName, result.FileName); + Assert.Equal(filePath, result.FilePath); + Assert.Contains("This is the first section.", string.Join("\n", result.Chunks)); + Assert.Contains("Console.WriteLine(\"Hello World\");", string.Join("\n", result.Chunks)); + Assert.Contains("This is the second section.", string.Join("\n", result.Chunks)); + Assert.Contains(result.Chunks, c => c.Contains("This is the second section.")); + } + #endregion ProcessSingleMarkdownFile +} + +#pragma warning restore CA1707 // Identifiers should not contain underscores diff --git a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj index eebf48b4..4ad5fc46 100644 --- a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj +++ b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj @@ -18,17 +18,18 @@ + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - diff --git a/EssentialCSharp.Chat/Program.cs b/EssentialCSharp.Chat/Program.cs index 12a6aa62..94fae072 100644 --- a/EssentialCSharp.Chat/Program.cs +++ b/EssentialCSharp.Chat/Program.cs @@ -1,252 +1,125 @@ using System.CommandLine; -using Microsoft.SemanticKernel.Text; +using System.Text.Json; +using EssentialCSharp.Chat.Common.Services; +using Microsoft.Extensions.Logging; namespace EssentialCSharp.Chat; public class Program { - private static readonly char[] _LineSeparators = ['\r', '\n']; + private static readonly JsonSerializerOptions _JsonOptions = new() { WriteIndented = true }; static int Main(string[] args) { - // Configure command-line options following System.CommandLine patterns - var directoryOption = new Option("--directory") + Option directoryOption = new("--directory") { - Description = "Directory containing markdown files to chunk", - DefaultValueFactory = _ => new DirectoryInfo(@"D:\EssentialCSharp.Web\EssentialCSharp.Web\Markdown\") + Description = "Directory containing markdown files.", + Required = true }; - - var maxTokensOption = new Option("--max-tokens") - { - Description = "Maximum tokens per chunk", - DefaultValueFactory = _ => 500 - }; - - var overlapTokensOption = new Option("--overlap") + Option filePatternOption = new("--file-pattern") { - Description = "Number of tokens to overlap between chunks", - DefaultValueFactory = _ => 50 - }; - - var chunkHeaderOption = new Option("--header") - { - Description = "Optional header to prepend to each chunk" - }; - - var filePatternOption = new Option("--pattern") - { - Description = "File pattern to match", + Description = "File pattern to match (e.g. *.md)", + Required = false, DefaultValueFactory = _ => "*.md" }; - - var outputFormatOption = new Option("--format") + Option outputDirectoryOption = new("--output-directory") { - Description = "Output format: summary, detailed, or json", - DefaultValueFactory = _ => "summary" + Description = "Directory to write chunked output files. If not provided, output is written to console.", + Required = false }; - // Create root command - var rootCommand = new RootCommand("Semantic Kernel TextChunker - Extract and Chunk Markdown Files") + RootCommand rootCommand = new("EssentialCSharp.Chat Utilities"); + + var chunkMarkdownCommand = new Command("chunk-markdown", "Chunk markdown files in a directory.") { directoryOption, - maxTokensOption, - overlapTokensOption, - chunkHeaderOption, filePatternOption, - outputFormatOption + outputDirectoryOption }; - - // Set the action for the root command - rootCommand.SetAction(parseResult => + chunkMarkdownCommand.SetAction(async parseResult => { var directory = parseResult.GetValue(directoryOption); - var maxTokens = parseResult.GetValue(maxTokensOption); - var overlapTokens = parseResult.GetValue(overlapTokensOption); - var chunkHeader = parseResult.GetValue(chunkHeaderOption); - var filePattern = parseResult.GetValue(filePatternOption); - var outputFormat = parseResult.GetValue(outputFormatOption); - - return ProcessMarkdownFiles(directory!, maxTokens, overlapTokens, chunkHeader, filePattern!, outputFormat!); - }); - - return rootCommand.Parse(args).Invoke(); - } - - /// - /// Process markdown files in the specified directory using Semantic Kernel's TextChunker - /// Following Microsoft Learn documentation for proper implementation - /// - internal static int ProcessMarkdownFiles( - DirectoryInfo directory, - int maxTokensPerParagraph, - int overlapTokens, - string? chunkHeader, - string filePattern, - string outputFormat) - { - try - { - // Validate input parameters - if (!directory.Exists) - { - Console.Error.WriteLine($"Error: Directory '{directory.FullName}' does not exist."); - return 1; - } - - if (maxTokensPerParagraph <= 0) - { - Console.Error.WriteLine("Error: max-tokens must be a positive number."); - return 1; - } - - if (overlapTokens < 0 || overlapTokens >= maxTokensPerParagraph) - { - Console.Error.WriteLine("Error: overlap-tokens must be between 0 and max-tokens."); - return 1; - } + var filePattern = parseResult.GetValue(filePatternOption) ?? "*.md"; + var outputDirectory = parseResult.GetValue(outputDirectoryOption); - // Find markdown files - var markdownFiles = directory.GetFiles(filePattern, SearchOption.TopDirectoryOnly); - - if (markdownFiles.Length == 0) + using var loggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole()); + var logger = loggerFactory.CreateLogger(); + var service = new MarkdownChunkingService(logger); + try { - Console.WriteLine($"No files matching pattern '{filePattern}' found in '{directory.FullName}'"); - return 0; - } - - Console.WriteLine($"Processing {markdownFiles.Length} markdown files..."); - Console.WriteLine($"Max tokens per chunk: {maxTokensPerParagraph}"); - Console.WriteLine($"Overlap tokens: {overlapTokens} ({(double)overlapTokens / maxTokensPerParagraph * 100:F1}%)"); - Console.WriteLine($"Chunk header: {(string.IsNullOrEmpty(chunkHeader) ? "None" : $"'{chunkHeader}'")}"); - Console.WriteLine(); - - int totalChunks = 0; - var results = new List(); + if (directory is null) + { + Console.Error.WriteLine("Error: Directory is required."); + return; + } + var results = await service.ProcessMarkdownFilesAsync(directory, filePattern); - foreach (var file in markdownFiles) - { - var result = ProcessSingleMarkdownFile(file, maxTokensPerParagraph, overlapTokens, chunkHeader); - results.Add(result); - totalChunks += result.ChunkCount; + int maxChunkLength = 0; + int minChunkLength = 0; - // Output per-file summary - Console.WriteLine($"File: {file.Name}"); - Console.WriteLine($" Original size: {result.OriginalCharCount:N0} characters"); - Console.WriteLine($" Chunks created: {result.ChunkCount}"); - Console.WriteLine($" Average chunk size: {(result.ChunkCount > 0 ? result.TotalChunkCharacters / result.ChunkCount : 0):N0} characters"); - Console.WriteLine(); + void WriteChunkingResult(FileChunkingResult result, TextWriter writer) + { + // lets build up some stats over the chunking + var chunkAverage = result.Chunks.Average(chunk => chunk.Length); + var chunkMedian = result.Chunks.OrderBy(chunk => chunk.Length).ElementAt(result.Chunks.Count / 2).Length; + var chunkMax = result.Chunks.Max(chunk => chunk.Length); + var chunkMin = result.Chunks.Min(chunk => chunk.Length); + var chunkTotal = result.Chunks.Sum(chunk => chunk.Length); + var chunkStandardDeviation = Math.Sqrt(result.Chunks.Average(chunk => Math.Pow(chunk.Length - chunkAverage, 2))); + var numberOfOutliers = result.Chunks.Count(chunk => chunk.Length > chunkAverage + chunkStandardDeviation); + + if (chunkMax > maxChunkLength) maxChunkLength = chunkMax; + if (chunkMin < minChunkLength || minChunkLength == 0) minChunkLength = chunkMin; + + writer.WriteLine($"File: {result.FileName}"); + writer.WriteLine($"Number of Chunks: {result.ChunkCount}"); + writer.WriteLine($"Average Chunk Length: {chunkAverage}"); + writer.WriteLine($"Median Chunk Length: {chunkMedian}"); + writer.WriteLine($"Max Chunk Length: {chunkMax}"); + writer.WriteLine($"Min Chunk Length: {chunkMin}"); + writer.WriteLine($"Total Chunk Characters: {chunkTotal}"); + writer.WriteLine($"Standard Deviation: {chunkStandardDeviation}"); + writer.WriteLine($"Number of Outliers: {numberOfOutliers}"); + writer.WriteLine($"Original Character Count: {result.OriginalCharCount}"); + writer.WriteLine($"New Character Count: {result.TotalChunkCharacters}"); + foreach (var chunk in result.Chunks) + { + writer.WriteLine(); + writer.WriteLine(chunk); + } + } - // Output detailed chunks if requested - if (outputFormat.Equals("detailed", StringComparison.OrdinalIgnoreCase)) + if (outputDirectory != null) { - OutputDetailedChunks(result); + if (!outputDirectory.Exists) + outputDirectory.Create(); + foreach (var result in results) + { + var outputFile = Path.Combine(outputDirectory.FullName, Path.GetFileNameWithoutExtension(result.FileName) + ".chunks.txt"); + using var writer = new StreamWriter(outputFile, false); + WriteChunkingResult(result, writer); + Console.WriteLine($"Wrote: {outputFile}"); + } } + else + { + foreach (var result in results) + { + WriteChunkingResult(result, Console.Out); + } + } + Console.WriteLine($"Max Chunk Length: {maxChunkLength}"); + Console.WriteLine($"Min Chunk Length: {minChunkLength}"); } - - // Output summary - Console.WriteLine("=== SUMMARY ==="); - Console.WriteLine($"Total files processed: {markdownFiles.Length}"); - Console.WriteLine($"Total chunks created: {totalChunks}"); - Console.WriteLine($"Average chunks per file: {(markdownFiles.Length > 0 ? (double)totalChunks / markdownFiles.Length : 0):F1}"); - - // Output JSON if requested - if (outputFormat.Equals("json", StringComparison.OrdinalIgnoreCase)) + catch (Exception ex) { - OutputJsonResults(results); + Console.Error.WriteLine($"Error: {ex.Message}"); + return; } + }); + rootCommand.Subcommands.Add(chunkMarkdownCommand); - return 0; - } - catch (Exception ex) - { - Console.Error.WriteLine($"Error: {ex.Message}"); - return 1; - } - } - - /// - /// Process a single markdown file using Semantic Kernel's SplitMarkdownParagraphs method - /// Implementation follows Microsoft Learn documentation exactly - /// - internal static FileChunkingResult ProcessSingleMarkdownFile( - FileInfo file, - int maxTokensPerParagraph, - int overlapTokens, - string? chunkHeader) - { - // Read the markdown content - var content = File.ReadAllText(file.FullName); - - // Prepare lines for chunking - following Microsoft examples - var lines = content.Split(_LineSeparators, StringSplitOptions.RemoveEmptyEntries).ToList(); - - // Apply Semantic Kernel TextChunker.SplitMarkdownParagraphs - // Following the exact API signature from Microsoft Learn documentation - // Suppress the experimental warning as this is the intended usage per documentation -#pragma warning disable SKEXP0050 - var chunks = TextChunker.SplitMarkdownParagraphs( - lines, - maxTokensPerParagraph, - overlapTokens, - chunkHeader); -#pragma warning restore SKEXP0050 - - // Calculate statistics - var result = new FileChunkingResult - { - FileName = file.Name, - FilePath = file.FullName, - OriginalCharCount = content.Length, - ChunkCount = chunks.Count, - Chunks = chunks, - TotalChunkCharacters = chunks.Sum(c => c.Length) - }; - - return result; - } - - /// - /// Output detailed chunk information for inspection - /// - internal static void OutputDetailedChunks(FileChunkingResult result) - { - Console.WriteLine($"=== DETAILED CHUNKS for {result.FileName} ==="); - - for (int i = 0; i < result.Chunks.Count; i++) - { - var chunk = result.Chunks[i]; - Console.WriteLine($"Chunk {i + 1}/{result.Chunks.Count}:"); - Console.WriteLine($" Length: {chunk.Length} characters"); - Console.WriteLine($" Preview: {chunk.Substring(0, Math.Min(100, chunk.Length)).Replace('\n', ' ').Replace('\r', ' ')}..."); - Console.WriteLine(" ---"); - } - Console.WriteLine(); + return rootCommand.Parse(args).Invoke(); } - /// - /// Output results in JSON format for programmatic consumption - /// - internal static void OutputJsonResults(List results) - { - Console.WriteLine(); - Console.WriteLine("=== JSON OUTPUT ==="); - Console.WriteLine("{"); - Console.WriteLine(" \"results\": ["); - - for (int i = 0; i < results.Count; i++) - { - var result = results[i]; - Console.WriteLine(" {"); - Console.WriteLine($" \"fileName\": \"{result.FileName}\","); - Console.WriteLine($" \"filePath\": \"{result.FilePath.Replace("\\", "\\\\")}\","); - Console.WriteLine($" \"originalCharCount\": {result.OriginalCharCount},"); - Console.WriteLine($" \"chunkCount\": {result.ChunkCount},"); - Console.WriteLine($" \"totalChunkCharacters\": {result.TotalChunkCharacters},"); - Console.WriteLine($" \"averageChunkSize\": {(result.ChunkCount > 0 ? result.TotalChunkCharacters / result.ChunkCount : 0)}"); - Console.WriteLine(i < results.Count - 1 ? " }," : " }"); - } - - Console.WriteLine(" ]"); - Console.WriteLine("}"); - } } diff --git a/EssentialCSharp.Chat/packages.config b/EssentialCSharp.Chat/packages.config deleted file mode 100644 index 61c14644..00000000 --- a/EssentialCSharp.Chat/packages.config +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/EssentialCSharp.Chat/requirements.txt b/EssentialCSharp.Chat/requirements.txt deleted file mode 100644 index 1ed83edc..00000000 --- a/EssentialCSharp.Chat/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -semantic-kernel==0.9.3b1 -mistune==3.0.1 \ No newline at end of file diff --git a/EssentialCSharp.Web.sln b/EssentialCSharp.Web.sln index 66ae444a..4eacae36 100644 --- a/EssentialCSharp.Web.sln +++ b/EssentialCSharp.Web.sln @@ -1,4 +1,5 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 + +Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.32014.148 MinimumVisualStudioVersion = 10.0.40219.1 @@ -26,6 +27,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.Chat.Common EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.VectorDbBuilder", "EssentialCSharp.VectorDbBuilder\EssentialCSharp.VectorDbBuilder.csproj", "{8F7A5E12-B234-4C8A-9D45-12F456789ABC}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.Chat.Tests", "EssentialCSharp.Chat.Tests\EssentialCSharp.Chat.Tests.csproj", "{05CC9D8A-D928-4537-AD09-737C43DFC00D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -52,6 +55,10 @@ Global {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Debug|Any CPU.Build.0 = Debug|Any CPU {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Release|Any CPU.ActiveCfg = Release|Any CPU {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Release|Any CPU.Build.0 = Release|Any CPU + {05CC9D8A-D928-4537-AD09-737C43DFC00D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05CC9D8A-D928-4537-AD09-737C43DFC00D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05CC9D8A-D928-4537-AD09-737C43DFC00D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05CC9D8A-D928-4537-AD09-737C43DFC00D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From e0b871bf5c730b08ae2627ef6556deaeaa6e0a7f Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Thu, 17 Jul 2025 00:22:41 -0700 Subject: [PATCH 06/22] Markdown chunking and embedding service setup --- Directory.Packages.props | 1 + .../EssentialCSharp.Chat.Common.csproj | 1 + .../Models/AIOptions.cs | 22 +++ .../Models/BookContentChunk.cs | 4 +- .../Services/ChunkingResultExtensions.cs | 57 +++++++ .../Services/EmbeddingService.cs | 159 ++++-------------- .../EssentialCSharp.Chat.csproj | 4 + EssentialCSharp.Chat/Program.cs | 74 ++++++++ EssentialCSharp.Web/appsettings.json | 6 + 9 files changed, 199 insertions(+), 129 deletions(-) create mode 100644 EssentialCSharp.Chat.Shared/Models/AIOptions.cs create mode 100644 EssentialCSharp.Chat.Shared/Services/ChunkingResultExtensions.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index e94603b5..4ab69113 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -15,6 +15,7 @@ + diff --git a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj index fb19d698..8eb086c3 100644 --- a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj +++ b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj @@ -13,6 +13,7 @@ + diff --git a/EssentialCSharp.Chat.Shared/Models/AIOptions.cs b/EssentialCSharp.Chat.Shared/Models/AIOptions.cs new file mode 100644 index 00000000..89ba03c7 --- /dev/null +++ b/EssentialCSharp.Chat.Shared/Models/AIOptions.cs @@ -0,0 +1,22 @@ +namespace EssentialCSharp.Chat; + +public class AIOptions +{ + /// + /// The Azure OpenAI deployment name for text embedding generation. + /// + public string VectorGenerationDeploymentName { get; set; } = string.Empty; + /// + /// The Azure OpenAI endpoint URL. + /// + public string Endpoint { get; set; } = string.Empty; + /// + /// The API key for accessing Azure OpenAI services. + /// + public string ApiKey { get; set; } = string.Empty; + + /// + /// The PostgreSQL connection string for the vector store. + /// + public string PostgresConnectionString { get; set; } = string.Empty; +} diff --git a/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs b/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs index 959dc482..fd9cf539 100644 --- a/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs +++ b/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs @@ -6,7 +6,7 @@ namespace EssentialCSharp.Chat.Common.Models; /// Represents a chunk of markdown content for vector search /// Following Semantic Kernel Vector Store best practices from Microsoft docs /// -public sealed class BookContentChunk +public class BookContentChunk { /// /// Unique identifier for the chunk - serves as the vector store key @@ -48,6 +48,6 @@ public sealed class BookContentChunk /// Vector embedding for the chunk text - will be generated by embedding service /// Using 1536 dimensions for Azure OpenAI text-embedding-ada-002 /// - [VectorStoreVector(Dimensions: 4, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Hnsw)] + [VectorStoreVector(Dimensions: 1536, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Hnsw)] public ReadOnlyMemory? TextEmbedding { get; set; } } diff --git a/EssentialCSharp.Chat.Shared/Services/ChunkingResultExtensions.cs b/EssentialCSharp.Chat.Shared/Services/ChunkingResultExtensions.cs new file mode 100644 index 00000000..c5709d06 --- /dev/null +++ b/EssentialCSharp.Chat.Shared/Services/ChunkingResultExtensions.cs @@ -0,0 +1,57 @@ +using System.Security.Cryptography; +using System.Text; +using EssentialCSharp.Chat.Common.Models; + +namespace EssentialCSharp.Chat.Common.Services; + +public static class ChunkingResultExtensions +{ + public static List ToBookContentChunks(this FileChunkingResult result) + { + var chunks = new List(); + int? chapterNumber = ExtractChapterNumber(result.FileName); + + foreach (var chunk in result.Chunks) + { + string chunkText = chunk; + string contentHash = ComputeSha256Hash(chunkText); + + chunks.Add(new BookContentChunk + { + Id = Guid.NewGuid().ToString(), + FileName = result.FileName, + Heading = ExtractHeading(chunkText), + ChunkText = chunkText, + ChapterNumber = chapterNumber, + ContentHash = contentHash + }); + } + return chunks; + } + + private static string ExtractHeading(string chunkText) + { + // get characters until the first " - " or newline + var firstLine = chunkText.Split(["\r\n", "\r", "\n"], StringSplitOptions.None)[0]; + var headingParts = firstLine.Split([" - "], StringSplitOptions.None); + return headingParts.Length > 0 ? headingParts[0].Trim() : string.Empty; + } + + private static int ExtractChapterNumber(string fileName) + { + // Example: "Chapter01.md" -> 1 + // Regex: Chapter(?[0-9]{2}) + var match = System.Text.RegularExpressions.Regex.Match(fileName, @"Chapter(?\d{2})"); + if (match.Success && int.TryParse(match.Groups["ChapterNumber"].Value, out int chapterNumber)) + { + return chapterNumber; + } + throw new InvalidOperationException($"File name '{fileName}' does not contain a valid chapter number in the expected format."); + } + + private static string ComputeSha256Hash(string text) + { + var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(text)); + return Convert.ToHexStringLower(bytes); + } +} diff --git a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs index 42079c26..cae93d32 100644 --- a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs +++ b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs @@ -1,130 +1,35 @@ -namespace EssentialCSharp.Chat.Common.Services; - -///// -///// Service for generating embeddings for markdown chunks using Azure OpenAI -///// Following Microsoft best practices for Semantic Kernel embedding generation -///// -//public class EmbeddingService -//{ -// private readonly IEmbeddingGenerator> _EmbeddingGenerator; - -// public EmbeddingService(IEmbeddingGenerator> embeddingGenerator) -// { -// _EmbeddingGenerator = embeddingGenerator ?? throw new ArgumentNullException(nameof(embeddingGenerator)); -// } - -// /// -// /// Generate embeddings for a collection of markdown chunks -// /// -// /// The chunks to generate embeddings for -// /// Cancellation token -// /// The chunks with embeddings populated -// public async Task> GenerateEmbeddingsAsync( -// IList chunks, -// CancellationToken cancellationToken = default) -// { -// if (!chunks.Any()) -// return chunks; - -// Console.WriteLine($"Generating embeddings for {chunks.Count} chunks..."); - -// try -// { -// // Generate embeddings one by one to avoid rate limits -// for (int i = 0; i < chunks.Count; i++) -// { -// var chunk = chunks[i]; -// if (string.IsNullOrWhiteSpace(chunk.ChunkText)) -// continue; - -// var embedding = await _EmbeddingGenerator.GenerateEmbeddingAsync( -// chunk.ChunkText, -// options: null, -// cancellationToken); -// chunk.TextEmbedding = new ReadOnlyMemory(embedding.Vector.ToArray()); - -// // Show progress -// if ((i + 1) % 10 == 0) -// { -// Console.WriteLine($" Generated embeddings for {i + 1}/{chunks.Count} chunks"); -// } -// } +using EssentialCSharp.Chat.Common.Models; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Embeddings; -// var successfulEmbeddings = chunks.Count(c => c.TextEmbedding.HasValue); -// Console.WriteLine($"✅ Successfully generated embeddings for {successfulEmbeddings}/{chunks.Count} chunks"); -// if (successfulEmbeddings > 0) -// { -// Console.WriteLine($" Vector dimensions: {chunks.First(c => c.TextEmbedding.HasValue).TextEmbedding!.Value.Length}"); -// } -// } -// catch (Exception ex) -// { -// Console.WriteLine($"❌ Error generating embeddings: {ex.Message}"); -// throw; -// } - -// return chunks; -// } - -// /// -// /// Generate embedding for a single text -// /// -// /// The text to generate embedding for -// /// Cancellation token -// /// The generated embedding -// public async Task> GenerateEmbeddingAsync( -// string text, -// CancellationToken cancellationToken = default) -// { -// if (string.IsNullOrWhiteSpace(text)) -// throw new ArgumentException("Text cannot be null or empty", nameof(text)); - -// try -// { -// var embedding = await _EmbeddingGenerator.GenerateEmbeddingAsync( -// text, -// options: null, -// cancellationToken); -// return new ReadOnlyMemory(embedding.Vector.ToArray()); -// } -// catch (Exception ex) -// { -// Console.WriteLine($"❌ Error generating embedding for text: {ex.Message}"); -// throw; -// } -// } - -// /// -// /// Get embedding generation statistics -// /// -// /// The chunks that have embeddings -// /// Statistics about the embeddings -// public static EmbeddingStatistics GetEmbeddingStatistics(IList chunks) -// { -// var chunksWithEmbeddings = chunks.Where(c => c.TextEmbedding.HasValue).ToList(); - -// return new EmbeddingStatistics -// { -// TotalChunks = chunks.Count, -// ChunksWithEmbeddings = chunksWithEmbeddings.Count, -// EmbeddingDimensions = chunksWithEmbeddings.FirstOrDefault()?.TextEmbedding?.Length ?? 0, -// AverageTextLength = chunks.Any() ? (int)chunks.Average(c => c.ChunkText.Length) : 0, -// TotalTextCharacters = chunks.Sum(c => c.ChunkText.Length) -// }; -// } -//} - -///// -///// Statistics about embedding generation -///// -//public record EmbeddingStatistics -//{ -// public int TotalChunks { get; init; } -// public int ChunksWithEmbeddings { get; init; } -// public int EmbeddingDimensions { get; init; } -// public int AverageTextLength { get; init; } -// public int TotalTextCharacters { get; init; } +namespace EssentialCSharp.Chat.Common.Services; -// public double EmbeddingCoverage => TotalChunks > 0 ? (double)ChunksWithEmbeddings / TotalChunks * 100 : 0; -//} +/// +/// Service for generating embeddings for markdown chunks using Azure OpenAI +/// +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +public class EmbeddingService(VectorStore vectorStore, ITextEmbeddingGenerationService textEmbeddingGenerationService) +{ + /// + /// Generate an embedding for each text paragraph and upload it to the specified collection. + /// + /// The name of the collection to upload the text paragraphs to. + /// The text paragraphs to upload. + /// An async task. + public async Task GenerateEmbeddingsAndUpload(string collectionName, IEnumerable bookContents) + { + var collection = vectorStore.GetCollection(collectionName); + await collection.EnsureCollectionExistsAsync(); + + foreach (var chunk in bookContents) + { + // Generate the text embedding. + chunk.TextEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(chunk.ChunkText); + + await collection.UpsertAsync(chunk); + } + Console.WriteLine($"Successfully generated embeddings and uploaded {bookContents.Count()} chunks to collection '{collectionName}'."); + } +} +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. diff --git a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj index 4ad5fc46..4c2fe3e3 100644 --- a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj +++ b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj @@ -18,11 +18,15 @@ + + + + all diff --git a/EssentialCSharp.Chat/Program.cs b/EssentialCSharp.Chat/Program.cs index 94fae072..02f3b8b0 100644 --- a/EssentialCSharp.Chat/Program.cs +++ b/EssentialCSharp.Chat/Program.cs @@ -1,7 +1,10 @@ using System.CommandLine; using System.Text.Json; using EssentialCSharp.Chat.Common.Services; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; namespace EssentialCSharp.Chat; @@ -11,6 +14,9 @@ public class Program static int Main(string[] args) { + + +#pragma warning restore SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. Option directoryOption = new("--directory") { Description = "Directory containing markdown files.", @@ -36,6 +42,72 @@ static int Main(string[] args) filePatternOption, outputDirectoryOption }; + + var buildVectorDbCommand = new Command("build-vector-db", "Build a vector database from markdown chunks.") + { + directoryOption, + filePatternOption, + }; + + buildVectorDbCommand.SetAction(async parseResult => + { + // Replace with your values. + IConfigurationRoot config = new ConfigurationBuilder() + .SetBasePath(IntelliTect.Multitool.RepositoryPaths.GetDefaultRepoRoot()) + .AddJsonFile("EssentialCSharp.Web/appsettings.json") + .AddUserSecrets() + .AddEnvironmentVariables() + .Build(); + + AIOptions aiOptions = config.GetRequiredSection("AIOptions").Get() ?? throw new InvalidOperationException( + "AIOptions section is missing or not configured correctly in appsettings.json or environment variables."); + + // Register Azure OpenAI text embedding generation service and Redis vector store. +#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var builder = Kernel.CreateBuilder() + .AddAzureOpenAITextEmbeddingGeneration(aiOptions.VectorGenerationDeploymentName, aiOptions.Endpoint, aiOptions.ApiKey); +#pragma warning restore SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + builder.Services.AddLogging(loggingBuilder => + { + loggingBuilder.AddSimpleConsole(options => + { + options.TimestampFormat = "HH:mm:ss "; + options.SingleLine = true; + }); + }); + + builder.Services.AddPostgresVectorStore( + aiOptions.PostgresConnectionString); + + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + + // Build the kernel and get the data uploader. + var kernel = builder.Build(); + var directory = parseResult.GetValue(directoryOption); + var filePattern = parseResult.GetValue(filePatternOption) ?? "*.md"; + var markdownService = kernel.GetRequiredService(); + try + { + if (directory is null) + { + Console.Error.WriteLine("Error: Directory is required."); + return; + } + var results = await markdownService.ProcessMarkdownFilesAsync(directory, filePattern); + // Convert results to BookContentChunks + var bookContentChunks = results.SelectMany(result => result.ToBookContentChunks()).ToList(); + // Generate embeddings and upload to vector store + var embeddingService = kernel.GetRequiredService(); + await embeddingService.GenerateEmbeddingsAndUpload("markdown_chunks", bookContentChunks); + Console.WriteLine($"Successfully processed {bookContentChunks.Count} chunks."); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Error: {ex.Message}"); + throw; + } + }); chunkMarkdownCommand.SetAction(async parseResult => { var directory = parseResult.GetValue(directoryOption); @@ -118,8 +190,10 @@ void WriteChunkingResult(FileChunkingResult result, TextWriter writer) } }); rootCommand.Subcommands.Add(chunkMarkdownCommand); + rootCommand.Subcommands.Add(buildVectorDbCommand); return rootCommand.Parse(args).Invoke(); } + } diff --git a/EssentialCSharp.Web/appsettings.json b/EssentialCSharp.Web/appsettings.json index 711f6a36..5780834f 100644 --- a/EssentialCSharp.Web/appsettings.json +++ b/EssentialCSharp.Web/appsettings.json @@ -9,5 +9,11 @@ "HCaptcha": { "SecretKey": "0x0000000000000000000000000000000000000000", "SiteKey": "10000000-ffff-ffff-ffff-000000000001" + }, + "AIOptions": { + "VectorGenerationDeploymentName": "gpt-35-turbo", + "Endpoint": "your-endpoint-here", + "ApiKey": "your-api-key-here", + "PostgresConnectionString": "your-postgres-connection-string-here" } } \ No newline at end of file From 41915c99859c41eccebff511402aa51e0dda5c94 Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Thu, 17 Jul 2025 00:52:52 -0700 Subject: [PATCH 07/22] Embeddings working --- .../Services/EmbeddingService.cs | 23 +++++++++++++------ EssentialCSharp.Chat/Program.cs | 4 ++-- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs index cae93d32..32c91e5e 100644 --- a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs +++ b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs @@ -2,7 +2,6 @@ using Microsoft.Extensions.VectorData; using Microsoft.SemanticKernel.Embeddings; - namespace EssentialCSharp.Chat.Common.Services; /// @@ -11,24 +10,34 @@ namespace EssentialCSharp.Chat.Common.Services; #pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. public class EmbeddingService(VectorStore vectorStore, ITextEmbeddingGenerationService textEmbeddingGenerationService) { + public string CollectionName { get; } = "markdown_chunks"; /// /// Generate an embedding for each text paragraph and upload it to the specified collection. /// /// The name of the collection to upload the text paragraphs to. /// The text paragraphs to upload. /// An async task. - public async Task GenerateEmbeddingsAndUpload(string collectionName, IEnumerable bookContents) + public async Task GenerateEmbeddingsAndUpload(IEnumerable bookContents, CancellationToken cancellationToken, string? collectionName = null) { + collectionName ??= CollectionName; + var collection = vectorStore.GetCollection(collectionName); - await collection.EnsureCollectionExistsAsync(); + await collection.EnsureCollectionExistsAsync(cancellationToken); + + ParallelOptions parallelOptions = new() + { + MaxDegreeOfParallelism = 5, + CancellationToken = cancellationToken + }; - foreach (var chunk in bookContents) + await Parallel.ForEachAsync(bookContents, parallelOptions, async (chunk, cancellationToken) => { // Generate the text embedding. - chunk.TextEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(chunk.ChunkText); + chunk.TextEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(chunk.ChunkText, cancellationToken: cancellationToken); - await collection.UpsertAsync(chunk); - } + await collection.UpsertAsync(chunk, cancellationToken); + Console.WriteLine($"Uploaded chunk '{chunk.Id}' to collection '{collectionName}' for file '{chunk.FileName}' with heading '{chunk.Heading}'."); + }); Console.WriteLine($"Successfully generated embeddings and uploaded {bookContents.Count()} chunks to collection '{collectionName}'."); } } diff --git a/EssentialCSharp.Chat/Program.cs b/EssentialCSharp.Chat/Program.cs index 02f3b8b0..5045f1f1 100644 --- a/EssentialCSharp.Chat/Program.cs +++ b/EssentialCSharp.Chat/Program.cs @@ -49,7 +49,7 @@ static int Main(string[] args) filePatternOption, }; - buildVectorDbCommand.SetAction(async parseResult => + buildVectorDbCommand.SetAction(async (ParseResult parseResult, CancellationToken cancellationToken) => { // Replace with your values. IConfigurationRoot config = new ConfigurationBuilder() @@ -99,7 +99,7 @@ static int Main(string[] args) var bookContentChunks = results.SelectMany(result => result.ToBookContentChunks()).ToList(); // Generate embeddings and upload to vector store var embeddingService = kernel.GetRequiredService(); - await embeddingService.GenerateEmbeddingsAndUpload("markdown_chunks", bookContentChunks); + await embeddingService.GenerateEmbeddingsAndUpload(bookContentChunks, cancellationToken, "markdown_chunks"); Console.WriteLine($"Successfully processed {bookContentChunks.Count} chunks."); } catch (Exception ex) From cfd6f3c6f916fae015be0cb07b06824c0b82c8af Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Thu, 17 Jul 2025 02:08:09 -0700 Subject: [PATCH 08/22] Begin Chat feature work --- .../Models/AIOptions.cs | 11 + .../Models/BookContentChunk.cs | 7 +- .../Services/AIChatService.cs | 43 ++++ .../Services/AISearchService.cs | 48 ++++ .../Services/EmbeddingService.cs | 21 +- .../Services/VectorStoreService.cs | 238 ------------------ EssentialCSharp.Chat/Program.cs | 24 +- 7 files changed, 134 insertions(+), 258 deletions(-) create mode 100644 EssentialCSharp.Chat.Shared/Services/AIChatService.cs create mode 100644 EssentialCSharp.Chat.Shared/Services/AISearchService.cs delete mode 100644 EssentialCSharp.Chat.Shared/Services/VectorStoreService.cs diff --git a/EssentialCSharp.Chat.Shared/Models/AIOptions.cs b/EssentialCSharp.Chat.Shared/Models/AIOptions.cs index 89ba03c7..202d50f2 100644 --- a/EssentialCSharp.Chat.Shared/Models/AIOptions.cs +++ b/EssentialCSharp.Chat.Shared/Models/AIOptions.cs @@ -6,6 +6,17 @@ public class AIOptions /// The Azure OpenAI deployment name for text embedding generation. /// public string VectorGenerationDeploymentName { get; set; } = string.Empty; + + /// + /// The Azure OpenAI deployment name for chat completions. + /// + public string ChatDeploymentName { get; set; } = string.Empty; + + /// + /// The system prompt to use for the chat model. + /// + public string SystemPrompt { get; set; } = string.Empty; + /// /// The Azure OpenAI endpoint URL. /// diff --git a/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs b/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs index fd9cf539..276c4b7f 100644 --- a/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs +++ b/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs @@ -3,10 +3,9 @@ namespace EssentialCSharp.Chat.Common.Models; /// -/// Represents a chunk of markdown content for vector search -/// Following Semantic Kernel Vector Store best practices from Microsoft docs +/// Represents a chunk of book content for vector search /// -public class BookContentChunk +public sealed class BookContentChunk { /// /// Unique identifier for the chunk - serves as the vector store key @@ -46,7 +45,7 @@ public class BookContentChunk /// /// Vector embedding for the chunk text - will be generated by embedding service - /// Using 1536 dimensions for Azure OpenAI text-embedding-ada-002 + /// Using 1536 dimensions for Azure OpenAI text-embedding-3-small-v1 /// [VectorStoreVector(Dimensions: 1536, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Hnsw)] public ReadOnlyMemory? TextEmbedding { get; set; } diff --git a/EssentialCSharp.Chat.Shared/Services/AIChatService.cs b/EssentialCSharp.Chat.Shared/Services/AIChatService.cs new file mode 100644 index 00000000..421b89bd --- /dev/null +++ b/EssentialCSharp.Chat.Shared/Services/AIChatService.cs @@ -0,0 +1,43 @@ +using System.ClientModel; +using Azure.AI.OpenAI; +using Microsoft.Extensions.Options; +using OpenAI.Chat; + +namespace EssentialCSharp.Chat.Common.Services; + +public class AIChatService(AzureOpenAIClient aIClient, IOptions options) +{ + private readonly AIOptions _Options = options.Value; + private ChatClient ChatClient { get; } = aIClient.GetChatClient(options.Value.ChatDeploymentName); + + public async Task GetChatCompletion(string prompt) + { + var response = await ChatClient.CompleteChatAsync([ + new SystemChatMessage(_Options.SystemPrompt), + new UserChatMessage(prompt) + ]); + + // Todo: Handle response errors and check for multiple messages? + return response.Value.Content[0].Text; + } + + // TODO: Implement streaming chat completions + public AsyncCollectionResult GetChatCompletionStream(string prompt) + { + return ChatClient.CompleteChatStreamingAsync([ + new SystemChatMessage(_Options.SystemPrompt), + new UserChatMessage(prompt) + ]); + } + + // TODO: Implement batch chat completions for hybrid search + // public async Task> GetBatchChatCompletionsAsync(IEnumerable prompts) + // { + //#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + // var batchClient = aIClient.GetBatchClient(); + //#pragma warning restore OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + // } + + // TODO: Look into using UserSecurityContext (https://learn.microsoft.com/en-us/azure/defender-for-cloud/gain-end-user-context-ai) +} diff --git a/EssentialCSharp.Chat.Shared/Services/AISearchService.cs b/EssentialCSharp.Chat.Shared/Services/AISearchService.cs new file mode 100644 index 00000000..fb8eea61 --- /dev/null +++ b/EssentialCSharp.Chat.Shared/Services/AISearchService.cs @@ -0,0 +1,48 @@ +using EssentialCSharp.Chat.Common.Models; +using Microsoft.Extensions.VectorData; + +namespace EssentialCSharp.Chat.Common.Services; + +#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +public class AISearchService(VectorStore vectorStore, EmbeddingService embeddingService) +#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +{ + // TODO: Implement Hybrid Search functionality, may need to switch db providers to support full text search? + //public async Task ExecuteHybridSearch(string query, string? collectionName = null) + //{ + // collectionName ??= EmbeddingService.CollectionName; + + // IKeywordHybridSearchable collection = (IKeywordHybridSearchable)vectorStore.GetCollection(collectionName); + + // ReadOnlyMemory searchVector = await embeddingService.GenerateEmbeddingAsync(query); + + // var hybridSearchOptions = new HybridSearchOptions + // { + + // }; + + // var searchResults = await collection.HybridSearchAsync (searchVector, ["C#"], top: 3); + // foreach (var result in results) + // { + // Console.WriteLine($"Found chunk: {result.Value.Heading} in file {result.Value.FileName}"); + // } + //} + + public async Task>> ExecuteVectorSearch(string query, string? collectionName = null) + { + collectionName ??= EmbeddingService.CollectionName; + + VectorStoreCollection collection = vectorStore.GetCollection(collectionName); + + ReadOnlyMemory searchVector = await embeddingService.GenerateEmbeddingAsync(query); + + var vectorSearchOptions = new VectorSearchOptions + { + VectorProperty = x => x.TextEmbedding, + }; + + var searchResults = collection.SearchAsync(searchVector, options: vectorSearchOptions, top: 3); + + return searchResults; + } +} diff --git a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs index 32c91e5e..4caaa2e1 100644 --- a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs +++ b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs @@ -10,14 +10,25 @@ namespace EssentialCSharp.Chat.Common.Services; #pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. public class EmbeddingService(VectorStore vectorStore, ITextEmbeddingGenerationService textEmbeddingGenerationService) { - public string CollectionName { get; } = "markdown_chunks"; + public static string CollectionName { get; } = "markdown_chunks"; + + /// + /// Generate an embedding for the given text. + /// + /// The text to generate an embedding for. + /// The cancellation token. + /// A search vector as ReadOnlyMemory<float>. + public async Task> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default) + { + return await textEmbeddingGenerationService.GenerateEmbeddingAsync(text, cancellationToken: cancellationToken); + } + /// /// Generate an embedding for each text paragraph and upload it to the specified collection. /// /// The name of the collection to upload the text paragraphs to. - /// The text paragraphs to upload. /// An async task. - public async Task GenerateEmbeddingsAndUpload(IEnumerable bookContents, CancellationToken cancellationToken, string? collectionName = null) + public async Task GenerateBookContentEmbeddingsAndUploadToVectorStore(IEnumerable bookContents, CancellationToken cancellationToken, string? collectionName = null) { collectionName ??= CollectionName; @@ -32,8 +43,8 @@ public async Task GenerateEmbeddingsAndUpload(IEnumerable book await Parallel.ForEachAsync(bookContents, parallelOptions, async (chunk, cancellationToken) => { - // Generate the text embedding. - chunk.TextEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(chunk.ChunkText, cancellationToken: cancellationToken); + // Generate the text embedding using the new method. + chunk.TextEmbedding = await GenerateEmbeddingAsync(chunk.ChunkText, cancellationToken); await collection.UpsertAsync(chunk, cancellationToken); Console.WriteLine($"Uploaded chunk '{chunk.Id}' to collection '{collectionName}' for file '{chunk.FileName}' with heading '{chunk.Heading}'."); diff --git a/EssentialCSharp.Chat.Shared/Services/VectorStoreService.cs b/EssentialCSharp.Chat.Shared/Services/VectorStoreService.cs deleted file mode 100644 index e21e6da1..00000000 --- a/EssentialCSharp.Chat.Shared/Services/VectorStoreService.cs +++ /dev/null @@ -1,238 +0,0 @@ -namespace EssentialCSharp.Chat.Common.Services; - -///// -///// Service for storing and retrieving markdown chunks in PostgreSQL with pgvector -///// Following Microsoft best practices for Semantic Kernel Vector Store -///// -//public class VectorStoreService -//{ -// private readonly PostgresVectorStore _vectorStore; -// private readonly string _collectionName; - -// public VectorStoreService(string connectionString, string collectionName = "markdown_chunks") -// { -// if (string.IsNullOrWhiteSpace(connectionString)) -// throw new ArgumentException("Connection string cannot be null or empty", nameof(connectionString)); - -// _collectionName = collectionName; - -// // Create PostgreSQL vector store using the connection string -// _vectorStore = new PostgresVectorStore(connectionString); -// } - -// public VectorStoreService(PostgresVectorStore vectorStore, string collectionName = "markdown_chunks") -// { -// _vectorStore = vectorStore ?? throw new ArgumentNullException(nameof(vectorStore)); -// _collectionName = collectionName; -// } - -// /// -// /// Initialize the vector store collection -// /// -// /// Cancellation token -// public async Task InitializeAsync(CancellationToken cancellationToken = default) -// { -// Console.WriteLine($"Initializing vector store collection '{_collectionName}'..."); - -// try -// { -// var collection = GetCollection(); - -// // Create the collection if it doesn't exist -// await collection.CreateCollectionIfNotExistsAsync(cancellationToken); - -// Console.WriteLine($"✅ Vector store collection '{_collectionName}' is ready"); -// } -// catch (Exception ex) -// { -// Console.WriteLine($"❌ Error initializing vector store: {ex.Message}"); -// throw; -// } -// } - -// /// -// /// Store markdown chunks in the vector store -// /// -// /// The chunks to store -// /// Cancellation token -// public async Task StoreChunksAsync(IList chunks, CancellationToken cancellationToken = default) -// { -// if (!chunks.Any()) -// { -// Console.WriteLine("No chunks to store"); -// return; -// } - -// Console.WriteLine($"Storing {chunks.Count} chunks in vector store..."); - -// try -// { -// var collection = GetCollection(); - -// // Store chunks in batches to avoid overwhelming the database -// const int batchSize = 100; -// int stored = 0; - -// for (int i = 0; i < chunks.Count; i += batchSize) -// { -// var batch = chunks.Skip(i).Take(batchSize).ToList(); - -// // Upsert chunks (insert or update if exists) -// await collection.UpsertBatchAsync(batch, cancellationToken); - -// stored += batch.Count; -// Console.WriteLine($" Stored {stored}/{chunks.Count} chunks"); -// } - -// Console.WriteLine($"✅ Successfully stored {chunks.Count} chunks in vector store"); -// } -// catch (Exception ex) -// { -// Console.WriteLine($"❌ Error storing chunks: {ex.Message}"); -// throw; -// } -// } - -// /// -// /// Search for similar chunks using vector similarity -// /// -// /// The query embedding to search with -// /// Maximum number of results to return -// /// Minimum relevance score (0.0 to 1.0) -// /// Cancellation token -// /// Similar chunks ordered by relevance -// public async Task>> SearchSimilarChunksAsync( -// ReadOnlyMemory queryEmbedding, -// int limit = 10, -// double minRelevanceScore = 0.0, -// CancellationToken cancellationToken = default) -// { -// Console.WriteLine($"Searching for similar chunks (limit: {limit}, min score: {minRelevanceScore:F2})..."); - -// try -// { -// var collection = GetCollection(); - -// // Perform vector search -// var searchResults = await collection.VectorizedSearchAsync( -// queryEmbedding, -// new() -// { -// Top = limit, -// Filter = null // Could add filters for chapter, file, etc. -// }, -// cancellationToken); - -// var results = await searchResults -// .Where(r => r.Score >= minRelevanceScore) -// .ToListAsync(cancellationToken); - -// Console.WriteLine($"✅ Found {results.Count} similar chunks"); - -// return results; -// } -// catch (Exception ex) -// { -// Console.WriteLine($"❌ Error searching chunks: {ex.Message}"); -// throw; -// } -// } - -// /// -// /// Get chunks by file name -// /// -// /// The file name to search for -// /// Cancellation token -// /// Chunks from the specified file -// public async Task> GetChunksByFileAsync( -// string fileName, -// CancellationToken cancellationToken = default) -// { -// try -// { -// var collection = GetCollection(); - -// // Use VectorSearchOptions.Filter instead of VectorSearchFilter -// var options = new VectorSearchOptions -// { -// Filter = new VectorSearchFilter().EqualTo(nameof(BookContentChunk.FileName), fileName), -// Top = 1000 // Large number to get all chunks -// }; - -// var results = await collection.VectorizedSearchAsync( -// new ReadOnlyMemory(new float[1536]), // Dummy vector, we're filtering not searching -// options, -// cancellationToken); - -// var chunks = await results.Select(r => r.Record).ToListAsync(cancellationToken); - -// return chunks; -// } -// catch (Exception ex) -// { -// Console.WriteLine($"❌ Error getting chunks by file: {ex.Message}"); -// throw; -// } -// } - -// /// -// /// Get collection statistics -// /// -// /// Cancellation token -// /// Statistics about the collection -// public async Task GetStatisticsAsync(CancellationToken cancellationToken = default) -// { -// try -// { -// var collection = GetCollection(); - -// // Get all chunks to calculate statistics -// var searchResults = await collection.VectorizedSearchAsync( -// new ReadOnlyMemory(new float[1536]), // Dummy vector -// new() { Top = 10000 }, // Large number to get all -// cancellationToken); - -// var allChunks = await searchResults.Select(r => r.Record).ToListAsync(cancellationToken); - -// var fileGroups = allChunks.GroupBy(c => c.FileName).ToList(); - -// return new VectorStoreStatistics -// { -// TotalChunks = allChunks.Count, -// TotalFiles = fileGroups.Count, -// AverageChunksPerFile = fileGroups.Any() ? fileGroups.Average(g => g.Count()) : 0, -// ChunksWithEmbeddings = allChunks.Count(c => c.TextEmbedding.HasValue), -// TotalTokens = allChunks.Sum(c => c.TokenCount) -// }; -// } -// catch (Exception ex) -// { -// Console.WriteLine($"❌ Error getting statistics: {ex.Message}"); -// throw; -// } -// } - -// private VectorStoreCollection GetCollection() -// { -// return _vectorStore.GetCollection(_collectionName); -// } - -// public void Dispose() -// { -// _vectorStore?.Dispose(); -// } -//} - -///// -///// Statistics about the vector store -///// -//public record VectorStoreStatistics -//{ -// public int TotalChunks { get; init; } -// public int TotalFiles { get; init; } -// public double AverageChunksPerFile { get; init; } -// public int ChunksWithEmbeddings { get; init; } -// public int TotalTokens { get; init; } - -// public double EmbeddingCoverage => TotalChunks > 0 ? (double)ChunksWithEmbeddings / TotalChunks * 100 : 0; -//} diff --git a/EssentialCSharp.Chat/Program.cs b/EssentialCSharp.Chat/Program.cs index 5045f1f1..af3b30ec 100644 --- a/EssentialCSharp.Chat/Program.cs +++ b/EssentialCSharp.Chat/Program.cs @@ -51,7 +51,6 @@ static int Main(string[] args) buildVectorDbCommand.SetAction(async (ParseResult parseResult, CancellationToken cancellationToken) => { - // Replace with your values. IConfigurationRoot config = new ConfigurationBuilder() .SetBasePath(IntelliTect.Multitool.RepositoryPaths.GetDefaultRepoRoot()) .AddJsonFile("EssentialCSharp.Web/appsettings.json") @@ -59,22 +58,25 @@ static int Main(string[] args) .AddEnvironmentVariables() .Build(); + var builder = Kernel.CreateBuilder(); + AIOptions aiOptions = config.GetRequiredSection("AIOptions").Get() ?? throw new InvalidOperationException( - "AIOptions section is missing or not configured correctly in appsettings.json or environment variables."); + "AIOptions section is missing or not configured correctly in appsettings.json or environment variables."); + builder.Services.Configure(config.GetRequiredSection("AIOptions")); - // Register Azure OpenAI text embedding generation service and Redis vector store. + // Register Azure OpenAI text embedding generation service #pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - var builder = Kernel.CreateBuilder() - .AddAzureOpenAITextEmbeddingGeneration(aiOptions.VectorGenerationDeploymentName, aiOptions.Endpoint, aiOptions.ApiKey); + builder.AddAzureOpenAITextEmbeddingGeneration(aiOptions.VectorGenerationDeploymentName, aiOptions.Endpoint, aiOptions.ApiKey); #pragma warning restore SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + builder.Services.AddLogging(loggingBuilder => - { - loggingBuilder.AddSimpleConsole(options => { - options.TimestampFormat = "HH:mm:ss "; - options.SingleLine = true; + loggingBuilder.AddSimpleConsole(options => + { + options.TimestampFormat = "HH:mm:ss "; + options.SingleLine = true; + }); }); - }); builder.Services.AddPostgresVectorStore( aiOptions.PostgresConnectionString); @@ -99,7 +101,7 @@ static int Main(string[] args) var bookContentChunks = results.SelectMany(result => result.ToBookContentChunks()).ToList(); // Generate embeddings and upload to vector store var embeddingService = kernel.GetRequiredService(); - await embeddingService.GenerateEmbeddingsAndUpload(bookContentChunks, cancellationToken, "markdown_chunks"); + await embeddingService.GenerateBookContentEmbeddingsAndUploadToVectorStore(bookContentChunks, cancellationToken, "markdown_chunks"); Console.WriteLine($"Successfully processed {bookContentChunks.Count} chunks."); } catch (Exception ex) From f9f502d2d2493ca319459584e8fa191aa9afdedb Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Tue, 22 Jul 2025 20:40:59 -0700 Subject: [PATCH 09/22] Initial Chat AI --- Directory.Packages.props | 13 +- .../EssentialCSharp.Chat.Common.csproj | 8 +- .../Extensions/ServiceCollectionExtensions.cs | 0 .../Models/BookContentChunk.cs | 2 + .../Services/AIChatService.cs | 348 ++++++++++++++++-- .../Services/EmbeddingService.cs | 8 +- .../EssentialCSharp.Chat.csproj | 2 +- EssentialCSharp.Chat/Program.cs | 224 ++++++++++- .../Properties/launchSettings.json | 8 + .../EssentialCSharp.VectorDbBuilder.csproj | 2 +- .../AIServiceCollectionExtensions.cs | 61 +++ EssentialCSharp.Web/Program.cs | 11 +- 12 files changed, 638 insertions(+), 49 deletions(-) create mode 100644 EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs create mode 100644 EssentialCSharp.Chat/Properties/launchSettings.json create mode 100644 EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 4ab69113..31429472 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -16,7 +16,8 @@ - + + @@ -44,18 +45,16 @@ - + - - - - - + + + diff --git a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj index 8eb086c3..ae22e34a 100644 --- a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj +++ b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj @@ -10,12 +10,12 @@ - - - + + + + - all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs b/EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..e69de29b diff --git a/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs b/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs index 276c4b7f..e95625b5 100644 --- a/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs +++ b/EssentialCSharp.Chat.Shared/Models/BookContentChunk.cs @@ -46,6 +46,8 @@ public sealed class BookContentChunk /// /// Vector embedding for the chunk text - will be generated by embedding service /// Using 1536 dimensions for Azure OpenAI text-embedding-3-small-v1 + /// Use CosineSimilarity distance function since we are using text-embedding-3 (https://platform.openai.com/docs/guides/embeddings#which-distance-function-should-i-use) + /// Postgres supports only Hnsw: https://learn.microsoft.com/en-us/semantic-kernel/concepts/vector-store-connectors/out-of-the-box-connectors/postgres-connector?pivots=programming-language-csharp&WT.mc_id=8B97120A00B57354 /// [VectorStoreVector(Dimensions: 1536, DistanceFunction = DistanceFunction.CosineSimilarity, IndexKind = IndexKind.Hnsw)] public ReadOnlyMemory? TextEmbedding { get; set; } diff --git a/EssentialCSharp.Chat.Shared/Services/AIChatService.cs b/EssentialCSharp.Chat.Shared/Services/AIChatService.cs index 421b89bd..ac47a45f 100644 --- a/EssentialCSharp.Chat.Shared/Services/AIChatService.cs +++ b/EssentialCSharp.Chat.Shared/Services/AIChatService.cs @@ -1,43 +1,341 @@ -using System.ClientModel; using Azure.AI.OpenAI; using Microsoft.Extensions.Options; -using OpenAI.Chat; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using OpenAI.Responses; namespace EssentialCSharp.Chat.Common.Services; -public class AIChatService(AzureOpenAIClient aIClient, IOptions options) +/// +/// Service for handling AI chat completions using the OpenAI Responses API +/// +public class AIChatService { - private readonly AIOptions _Options = options.Value; - private ChatClient ChatClient { get; } = aIClient.GetChatClient(options.Value.ChatDeploymentName); + private readonly AIOptions _Options; + private readonly AzureOpenAIClient _AzureClient; + private readonly OpenAIResponseClient _ResponseClient; + private readonly AISearchService _SearchService; - public async Task GetChatCompletion(string prompt) + public AIChatService(IOptions options, AISearchService searchService) { - var response = await ChatClient.CompleteChatAsync([ - new SystemChatMessage(_Options.SystemPrompt), - new UserChatMessage(prompt) - ]); + _Options = options.Value; + _SearchService = searchService; - // Todo: Handle response errors and check for multiple messages? - return response.Value.Content[0].Text; + // Initialize Azure OpenAI client and get the Response Client from it + _AzureClient = new AzureOpenAIClient( + new Uri(_Options.Endpoint), + new System.ClientModel.ApiKeyCredential(_Options.ApiKey)); + + _ResponseClient = _AzureClient.GetOpenAIResponseClient(_Options.ChatDeploymentName); } - // TODO: Implement streaming chat completions - public AsyncCollectionResult GetChatCompletionStream(string prompt) + /// + /// Gets a single chat completion response with all optional features + /// + /// The user's input prompt + /// Optional system prompt to override the default + /// Previous response ID to maintain conversation context + /// Optional tools for the AI to use + /// Optional reasoning effort level for reasoning models + /// Enable vector search for contextual information + /// Cancellation token + /// The AI response text and response ID for conversation continuity + public async Task<(string response, string responseId)> GetChatCompletion( + string prompt, + string? systemPrompt = null, + string? previousResponseId = null, + IMcpClient? mcpClient = null, + IEnumerable? tools = null, + ResponseReasoningEffortLevel? reasoningEffortLevel = null, + bool enableContextualSearch = false, + CancellationToken cancellationToken = default) { - return ChatClient.CompleteChatStreamingAsync([ - new SystemChatMessage(_Options.SystemPrompt), - new UserChatMessage(prompt) - ]); + var responseOptions = await CreateResponseOptionsAsync(previousResponseId, tools, reasoningEffortLevel, mcpClient: mcpClient, cancellationToken: cancellationToken); + var enrichedPrompt = await EnrichPromptWithContext(prompt, enableContextualSearch, cancellationToken); + return await GetChatCompletionCore(enrichedPrompt, responseOptions, systemPrompt, cancellationToken); } - // TODO: Implement batch chat completions for hybrid search - // public async Task> GetBatchChatCompletionsAsync(IEnumerable prompts) - // { - //#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - // var batchClient = aIClient.GetBatchClient(); - //#pragma warning restore OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + /// + /// Gets a streaming chat completion response with all optional features + /// + /// The user's input prompt + /// Optional system prompt to override the default + /// Previous response ID to maintain conversation context + /// Optional tools for the AI to use + /// Optional reasoning effort level for reasoning models + /// Enable vector search for contextual information + /// Cancellation token + /// An async enumerable of response text chunks and final response ID + public async IAsyncEnumerable<(string text, string? responseId)> GetChatCompletionStream( + string prompt, + string? systemPrompt = null, + string? previousResponseId = null, + IMcpClient? mcpClient = null, + IEnumerable? tools = null, + ResponseReasoningEffortLevel? reasoningEffortLevel = null, + bool enableContextualSearch = false, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // Add logging to help debug the conversation state issue + System.Diagnostics.Debug.WriteLine($"GetChatCompletionStream called with previousResponseId: {previousResponseId}"); + + var responseOptions = await CreateResponseOptionsAsync(previousResponseId, tools, reasoningEffortLevel, mcpClient: mcpClient, cancellationToken: cancellationToken); + var enrichedPrompt = await EnrichPromptWithContext(prompt, enableContextualSearch, cancellationToken); + + // Construct the user input with system context if provided + var systemContext = systemPrompt ?? _Options.SystemPrompt; + + // Create the streaming response using the Responses API + var streamingUpdates = _ResponseClient.CreateResponseStreamingAsync( + [ + ResponseItem.CreateUserMessageItem(enrichedPrompt), + ResponseItem.CreateSystemMessageItem(systemContext), + ], + options: responseOptions, + cancellationToken: cancellationToken); + + await foreach (var result in ProcessStreamingUpdatesAsync(streamingUpdates, responseOptions, mcpClient, cancellationToken)) + { + yield return result; + } + } + + /// + /// Enriches the user prompt with contextual information from vector search + /// + private async Task EnrichPromptWithContext(string prompt, bool enableContextualSearch, CancellationToken cancellationToken) + { + if (!enableContextualSearch || _SearchService == null) + { + return prompt; + } + + try + { + var searchResults = await _SearchService.ExecuteVectorSearch(prompt); + var contextualInfo = new System.Text.StringBuilder(); + + contextualInfo.AppendLine("## Contextual Information"); + contextualInfo.AppendLine("The following information might be relevant to your question:"); + contextualInfo.AppendLine(); + + await foreach (var result in searchResults) + { + contextualInfo.AppendLine(System.Globalization.CultureInfo.InvariantCulture, $"**From: {result.Record.Heading}**"); + contextualInfo.AppendLine(result.Record.ChunkText); + contextualInfo.AppendLine(); + } + + contextualInfo.AppendLine("## User Question"); + contextualInfo.AppendLine(prompt); + + return contextualInfo.ToString(); + } + catch (Exception ex) + { + // Log the error but don't fail the request + System.Diagnostics.Debug.WriteLine($"Error enriching prompt with context: {ex.Message}"); + return prompt; + } + } + + /// + /// Processes streaming updates from the OpenAI Responses API, handling both regular responses and function calls + /// + private async IAsyncEnumerable<(string text, string? responseId)> ProcessStreamingUpdatesAsync( + IAsyncEnumerable streamingUpdates, + ResponseCreationOptions responseOptions, + IMcpClient? mcpClient, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + string? responseId = null; + + await foreach (var update in streamingUpdates.WithCancellation(cancellationToken)) + { + if (update is StreamingResponseCreatedUpdate created) + { + // Remember the response ID for later function calls + responseId = created.Response.Id; + } + else if (update is StreamingResponseOutputItemDoneUpdate itemDone) + { + // Check if this is a function call that needs to be executed + if (itemDone.Item is FunctionCallResponseItem functionCallItem && mcpClient != null) + { + // Execute the function call and stream its response + await foreach (var functionResult in ExecuteFunctionCallAsync(functionCallItem, responseOptions, mcpClient, cancellationToken)) + { + if (functionResult.responseId != null) + { + responseId = functionResult.responseId; + } + yield return functionResult; + } + } + } + else if (update is StreamingResponseOutputTextDeltaUpdate deltaUpdate) + { + yield return (deltaUpdate.Delta.ToString(), null); + } + else if (update is StreamingResponseCompletedUpdate completedUpdate) + { + responseId = completedUpdate.Response.Id; + yield return (string.Empty, responseId); // Signal completion with response ID + } + } + } + + /// + /// Executes a function call and streams the response + /// + private async IAsyncEnumerable<(string text, string? responseId)> ExecuteFunctionCallAsync( + FunctionCallResponseItem functionCallItem, + ResponseCreationOptions responseOptions, + IMcpClient mcpClient, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // A dictionary of arguments to pass to the tool. Each key represents a parameter name, and its associated value represents the argument value. + Dictionary arguments = new Dictionary(); + // example JsonResponse: + // "{\"question\":\"Azure OpenAI Responses API (Preview)\"}" + var jsonResponse = functionCallItem.FunctionArguments.ToString(); + var jsonArguments = System.Text.Json.JsonSerializer.Deserialize>(jsonResponse) ?? new Dictionary(); - // } + // Convert JsonElement values to their actual types + foreach (var kvp in jsonArguments) + { + if (kvp.Value is System.Text.Json.JsonElement jsonElement) + { + arguments[kvp.Key] = jsonElement.ValueKind switch + { + System.Text.Json.JsonValueKind.String => jsonElement.GetString(), + System.Text.Json.JsonValueKind.Number => jsonElement.GetDecimal(), + System.Text.Json.JsonValueKind.True => true, + System.Text.Json.JsonValueKind.False => false, + System.Text.Json.JsonValueKind.Null => null, + _ => jsonElement.ToString() + }; + } + else + { + arguments[kvp.Key] = kvp.Value; + } + } + + // Execute the function call using the MCP client + var toolResult = await mcpClient.CallToolAsync( + functionCallItem.FunctionName, + arguments: arguments, + cancellationToken: cancellationToken); + + // Create input items with both the function call and the result + // This matches the Python pattern: append both tool_call and result + var inputItems = new List + { + functionCallItem, // The original function call + new FunctionCallOutputResponseItem(functionCallItem.CallId, string.Join("", toolResult.Content.Where(x => x.Type == "text").OfType().Select(x => x.Text))) + }; + + // Stream the function call response using the same processing logic + var functionResponseStream = _ResponseClient.CreateResponseStreamingAsync( + inputItems, + responseOptions, + cancellationToken); + + await foreach (var result in ProcessStreamingUpdatesAsync(functionResponseStream, responseOptions, mcpClient, cancellationToken)) + { + yield return result; + } + } + + /// + /// Creates response options with optional features + /// + private static async Task CreateResponseOptionsAsync( + string? previousResponseId = null, + IEnumerable? tools = null, + ResponseReasoningEffortLevel? reasoningEffortLevel = null, + IMcpClient? mcpClient = null, + CancellationToken cancellationToken = default + ) + { + var options = new ResponseCreationOptions(); + + // Add conversation context if available + if (!string.IsNullOrEmpty(previousResponseId)) + { + options.PreviousResponseId = previousResponseId; + } + + // Add tools if provided + if (tools != null) + { + foreach (var tool in tools) + { + options.Tools.Add(tool); + } + } + + if (mcpClient is not null) + { + await foreach (McpClientTool tool in mcpClient.EnumerateToolsAsync(cancellationToken: cancellationToken)) + { + options.Tools.Add(ResponseTool.CreateFunctionTool(tool.Name, tool.Description, BinaryData.FromString(tool.JsonSchema.GetRawText()))); + } + } + + // Add reasoning options if specified + if (reasoningEffortLevel.HasValue) + { + options.ReasoningOptions = new ResponseReasoningOptions() + { + ReasoningEffortLevel = reasoningEffortLevel.Value + }; + } + + return options; + } + + /// + /// Core method for getting chat completions with configurable response options + /// + private async Task<(string response, string responseId)> GetChatCompletionCore( + string prompt, + ResponseCreationOptions responseOptions, + string? systemPrompt = null, + CancellationToken cancellationToken = default) + { + // Construct the user input with system context if provided + var systemContext = systemPrompt ?? _Options.SystemPrompt; + + // Create the response using the Responses API + var response = await _ResponseClient.CreateResponseAsync([ + ResponseItem.CreateUserMessageItem(prompt), + ResponseItem.CreateSystemMessageItem(systemPrompt), + ], + options: responseOptions, + cancellationToken: cancellationToken); + + // Extract the message content and response ID + string responseText = string.Empty; + string responseId = response.Value.Id; + + foreach (var outputItem in response.Value.OutputItems) + { + if (outputItem is MessageResponseItem messageItem && + messageItem.Role == MessageRole.Assistant) + { + var textContent = messageItem.Content?.FirstOrDefault()?.Text; + if (!string.IsNullOrEmpty(textContent)) + { + responseText = textContent; + break; + } + } + } + + return (responseText, responseId); + } // TODO: Look into using UserSecurityContext (https://learn.microsoft.com/en-us/azure/defender-for-cloud/gain-end-user-context-ai) } diff --git a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs index 4caaa2e1..5839c8b4 100644 --- a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs +++ b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs @@ -1,6 +1,6 @@ using EssentialCSharp.Chat.Common.Models; +using Microsoft.Extensions.AI; using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Embeddings; namespace EssentialCSharp.Chat.Common.Services; @@ -8,7 +8,7 @@ namespace EssentialCSharp.Chat.Common.Services; /// Service for generating embeddings for markdown chunks using Azure OpenAI /// #pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. -public class EmbeddingService(VectorStore vectorStore, ITextEmbeddingGenerationService textEmbeddingGenerationService) +public class EmbeddingService(VectorStore vectorStore, IEmbeddingGenerator> embeddingGenerator) { public static string CollectionName { get; } = "markdown_chunks"; @@ -20,7 +20,8 @@ public class EmbeddingService(VectorStore vectorStore, ITextEmbeddingGenerationS /// A search vector as ReadOnlyMemory<float>. public async Task> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default) { - return await textEmbeddingGenerationService.GenerateEmbeddingAsync(text, cancellationToken: cancellationToken); + var embedding = await embeddingGenerator.GenerateAsync(text, cancellationToken: cancellationToken); + return embedding.Vector; } /// @@ -33,6 +34,7 @@ public async Task GenerateBookContentEmbeddingsAndUploadToVectorStore(IEnumerabl collectionName ??= CollectionName; var collection = vectorStore.GetCollection(collectionName); + await collection.EnsureCollectionDeletedAsync(cancellationToken); await collection.EnsureCollectionExistsAsync(cancellationToken); ParallelOptions parallelOptions = new() diff --git a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj index 4c2fe3e3..54cae5f5 100644 --- a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj +++ b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj @@ -21,10 +21,10 @@ - + diff --git a/EssentialCSharp.Chat/Program.cs b/EssentialCSharp.Chat/Program.cs index af3b30ec..aadb5b40 100644 --- a/EssentialCSharp.Chat/Program.cs +++ b/EssentialCSharp.Chat/Program.cs @@ -1,10 +1,11 @@ -using System.CommandLine; +using System.CommandLine; using System.Text.Json; using EssentialCSharp.Chat.Common.Services; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; +using OpenAI.Responses; namespace EssentialCSharp.Chat; @@ -49,6 +50,14 @@ static int Main(string[] args) filePatternOption, }; + var chatCommand = new Command("chat", "Start an interactive AI chat session.") + { + new Option("--stream"), + new Option("--web-search"), + new Option("--contextual-search"), + new Option("--system-prompt") + }; + buildVectorDbCommand.SetAction(async (ParseResult parseResult, CancellationToken cancellationToken) => { IConfigurationRoot config = new ConfigurationBuilder() @@ -66,7 +75,13 @@ static int Main(string[] args) // Register Azure OpenAI text embedding generation service #pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - builder.AddAzureOpenAITextEmbeddingGeneration(aiOptions.VectorGenerationDeploymentName, aiOptions.Endpoint, aiOptions.ApiKey); + // Replace the obsolete method call with the new method + builder.AddAzureOpenAIEmbeddingGenerator(aiOptions.VectorGenerationDeploymentName, aiOptions.Endpoint, aiOptions.ApiKey); + + builder.AddAzureOpenAIChatClient( + aiOptions.ChatDeploymentName, + aiOptions.Endpoint, + aiOptions.ApiKey); #pragma warning restore SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. builder.Services.AddLogging(loggingBuilder => @@ -110,6 +125,210 @@ static int Main(string[] args) throw; } }); + + chatCommand.SetAction(async (ParseResult parseResult, CancellationToken cancellationToken) => + { + IConfigurationRoot config = new ConfigurationBuilder() + .SetBasePath(IntelliTect.Multitool.RepositoryPaths.GetDefaultRepoRoot()) + .AddJsonFile("EssentialCSharp.Web/appsettings.json") + .AddUserSecrets() + .AddEnvironmentVariables() + .Build(); + + // https://learn.microsoft.com/api/mcp + + //SseClientTransport microsoftLearnMcp = new SseClientTransport( + // new SseClientTransportOptions + // { + // Name = "Microsoft Learn MCP", + // Endpoint = new Uri("https://learn.microsoft.com/api/mcp"), + // }); + + //IMcpClient mcpClient = await McpClientFactory.CreateAsync(clientTransport: microsoftLearnMcp, cancellationToken: cancellationToken); + + var enableStreaming = parseResult.GetValue("--stream"); + var enableWebSearch = parseResult.GetValue("--web-search"); + enableWebSearch = false; + var customSystemPrompt = parseResult.GetValue("--system-prompt"); + var enableContextualSearch = parseResult.GetValue("--contextual-search"); + enableContextualSearch = true; + + + AIOptions aiOptions = config.GetRequiredSection("AIOptions").Get() ?? throw new InvalidOperationException( + "AIOptions section is missing or not configured correctly in appsettings.json or environment variables."); + + // Create service collection and register dependencies + var services = new ServiceCollection(); + services.Configure(config.GetRequiredSection("AIOptions")); + services.AddLogging(builder => builder.AddSimpleConsole(options => + { + options.TimestampFormat = "HH:mm:ss "; + options.SingleLine = true; + })); + + // Add services for contextual search if enabled + if (enableContextualSearch) + { +#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + services.AddAzureOpenAIEmbeddingGenerator(aiOptions.VectorGenerationDeploymentName, aiOptions.Endpoint, aiOptions.ApiKey); +#pragma warning restore SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + services.AddPostgresVectorStore(aiOptions.PostgresConnectionString); + services.AddSingleton(); + services.AddSingleton(); + } + + services.AddSingleton(); + + var serviceProvider = services.BuildServiceProvider(); + var aiChatService = serviceProvider.GetRequiredService(); + + Console.WriteLine("🤖 AI Chat Session Started!"); + Console.WriteLine("Features enabled:"); + Console.WriteLine($" • Streaming: {(enableStreaming ? "✅" : "❌")}"); + Console.WriteLine($" • Web Search: {(enableWebSearch ? "✅" : "❌")}"); + Console.WriteLine($" • Contextual Search: {(enableContextualSearch ? "✅" : "❌")}"); + if (!string.IsNullOrEmpty(customSystemPrompt)) + Console.WriteLine($" • Custom System Prompt: {customSystemPrompt}"); + Console.WriteLine(); + Console.WriteLine("Commands:"); + Console.WriteLine(" • 'exit' or 'quit' - End the chat session"); + Console.WriteLine(" • 'clear' - Start a new conversation context"); + Console.WriteLine(" • 'help' - Show this help message"); + Console.WriteLine(" • 'history' - Show conversation history"); + Console.WriteLine(" • Any other text - Chat with the AI"); + Console.WriteLine("====================================="); + + // Track conversation context with response IDs + string? previousResponseId = null; + var conversationHistory = new List<(string Role, string Content)>(); + + while (!cancellationToken.IsCancellationRequested) + { + Console.WriteLine(); + Console.Write("👤 You: "); + var userInput = Console.ReadLine(); + + if (string.IsNullOrWhiteSpace(userInput)) + continue; + + userInput = userInput.Trim(); + + if (userInput.Equals("exit", StringComparison.OrdinalIgnoreCase) || + userInput.Equals("quit", StringComparison.OrdinalIgnoreCase)) + { + Console.WriteLine("Goodbye! 👋"); + break; + } + + if (userInput.Equals("clear", StringComparison.OrdinalIgnoreCase)) + { + // Reset conversation context when PreviousResponseId is implemented + previousResponseId = null; + conversationHistory.Clear(); + Console.WriteLine("🧹 Conversation context cleared. Starting fresh!"); + continue; + } + + if (userInput.Equals("help", StringComparison.OrdinalIgnoreCase)) + { + Console.WriteLine(); + Console.WriteLine("Commands:"); + Console.WriteLine(" • 'exit' or 'quit' - End the chat session"); + Console.WriteLine(" • 'clear' - Start a new conversation context"); + Console.WriteLine(" • 'help' - Show this help message"); + Console.WriteLine(" • 'history' - Show conversation history"); + Console.WriteLine(" • Any other text - Chat with the AI"); + continue; + } + + if (userInput.Equals("history", StringComparison.OrdinalIgnoreCase)) + { + Console.WriteLine(); + Console.WriteLine("📜 Conversation History:"); + if (conversationHistory.Count == 0) + { + Console.WriteLine(" No conversation history yet."); + } + else + { + for (int i = 0; i < conversationHistory.Count; i++) + { + var (role, content) = conversationHistory[i]; + var emoji = role == "User" ? "👤" : "🤖"; + Console.WriteLine($" {i + 1}. {emoji} {role}: {content}"); + } + } + continue; + } + + conversationHistory.Add(("User", userInput)); + + try + { + Console.Write("🤖 AI: "); + + if (enableStreaming) + { + // Use streaming with optional tools and conversation context + var fullResponse = new System.Text.StringBuilder(); + + var tools = enableWebSearch ? new[] { ResponseTool.CreateWebSearchTool() } : null; + + await foreach (var (text, responseId) in aiChatService.GetChatCompletionStream( + prompt: userInput/*, mcpClient: mcpClient*/, previousResponseId: previousResponseId, enableContextualSearch: enableContextualSearch, systemPrompt: customSystemPrompt, cancellationToken: cancellationToken)) + { + if (!string.IsNullOrEmpty(text)) + { + Console.Write(text); + fullResponse.Append(text); + } + if (!string.IsNullOrEmpty(responseId)) + { + previousResponseId = responseId; // Update for next turn + } + } + Console.WriteLine(); + + conversationHistory.Add(("Assistant", fullResponse.ToString())); + } + else + { + // Non-streaming response with optional tools and conversation context + var tools = enableWebSearch ? new[] { ResponseTool.CreateWebSearchTool() } : null; + + var (response, responseId) = await aiChatService.GetChatCompletion( + prompt: userInput, previousResponseId: previousResponseId, enableContextualSearch: enableContextualSearch, systemPrompt: customSystemPrompt, cancellationToken: cancellationToken); + + Console.WriteLine(response); + conversationHistory.Add(("Assistant", response)); + + if (!string.IsNullOrEmpty(responseId)) + { + previousResponseId = responseId; + } + + + } + + Console.WriteLine(); + } + catch (OperationCanceledException) + { + Console.WriteLine(); + Console.WriteLine("Operation cancelled. Goodbye! 👋"); + break; + } + catch (Exception ex) + { + Console.WriteLine(); + Console.WriteLine($"❌ Error: {ex.Message}"); + if (ex.InnerException != null) + { + Console.WriteLine($" Details: {ex.InnerException.Message}"); + } + } + } + }); chunkMarkdownCommand.SetAction(async parseResult => { var directory = parseResult.GetValue(directoryOption); @@ -193,6 +412,7 @@ void WriteChunkingResult(FileChunkingResult result, TextWriter writer) }); rootCommand.Subcommands.Add(chunkMarkdownCommand); rootCommand.Subcommands.Add(buildVectorDbCommand); + rootCommand.Subcommands.Add(chatCommand); return rootCommand.Parse(args).Invoke(); } diff --git a/EssentialCSharp.Chat/Properties/launchSettings.json b/EssentialCSharp.Chat/Properties/launchSettings.json new file mode 100644 index 00000000..cc4de81e --- /dev/null +++ b/EssentialCSharp.Chat/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "EssentialCSharp.Chat": { + "commandName": "Project", + "commandLineArgs": "chat --stream --web-search" + } + } +} \ No newline at end of file diff --git a/EssentialCSharp.VectorDbBuilder/EssentialCSharp.VectorDbBuilder.csproj b/EssentialCSharp.VectorDbBuilder/EssentialCSharp.VectorDbBuilder.csproj index 116905c1..75a7b9c8 100644 --- a/EssentialCSharp.VectorDbBuilder/EssentialCSharp.VectorDbBuilder.csproj +++ b/EssentialCSharp.VectorDbBuilder/EssentialCSharp.VectorDbBuilder.csproj @@ -1,4 +1,4 @@ - + Exe diff --git a/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs b/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs new file mode 100644 index 00000000..4bafb4b2 --- /dev/null +++ b/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs @@ -0,0 +1,61 @@ +using EssentialCSharp.Chat.Common.Services; +using EssentialCSharp.Chat; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel; + +namespace EssentialCSharp.Web.Extensions; + +public static class AIServiceCollectionExtensions +{ + /// + /// Adds AI chat services to the dependency injection container + /// + public static IServiceCollection AddAIChatServices(this IServiceCollection services, IConfiguration configuration) + { + // Configure AI options from configuration + services.Configure(configuration.GetSection("AIOptions")); + + var aiOptions = configuration.GetSection("AIOptions").Get() + ?? throw new InvalidOperationException("AIOptions section is missing or not configured correctly in appsettings.json or environment variables."); + + // Validate required configuration + if (string.IsNullOrEmpty(aiOptions.Endpoint) || string.IsNullOrEmpty(aiOptions.ApiKey)) + { + throw new InvalidOperationException("Azure OpenAI Endpoint and ApiKey must be configured in AIOptions."); + } + + if (string.IsNullOrEmpty(aiOptions.PostgresConnectionString)) + { + throw new InvalidOperationException("PostgreSQL connection string must be configured in AIOptions for vector store."); + } + + +#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. + // Register Azure OpenAI services + services.AddAzureOpenAIEmbeddingGenerator( + aiOptions.VectorGenerationDeploymentName, + aiOptions.Endpoint, + aiOptions.ApiKey); + + services.AddAzureOpenAIChatClient( + aiOptions.ChatDeploymentName, + aiOptions.Endpoint, + aiOptions.ApiKey); + +#pragma warning restore SKEXP0010 + + // Add PostgreSQL vector store + services.AddPostgresVectorStore( + aiOptions.PostgresConnectionString + ); + + // Register AI services + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/EssentialCSharp.Web/Program.cs b/EssentialCSharp.Web/Program.cs index 56f97373..b0c04ba9 100644 --- a/EssentialCSharp.Web/Program.cs +++ b/EssentialCSharp.Web/Program.cs @@ -51,11 +51,11 @@ private static void Main(string[] args) if (!string.IsNullOrEmpty(appInsightsConnectionString)) { - builder.Services.AddOpenTelemetry().UseAzureMonitor( - options => - { - options.ConnectionString = appInsightsConnectionString; - }); + builder.Services.AddOpenTelemetry().UseAzureMonitor( + options => + { + options.ConnectionString = appInsightsConnectionString; + }); builder.Services.AddApplicationInsightsTelemetry(); builder.Services.AddServiceProfiler(); } @@ -209,7 +209,6 @@ private static void Main(string[] args) app.UseAuthorization(); app.UseMiddleware(); - app.MapRazorPages(); app.MapDefaultControllerRoute(); From 848cd5e472114bcd23fdd0dde55cb6a787361c6a Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Tue, 22 Jul 2025 22:49:41 -0700 Subject: [PATCH 10/22] Basic UI Chat working --- .../Services/AIChatService.cs | 25 +- .../Controllers/ChatController.cs | 151 +++++++ .../EssentialCSharp.Web.csproj | 4 + .../AIServiceCollectionExtensions.cs | 84 ++-- EssentialCSharp.Web/Program.cs | 13 + .../Views/Shared/_Layout.cshtml | 373 ++++++++++++++++++ EssentialCSharp.Web/appsettings.json | 8 +- EssentialCSharp.Web/wwwroot/js/chat-widget.js | 316 +++++++++++++++ EssentialCSharp.Web/wwwroot/js/site.js | 348 ++++++++-------- 9 files changed, 1104 insertions(+), 218 deletions(-) create mode 100644 EssentialCSharp.Web/Controllers/ChatController.cs create mode 100644 EssentialCSharp.Web/wwwroot/js/chat-widget.js diff --git a/EssentialCSharp.Chat.Shared/Services/AIChatService.cs b/EssentialCSharp.Chat.Shared/Services/AIChatService.cs index ac47a45f..b57fa132 100644 --- a/EssentialCSharp.Chat.Shared/Services/AIChatService.cs +++ b/EssentialCSharp.Chat.Shared/Services/AIChatService.cs @@ -86,11 +86,14 @@ public AIChatService(IOptions options, AISearchService searchService) var systemContext = systemPrompt ?? _Options.SystemPrompt; // Create the streaming response using the Responses API + List responseItems = [ResponseItem.CreateUserMessageItem(enrichedPrompt)]; + if (systemContext is not null) + { + responseItems.Add( + ResponseItem.CreateSystemMessageItem(systemContext)); + } var streamingUpdates = _ResponseClient.CreateResponseStreamingAsync( - [ - ResponseItem.CreateUserMessageItem(enrichedPrompt), - ResponseItem.CreateSystemMessageItem(systemContext), - ], + responseItems, options: responseOptions, cancellationToken: cancellationToken); @@ -308,11 +311,17 @@ private static async Task CreateResponseOptionsAsync( // Construct the user input with system context if provided var systemContext = systemPrompt ?? _Options.SystemPrompt; + // Create the streaming response using the Responses API + List responseItems = [ResponseItem.CreateUserMessageItem(prompt)]; + if (systemContext is not null) + { + responseItems.Add( + ResponseItem.CreateSystemMessageItem(systemContext)); + } + // Create the response using the Responses API - var response = await _ResponseClient.CreateResponseAsync([ - ResponseItem.CreateUserMessageItem(prompt), - ResponseItem.CreateSystemMessageItem(systemPrompt), - ], + var response = await _ResponseClient.CreateResponseAsync( + responseItems, options: responseOptions, cancellationToken: cancellationToken); diff --git a/EssentialCSharp.Web/Controllers/ChatController.cs b/EssentialCSharp.Web/Controllers/ChatController.cs new file mode 100644 index 00000000..cb6c0453 --- /dev/null +++ b/EssentialCSharp.Web/Controllers/ChatController.cs @@ -0,0 +1,151 @@ +using System.Text.Json; +using EssentialCSharp.Chat.Common.Services; +using Microsoft.AspNetCore.Mvc; + +namespace EssentialCSharp.Web.Controllers; + +[ApiController] +[Route("api/[controller]")] +public class ChatController : ControllerBase +{ + private readonly AIChatService _AiChatService; + private readonly ILogger _Logger; + + public ChatController(ILogger logger, AIChatService aiChatService) + { + _AiChatService = aiChatService; + _Logger = logger; + } + + [HttpPost("message")] + public async Task SendMessage([FromBody] ChatMessageRequest request, CancellationToken cancellationToken = default) + { + try + { + if (_AiChatService == null) + { + return StatusCode(503, new { error = "AI Chat service is not available. Please check the configuration and try again later." }); + } + + if (string.IsNullOrWhiteSpace(request.Message)) + { + return BadRequest(new { error = "Message cannot be empty." }); + } + + // TODO: Add user authentication check here when implementing auth + // if (!User.Identity.IsAuthenticated) + // { + // return Unauthorized(new { error = "User must be logged in to use chat." }); + // } + + // TODO: Add captcha verification here when implementing captcha + // if (!await _captchaService.VerifyAsync(request.CaptchaResponse)) + // { + // return BadRequest(new { error = "Captcha verification failed." }); + // } + + var (response, responseId) = await _AiChatService.GetChatCompletion( + prompt: request.Message, + systemPrompt: request.SystemPrompt, + previousResponseId: request.PreviousResponseId, + enableContextualSearch: request.EnableContextualSearch, + cancellationToken: cancellationToken); + + return Ok(new ChatMessageResponse + { + Response = response, + ResponseId = responseId, + Timestamp = DateTime.UtcNow + }); + } + catch (OperationCanceledException) + { + return StatusCode(499, new { error = "Request was cancelled." }); + } + catch (Exception ex) + { + _Logger.LogError(ex, "Error processing chat message: {Message}", request.Message); + return StatusCode(500, new { error = "An error occurred while processing your message. Please try again." }); + } + } + + [HttpPost("stream")] + public async Task StreamMessage([FromBody] ChatMessageRequest request, CancellationToken cancellationToken = default) + { + try + { + if (_AiChatService == null) + { + Response.StatusCode = 503; + await Response.WriteAsync(JsonSerializer.Serialize(new { error = "AI Chat service is not available. Please check the configuration and try again later." }), cancellationToken); + return; + } + + if (string.IsNullOrWhiteSpace(request.Message)) + { + Response.StatusCode = 400; + await Response.WriteAsync(JsonSerializer.Serialize(new { error = "Message cannot be empty." }), cancellationToken); + return; + } + + // TODO: Add user authentication check here when implementing auth + // TODO: Add captcha verification here when implementing captcha + + Response.ContentType = "text/event-stream"; + Response.Headers["Cache-Control"] = "no-cache"; + Response.Headers["Connection"] = "keep-alive"; + + await foreach (var (text, responseId) in _AiChatService.GetChatCompletionStream( + prompt: request.Message, + systemPrompt: request.SystemPrompt, + previousResponseId: request.PreviousResponseId, + enableContextualSearch: request.EnableContextualSearch, + cancellationToken: cancellationToken)) + { + if (!string.IsNullOrEmpty(text)) + { + var eventData = JsonSerializer.Serialize(new { type = "text", data = text }); + await Response.WriteAsync($"data: {eventData}\n\n", cancellationToken); + await Response.Body.FlushAsync(cancellationToken); + } + + if (!string.IsNullOrEmpty(responseId)) + { + var eventData = JsonSerializer.Serialize(new { type = "responseId", data = responseId }); + await Response.WriteAsync($"data: {eventData}\n\n", cancellationToken); + await Response.Body.FlushAsync(cancellationToken); + } + } + + await Response.WriteAsync("data: [DONE]\n\n", cancellationToken); + await Response.Body.FlushAsync(cancellationToken); + } + catch (OperationCanceledException) + { + Response.StatusCode = 499; + await Response.WriteAsync(JsonSerializer.Serialize(new { error = "Request was cancelled." }), cancellationToken); + } + catch (Exception ex) + { + _Logger.LogError(ex, "Error processing streaming chat message: {Message}", request.Message); + Response.StatusCode = 500; + await Response.WriteAsync(JsonSerializer.Serialize(new { error = "An error occurred while processing your message. Please try again." }), cancellationToken); + } + } +} + +public class ChatMessageRequest +{ + public string Message { get; set; } = string.Empty; + public string? SystemPrompt { get; set; } + public string? PreviousResponseId { get; set; } + public bool EnableContextualSearch { get; set; } = true; + public string? CaptchaResponse { get; set; } // For future captcha implementation +} + +public class ChatMessageResponse +{ + public string Response { get; set; } = string.Empty; + public string ResponseId { get; set; } = string.Empty; + public DateTime Timestamp { get; set; } +} diff --git a/EssentialCSharp.Web/EssentialCSharp.Web.csproj b/EssentialCSharp.Web/EssentialCSharp.Web.csproj index 4cb85e02..d3b4cb67 100644 --- a/EssentialCSharp.Web/EssentialCSharp.Web.csproj +++ b/EssentialCSharp.Web/EssentialCSharp.Web.csproj @@ -56,6 +56,10 @@ RemoveIdentityAssets + + + + diff --git a/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs b/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs index 4bafb4b2..a365b220 100644 --- a/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs +++ b/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs @@ -13,49 +13,63 @@ public static class AIServiceCollectionExtensions /// public static IServiceCollection AddAIChatServices(this IServiceCollection services, IConfiguration configuration) { - // Configure AI options from configuration - services.Configure(configuration.GetSection("AIOptions")); - - var aiOptions = configuration.GetSection("AIOptions").Get() - ?? throw new InvalidOperationException("AIOptions section is missing or not configured correctly in appsettings.json or environment variables."); - - // Validate required configuration - if (string.IsNullOrEmpty(aiOptions.Endpoint) || string.IsNullOrEmpty(aiOptions.ApiKey)) + try { - throw new InvalidOperationException("Azure OpenAI Endpoint and ApiKey must be configured in AIOptions."); - } + // Configure AI options from configuration + services.Configure(configuration.GetSection("AIOptions")); - if (string.IsNullOrEmpty(aiOptions.PostgresConnectionString)) - { - throw new InvalidOperationException("PostgreSQL connection string must be configured in AIOptions for vector store."); - } + var aiOptions = configuration.GetSection("AIOptions").Get(); + + // If AI options are missing or incomplete, log warning and skip registration + if (aiOptions == null) + { + throw new InvalidOperationException("AIOptions section is missing from configuration."); + } + // Validate required configuration + if (string.IsNullOrEmpty(aiOptions.Endpoint) || + aiOptions.Endpoint.Contains("your-azure-openai-endpoint") || + string.IsNullOrEmpty(aiOptions.ApiKey) || + aiOptions.ApiKey.Contains("your-azure-openai-api-key")) + { + throw new InvalidOperationException("Azure OpenAI Endpoint and ApiKey must be properly configured in AIOptions. Please update your configuration with valid values."); + } -#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. - // Register Azure OpenAI services - services.AddAzureOpenAIEmbeddingGenerator( - aiOptions.VectorGenerationDeploymentName, - aiOptions.Endpoint, - aiOptions.ApiKey); + if (string.IsNullOrEmpty(aiOptions.PostgresConnectionString) || + aiOptions.PostgresConnectionString.Contains("your-postgres-connection-string")) + { + throw new InvalidOperationException("PostgreSQL connection string must be properly configured in AIOptions for vector store. Please update your configuration with a valid connection string."); + } - services.AddAzureOpenAIChatClient( - aiOptions.ChatDeploymentName, - aiOptions.Endpoint, - aiOptions.ApiKey); + #pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. + // Register Azure OpenAI services + services.AddAzureOpenAIEmbeddingGenerator( + aiOptions.VectorGenerationDeploymentName, + aiOptions.Endpoint, + aiOptions.ApiKey); -#pragma warning restore SKEXP0010 + services.AddAzureOpenAIChatClient( + aiOptions.ChatDeploymentName, + aiOptions.Endpoint, + aiOptions.ApiKey); - // Add PostgreSQL vector store - services.AddPostgresVectorStore( - aiOptions.PostgresConnectionString - ); + // Add PostgreSQL vector store + services.AddPostgresVectorStore(aiOptions.PostgresConnectionString); + #pragma warning restore SKEXP0010 - // Register AI services - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); + // Register AI services + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); - return services; + return services; + } + catch (Exception) + { + // If AI services fail to register, don't register them at all + // The ChatController will handle the null service gracefully + throw; // Re-throw so Program.cs can log the specific error + } } } diff --git a/EssentialCSharp.Web/Program.cs b/EssentialCSharp.Web/Program.cs index b0c04ba9..5d403b27 100644 --- a/EssentialCSharp.Web/Program.cs +++ b/EssentialCSharp.Web/Program.cs @@ -92,6 +92,8 @@ private static void Main(string[] args) builder.Configuration .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) + .AddUserSecrets() + .AddEnvironmentVariables() .AddEnvironmentVariables(); builder.Services.ConfigureApplicationCookie(options => @@ -149,6 +151,17 @@ private static void Main(string[] args) builder.Services.AddHostedService(); builder.Services.AddScoped(); + // Add AI Chat services + try + { + builder.Services.AddAIChatServices(builder.Configuration); + logger.LogInformation("AI Chat services registered successfully."); + } + catch (Exception ex) + { + logger.LogWarning(ex, "AI Chat services could not be registered. Chat functionality will be unavailable."); + } + if (!builder.Environment.IsDevelopment()) { builder.Services.AddHttpClient(client => diff --git a/EssentialCSharp.Web/Views/Shared/_Layout.cshtml b/EssentialCSharp.Web/Views/Shared/_Layout.cshtml index 4f7d1198..fea27941 100644 --- a/EssentialCSharp.Web/Views/Shared/_Layout.cshtml +++ b/EssentialCSharp.Web/Views/Shared/_Layout.cshtml @@ -74,6 +74,9 @@ })(window, document, "clarity", "script", "g4keetzd2o"); + + + + @await RenderSectionAsync("Scripts", required: false) - @await RenderSectionAsync("HeadAppend", required: false) @@ -587,9 +263,6 @@ - diff --git a/EssentialCSharp.Web/wwwroot/css/chat-widget.css b/EssentialCSharp.Web/wwwroot/css/chat-widget.css new file mode 100644 index 00000000..31317175 --- /dev/null +++ b/EssentialCSharp.Web/wwwroot/css/chat-widget.css @@ -0,0 +1,329 @@ +/* AI Chat Widget Styles */ +.chat-widget { + position: fixed; + bottom: 20px; + right: 20px; + z-index: 1050; + font-family: 'Roboto', sans-serif; +} + +.chat-toggle { + width: 60px; + height: 60px; + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + transition: all 0.3s ease; + color: white; + font-size: 24px; + animation: chatPulse 3s infinite; +} + +.chat-toggle:hover { + transform: scale(1.05); + box-shadow: 0 6px 20px rgba(0, 0, 0, 0.2); + animation: none; +} + +@keyframes chatPulse { + 0%, 100% { + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + } + 50% { + box-shadow: 0 4px 12px rgba(102, 126, 234, 0.4), 0 0 0 10px rgba(102, 126, 234, 0.1); + } +} + +.chat-panel { + position: absolute; + bottom: 80px; + right: 0; + width: 380px; + height: 500px; + background: white; + border-radius: 12px; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1); + border: 1px solid #e1e5e9; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.chat-panel-header { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + padding: 16px; + display: flex; + align-items: center; + justify-content: space-between; +} + +.chat-panel-title { + font-weight: 600; + font-size: 16px; + display: flex; + align-items: center; +} + +.chat-panel-controls { + display: flex; + gap: 8px; +} + +.chat-btn { + background: none; + border: none; + color: white; + cursor: pointer; + padding: 4px 8px; + border-radius: 4px; + transition: background-color 0.2s; +} + +.chat-btn:hover { + background-color: rgba(255, 255, 255, 0.2); +} + +.chat-panel-body { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.chat-widget-messages { + flex: 1; + overflow-y: auto; + padding: 16px; + background: #f8f9fa; +} + +.chat-widget-input { + padding: 12px 16px; + background: white; + border-top: 1px solid #e1e5e9; +} + +.chat-widget-options { + display: flex; + align-items: center; + justify-content: space-between; +} + +.chat-message { + margin-bottom: 12px; + max-width: 85%; +} + +.chat-message.user { + margin-left: auto; +} + +.chat-message.user .message-content { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + padding: 8px 12px; + border-radius: 18px 18px 4px 18px; + word-wrap: break-word; +} + +/* Chat message formatting styles */ +.chat-message.assistant .message-content { + background: white; + color: #333; + padding: 8px 12px; + border-radius: 18px 18px 18px 4px; + border: 1px solid #e1e5e9; + word-wrap: break-word; + line-height: 1.4; +} + +.chat-message.assistant .message-content h1, +.chat-message.assistant .message-content h2, +.chat-message.assistant .message-content h3, +.chat-message.assistant .message-content h4, +.chat-message.assistant .message-content h5, +.chat-message.assistant .message-content h6 { + margin: 8px 0 4px 0; + font-weight: 600; + color: #2d3748; +} + +.chat-message.assistant .message-content h1 { font-size: 1.1em; } +.chat-message.assistant .message-content h2 { font-size: 1.05em; } +.chat-message.assistant .message-content h3 { font-size: 1em; } +.chat-message.assistant .message-content h4 { font-size: 0.95em; } +.chat-message.assistant .message-content h5 { font-size: 0.9em; } +.chat-message.assistant .message-content h6 { font-size: 0.85em; } + +.chat-message.assistant .message-content p { + margin: 4px 0; +} + +.chat-message.assistant .message-content ul, +.chat-message.assistant .message-content ol { + margin: 4px 0; + padding-left: 20px; +} + +.chat-message.assistant .message-content li { + margin: 2px 0; +} + +.chat-message.assistant .message-content pre { + background: #f6f8fa; + border: 1px solid #d1d9e0; + border-radius: 6px; + padding: 8px; + margin: 4px 0; + font-family: 'Courier New', Consolas, monospace; + font-size: 0.85em; + overflow-x: auto; + white-space: pre-wrap; +} + +.chat-message.assistant .message-content code { + background: #f6f8fa; + border: 1px solid #d1d9e0; + border-radius: 3px; + padding: 2px 4px; + font-family: 'Courier New', Consolas, monospace; + font-size: 0.9em; +} + +.chat-message.assistant .message-content pre code { + background: none; + border: none; + padding: 0; +} + +.chat-message.assistant .message-content blockquote { + border-left: 4px solid #667eea; + margin: 4px 0; + padding-left: 8px; + font-style: italic; + color: #6c757d; +} + +.chat-message.assistant .message-content strong { + font-weight: 600; +} + +.chat-message.assistant .message-content em { + font-style: italic; +} + +.chat-message.assistant .message-content a { + color: #667eea; + text-decoration: none; +} + +.chat-message.assistant .message-content a:hover { + text-decoration: underline; +} + +.chat-message.assistant .message-content table { + border-collapse: collapse; + width: 100%; + margin: 4px 0; + font-size: 0.9em; +} + +.chat-message.assistant .message-content th, +.chat-message.assistant .message-content td { + border: 1px solid #d1d9e0; + padding: 4px 8px; + text-align: left; +} + +.chat-message.assistant .message-content th { + background: #f6f8fa; + font-weight: 600; +} + +.chat-message.error .message-content { + background: #fff5f5; + color: #e53e3e; + padding: 8px 12px; + border-radius: 18px 18px 18px 4px; + border: 1px solid #fed7d7; + word-wrap: break-word; +} + +.chat-typing-indicator { + display: none; + margin-bottom: 12px; + max-width: 85%; +} + +.chat-typing-indicator .message-content { + background: white; + border: 1px solid #e1e5e9; + padding: 8px 12px; + border-radius: 18px 18px 18px 4px; + display: flex; + align-items: center; + gap: 8px; +} + +.chat-typing-dots { + display: flex; + gap: 4px; +} + +.chat-typing-dots span { + width: 6px; + height: 6px; + background-color: #6c757d; + border-radius: 50%; + animation: chatTyping 1.4s infinite; +} + +.chat-typing-dots span:nth-child(2) { + animation-delay: 0.2s; +} + +.chat-typing-dots span:nth-child(3) { + animation-delay: 0.4s; +} + +@keyframes chatTyping { + 0%, 60%, 100% { + transform: translateY(0); + opacity: 0.5; + } + 30% { + transform: translateY(-8px); + opacity: 1; + } +} + +.chat-history-restored { + background: #e3f2fd; + border: 1px solid #2196f3; + color: #1976d2; + padding: 8px 12px; + margin-bottom: 8px; + border-radius: 6px; + font-size: 0.85em; + text-align: center; +} + +/* Mobile responsiveness */ +@media (max-width: 768px) { + .chat-panel { + width: calc(100vw - 40px); + height: 400px; + bottom: 80px; + right: 20px; + left: 20px; + } + + .chat-toggle { + right: 20px; + } +} diff --git a/EssentialCSharp.Web/wwwroot/css/styles.css b/EssentialCSharp.Web/wwwroot/css/styles.css index fe346fb6..c6f2ab8d 100644 --- a/EssentialCSharp.Web/wwwroot/css/styles.css +++ b/EssentialCSharp.Web/wwwroot/css/styles.css @@ -1,5 +1,10 @@ /* Global Styles */ +/* Vue.js directive for hiding uncompiled templates */ +[v-cloak] { + display: none; +} + :root { --primary-color: rgb(30, 83, 160); --primary-color-rgb: 30, 83, 160; diff --git a/EssentialCSharp.Web/wwwroot/js/chat-widget.js b/EssentialCSharp.Web/wwwroot/js/chat-widget.js index 20feda67..f289413d 100644 --- a/EssentialCSharp.Web/wwwroot/js/chat-widget.js +++ b/EssentialCSharp.Web/wwwroot/js/chat-widget.js @@ -8,15 +8,16 @@ class ChatWidget { this.isOpen = false; this.isMinimized = false; this.previousResponseId = null; + this.conversationHistory = []; this.initializeElements(); this.bindEvents(); + this.loadChatHistory(); // Load existing chat history this.enableInput(); } initializeElements() { this.chatToggle = document.getElementById('chatToggle'); this.chatPanel = document.getElementById('chatPanel'); - this.chatClose = document.getElementById('chatClose'); this.chatMinimize = document.getElementById('chatMinimize'); this.chatInput = document.getElementById('chatWidgetInput'); this.chatSend = document.getElementById('chatWidgetSend'); @@ -26,7 +27,6 @@ class ChatWidget { bindEvents() { this.chatToggle.addEventListener('click', () => this.toggleChat()); - this.chatClose.addEventListener('click', () => this.closeChat()); this.chatMinimize.addEventListener('click', () => this.minimizeChat()); this.chatSend.addEventListener('click', () => this.sendMessage()); this.chatClear.addEventListener('click', () => this.clearChat()); @@ -38,17 +38,115 @@ class ChatWidget { } }); - // Close chat when clicking outside + // Minimize chat when clicking outside document.addEventListener('click', (e) => { if (this.isOpen && !this.chatPanel.contains(e.target) && !this.chatToggle.contains(e.target)) { - this.closeChat(); + this.minimizeChat(); } }); } + // Save chat history to session storage + saveChatHistory() { + try { + // Trim history to prevent storage overflow + this.trimConversationHistory(); + + const chatData = { + conversationHistory: this.conversationHistory, + previousResponseId: this.previousResponseId, + timestamp: new Date().toISOString(), + chatState: { + isOpen: this.isOpen, + isMinimized: this.isMinimized + } + }; + sessionStorage.setItem('chat-widget-history', JSON.stringify(chatData)); + } catch (error) { + console.warn('Failed to save chat history:', error); + } + } + + // Load chat history from session storage + loadChatHistory() { + try { + const saved = sessionStorage.getItem('chat-widget-history'); + if (saved) { + const chatData = JSON.parse(saved); + this.conversationHistory = chatData.conversationHistory || []; + this.previousResponseId = chatData.previousResponseId || null; + + console.log('Loaded chat history:', { + messageCount: this.conversationHistory.length, + previousResponseId: this.previousResponseId, + messages: this.conversationHistory.map(m => ({ role: m.role, contentLength: m.content?.length || 0 })) + }); + + // Restore chat state if it was open + if (chatData.chatState?.isOpen && !chatData.chatState?.isMinimized) { + // Delay opening to ensure DOM is ready + setTimeout(() => this.openChat(), 100); + } + + // Restore messages in the UI + this.restoreMessagesFromHistory(); + } else { + console.log('No chat history found in session storage'); + } + } catch (error) { + console.warn('Failed to load chat history:', error); + this.conversationHistory = []; + this.previousResponseId = null; + } + } + + // Restore messages in the UI from conversation history + restoreMessagesFromHistory() { + // Clear existing messages + this.chatMessages.innerHTML = ''; + + // Add welcome message if no history exists + if (this.conversationHistory.length === 0) { + this.chatMessages.innerHTML = ` +
+
+ 👋 Hello! I'm your AI assistant with access to Essential C# book content. How can I help you today? +
+
+ `; + return; + } + + // Debug logging + console.log('Restoring chat history:', this.conversationHistory.length, 'messages'); + + // Show history restoration indicator + const restoredIndicator = document.createElement('div'); + restoredIndicator.className = 'chat-history-restored'; + restoredIndicator.innerHTML = ` + Chat history restored (${this.conversationHistory.length} messages) + `; + this.chatMessages.appendChild(restoredIndicator); + + // Restore conversation history + this.conversationHistory.forEach((message, index) => { + console.log(`Restoring message ${index + 1}:`, { role: message.role, contentLength: message.content?.length || 0, content: message.content?.substring(0, 100) + '...' }); + this.addMessageToUI(message.role, message.content, false); // false = don't save to history again + }); + + this.scrollToBottom(); + + // Remove the indicator after 3 seconds + setTimeout(() => { + if (restoredIndicator.parentNode) { + restoredIndicator.remove(); + } + }, 3000); + } + toggleChat() { if (this.isOpen) { - this.closeChat(); + this.minimizeChat(); } else { this.openChat(); } @@ -60,17 +158,20 @@ class ChatWidget { this.chatPanel.style.display = 'flex'; this.chatInput.focus(); this.scrollToBottom(); + this.saveChatHistory(); // Save the chat state } closeChat() { this.isOpen = false; this.isMinimized = false; this.chatPanel.style.display = 'none'; + this.saveChatHistory(); // Save the chat state } minimizeChat() { this.isMinimized = true; this.chatPanel.style.display = 'none'; + this.saveChatHistory(); // Save the chat state } enableInput() { @@ -148,10 +249,22 @@ class ChatWidget { const data = line.slice(6); // Remove 'data: ' prefix if (data === '[DONE]') { - // Stream is complete + // Stream is complete - update the conversation history with final content + if (accumulatedText && this.conversationHistory.length > 0) { + // Find the last assistant message in history and update its content + for (let i = this.conversationHistory.length - 1; i >= 0; i--) { + if (this.conversationHistory[i].role === 'assistant' && + (!this.conversationHistory[i].content || this.conversationHistory[i].content === '')) { + this.conversationHistory[i].content = accumulatedText; + break; + } + } + } + if (currentResponseId) { this.previousResponseId = currentResponseId; } + this.saveChatHistory(); // Save the updated response ID and final content break; } @@ -185,10 +298,32 @@ class ChatWidget { } addMessage(role, content) { + const messageDiv = this.addMessageToUI(role, content, true); + + // Save to conversation history + this.conversationHistory.push({ + role: role, + content: content, + timestamp: new Date().toISOString() + }); + this.saveChatHistory(); + + return messageDiv; + } + + addMessageToUI(role, content, saveToHistory = false) { + console.log(`Adding message to UI: role=${role}, contentLength=${content?.length || 0}, saveToHistory=${saveToHistory}`); + const messageDiv = document.createElement('div'); messageDiv.className = `chat-message ${role}`; const formattedContent = role === 'assistant' ? this.formatMessage(content) : this.escapeHtml(content); + + if (role === 'assistant') { + console.log('Assistant message - original content:', content?.substring(0, 100) + '...'); + console.log('Assistant message - formatted content:', formattedContent?.substring(0, 100) + '...'); + } + messageDiv.innerHTML = `
${formattedContent}
`; @@ -234,7 +369,23 @@ class ChatWidget { `; + + // Clear history and storage + this.conversationHistory = []; this.previousResponseId = null; + try { + sessionStorage.removeItem('chat-widget-history'); + } catch (error) { + console.warn('Failed to clear chat history from storage:', error); + } + } + + // Limit conversation history to prevent storage overflow + trimConversationHistory() { + const maxMessages = 50; // Keep last 50 messages + if (this.conversationHistory.length > maxMessages) { + this.conversationHistory = this.conversationHistory.slice(-maxMessages); + } } scrollToBottom() { From 4005cea8232b62db7eed54e35842e9e87602d67f Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Wed, 23 Jul 2025 01:22:56 -0700 Subject: [PATCH 12/22] Vuetify --- .../Views/Shared/_Layout.cshtml | 160 ++++- .../wwwroot/css/chat-widget.css | 583 +++++++++++------- EssentialCSharp.Web/wwwroot/js/chat-module.js | 178 ++++++ EssentialCSharp.Web/wwwroot/js/chat.js | 213 +++++++ EssentialCSharp.Web/wwwroot/js/site.js | 9 +- 5 files changed, 902 insertions(+), 241 deletions(-) create mode 100644 EssentialCSharp.Web/wwwroot/js/chat-module.js create mode 100644 EssentialCSharp.Web/wwwroot/js/chat.js diff --git a/EssentialCSharp.Web/Views/Shared/_Layout.cshtml b/EssentialCSharp.Web/Views/Shared/_Layout.cshtml index f2caa488..75417692 100644 --- a/EssentialCSharp.Web/Views/Shared/_Layout.cshtml +++ b/EssentialCSharp.Web/Views/Shared/_Layout.cshtml @@ -47,6 +47,10 @@ + + + + @*Font Family*@ @@ -248,45 +252,140 @@ -
-
- -
+
+ + - - - -
- -
@@ -440,8 +454,6 @@ } - @if (!Context.Request.Path.StartsWithSegments("/Identity")) - {
@@ -484,8 +497,6 @@
- } -
diff --git a/EssentialCSharp.Web/wwwroot/css/chat-widget.css b/EssentialCSharp.Web/wwwroot/css/chat-widget.css index c320441c..b3b1e26c 100644 --- a/EssentialCSharp.Web/wwwroot/css/chat-widget.css +++ b/EssentialCSharp.Web/wwwroot/css/chat-widget.css @@ -188,9 +188,9 @@ background: white; border-radius: 16px; width: 90vw; - max-width: 500px; + max-width: 800px; height: 80vh; - max-height: 600px; + max-height: 700px; display: flex; flex-direction: column; box-shadow: @@ -203,6 +203,16 @@ border: 1px solid rgba(255, 255, 255, 0.2); } +/* Desktop-specific sizing for better space utilization */ +@media (min-width: 769px) { + .chat-card { + width: 80vw; + max-width: 900px; + height: 85vh; + max-height: 800px; + } +} + @keyframes slideUpEnhanced { 0% { opacity: 0; @@ -278,6 +288,13 @@ outline-offset: 2px; } +/* Header actions container */ +.chat-header-actions { + display: flex; + align-items: center; + gap: 8px; +} + /* Messages area */ .chat-messages { flex: 1; @@ -672,52 +689,6 @@ transform: none; } -/* Chat actions */ -.chat-actions { - display: flex; - align-items: center; - justify-content: space-between; -} - -.action-button { - background: rgba(25, 118, 210, 0.04); - border: 1px solid rgba(25, 118, 210, 0.12); - color: #1976d2; - cursor: pointer; - display: flex; - align-items: center; - gap: 6px; - padding: 10px 16px; - border-radius: 20px; - font-size: 12px; - font-weight: 500; - text-transform: uppercase; - letter-spacing: 0.5px; - transition: all 200ms cubic-bezier(0.4, 0, 0.2, 1); -} - -.action-button:hover:not(:disabled) { - background: rgba(25, 118, 210, 0.08); - border-color: rgba(25, 118, 210, 0.2); - transform: translateY(-1px); - box-shadow: 0 2px 4px rgba(25, 118, 210, 0.2); -} - -.action-button:focus { - outline: 3px solid rgba(25, 118, 210, 0.3); - outline-offset: 1px; -} - -.action-button:disabled { - color: rgba(0, 0, 0, 0.26); - cursor: not-allowed; -} - -.message-count { - font-size: 12px; - color: rgba(0, 0, 0, 0.6); -} - /* Elevation classes for consistency with Vuetify */ .elevation-6 { box-shadow: 0 3px 5px -1px rgba(0,0,0,.2), 0 6px 10px 0 rgba(0,0,0,.14), 0 1px 18px 0 rgba(0,0,0,.12) !important; @@ -747,8 +718,7 @@ .chat-button, .chat-overlay, .chat-card, - .send-button, - .action-button { + .send-button { animation: none; transition: none; } diff --git a/EssentialCSharp.Web/wwwroot/css/styles.css b/EssentialCSharp.Web/wwwroot/css/styles.css index bcb57f96..dd38f9b7 100644 --- a/EssentialCSharp.Web/wwwroot/css/styles.css +++ b/EssentialCSharp.Web/wwwroot/css/styles.css @@ -890,4 +890,142 @@ button.accept-policy { opacity: 0; } -/*end transitions*/ \ No newline at end of file +/*end transitions*/ + +/* Captcha Modal Styles */ +.captcha-modal-overlay { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 100%; + background: rgba(0, 0, 0, 0.6); + z-index: 2000; /* Higher than chat widget */ + display: flex; + align-items: center; + justify-content: center; + -webkit-backdrop-filter: blur(3px); + backdrop-filter: blur(3px); + animation: fadeIn 0.3s ease-out; +} + +.captcha-modal-card { + background: white; + border-radius: 16px; + box-shadow: 0 24px 48px rgba(0, 0, 0, 0.2); + max-width: 480px; + width: 90%; + max-height: 90vh; + overflow-y: auto; + animation: slideInScale 0.4s ease-out; +} + +.captcha-modal-header { + padding: 24px 24px 16px; + border-bottom: 1px solid rgba(0, 0, 0, 0.12); + text-align: center; +} + +.captcha-modal-title { + margin: 0; + font-size: 20px; + font-weight: 500; + color: #1976d2; + display: flex; + align-items: center; + justify-content: center; +} + +.captcha-modal-content { + padding: 24px; + text-align: center; +} + +.captcha-explanation { + margin: 0 0 24px; + font-size: 16px; + line-height: 1.6; + color: rgba(0, 0, 0, 0.8); +} + +.captcha-widget-container { + display: flex; + justify-content: center; + margin: 24px 0; + min-height: 78px; /* hCaptcha widget height */ +} + +.captcha-help-text { + margin: 24px 0 0; + font-size: 14px; + color: rgba(0, 0, 0, 0.6); + font-style: italic; +} + +/* Modal animations */ +@keyframes slideInScale { + 0% { + opacity: 0; + transform: translate(-50%, -60%) scale(0.8); + } + 100% { + opacity: 1; + transform: translate(0, 0) scale(1); + } +} + +/* Responsive adjustments for mobile */ +@media (max-width: 480px) { + .captcha-modal-card { + width: 95%; + margin: 16px; + border-radius: 12px; + } + + .captcha-modal-header { + padding: 20px 16px 12px; + } + + .captcha-modal-content { + padding: 20px 16px; + } + + .captcha-modal-title { + font-size: 18px; + } + + .captcha-explanation { + font-size: 15px; + } +} + +/* High contrast mode support */ +@media (prefers-contrast: high) { + .captcha-modal-overlay { + background: rgba(0, 0, 0, 0.8); + } + + .captcha-modal-card { + border: 2px solid #000; + } + + .captcha-modal-header { + border-bottom-color: #000; + } +} + +/* Reduced motion support */ +@media (prefers-reduced-motion: reduce) { + .captcha-modal-overlay, + .captcha-modal-card { + animation: none; + } + + .captcha-modal-overlay { + opacity: 1; + } + + .captcha-modal-card { + transform: none; + } +} diff --git a/EssentialCSharp.Web/wwwroot/js/chat-module.js b/EssentialCSharp.Web/wwwroot/js/chat-module.js index 386305b2..b66c44f9 100644 --- a/EssentialCSharp.Web/wwwroot/js/chat-module.js +++ b/EssentialCSharp.Web/wwwroot/js/chat-module.js @@ -1,5 +1,5 @@ // Chat Module - Vue.js composable for AI chat functionality -import { ref, nextTick, watch } from 'vue'; +import { ref, nextTick, watch, onMounted, onUnmounted } from 'vue'; export function useChatWidget() { // Authentication state @@ -13,11 +13,10 @@ export function useChatWidget() { const chatMessagesEl = ref(null); const chatInputField = ref(null); const lastResponseId = ref(null); - // Captcha and throttling state + + // Captcha state (currently not used but referenced in template) const showCaptcha = ref(false); const captchaSiteKey = ref(window.HCAPTCHA_SITE_KEY || ''); - const captchaResponse = ref(''); - const throttlingStatus = ref({ RequiresCaptcha: false, CurrentUsageCount: 0, ThresholdForCaptcha: 10 }); // Load chat history from localStorage on initialization function loadChatHistory() { @@ -76,7 +75,7 @@ export function useChatWidget() { watch(isAuthenticated, (newAuth, oldAuth) => { if (oldAuth === true && newAuth === false) { // User logged out, clear chat - clearChat(); + clearChatHistory(); } }); @@ -96,81 +95,23 @@ export function useChatWidget() { function closeChatDialog() { showChatDialog.value = false; - // Reset captcha when closing - showCaptcha.value = false; - captchaResponse.value = ''; - if (window.hcaptcha && document.getElementById('hcaptcha-modal')) { - window.hcaptcha.reset('hcaptcha-modal'); - } } - function clearChat() { + function clearChatHistory() { chatMessages.value = []; lastResponseId.value = null; saveChatHistory(); - // Reset captcha state - showCaptcha.value = false; - captchaResponse.value = ''; - if (window.hcaptcha && document.getElementById('hcaptcha-modal')) { - window.hcaptcha.reset('hcaptcha-modal'); - } - // Refresh throttling status - if (isAuthenticated.value) { - checkThrottlingStatus(); - } - } - - // Check throttling status from server - async function checkThrottlingStatus() { - if (!isAuthenticated.value) return; - try { - const response = await fetch('/api/chat/throttling-status', { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - } - }); - - if (response.ok) { - throttlingStatus.value = await response.json(); - showCaptcha.value = throttlingStatus.value.RequiresCaptcha; - - if (showCaptcha.value) { - // Initialize hCaptcha when needed - nextTick(() => { - if (window.hcaptcha && document.getElementById('hcaptcha-modal')) { - window.hcaptcha.render('hcaptcha-modal', { - sitekey: captchaSiteKey.value, - callback: 'onCaptchaSuccess', - 'expired-callback': 'onCaptchaExpired', - 'error-callback': 'onCaptchaError' - }); - } - }); - } + // Force a scroll to top to make it obvious the messages are gone + nextTick(() => { + if (chatMessagesEl.value) { + chatMessagesEl.value.scrollTop = 0; } - } catch (error) { - console.warn('Failed to check throttling status:', error); - } + }); } - // Captcha callback functions (need to be global) - window.onCaptchaSuccess = function(token) { - captchaResponse.value = token; - showCaptcha.value = false; - }; - - window.onCaptchaExpired = function() { - captchaResponse.value = ''; - showCaptcha.value = true; - }; - - window.onCaptchaError = function(error) { - console.error('hCaptcha error:', error); - captchaResponse.value = ''; - showCaptcha.value = true; - }; + // Remove captcha callback functions as they're no longer needed for chat + // The captcha service can still be used elsewhere in the application function scrollToBottom() { if (chatMessagesEl.value) { @@ -198,8 +139,6 @@ export function useChatWidget() { return 'rate-limit-error'; } else if (errorType === 'auth-error') { return 'error-message'; - } else if (errorType === 'captcha-error') { - return 'error-message'; } else if (errorType === 'validation-error') { return 'error-message'; } else { @@ -212,8 +151,6 @@ export function useChatWidget() { return 'fas fa-clock'; } else if (errorType === 'auth-error') { return 'fas fa-lock'; - } else if (errorType === 'captcha-error') { - return 'fas fa-shield-alt'; } else if (errorType === 'validation-error') { return 'fas fa-exclamation-circle'; } else if (errorType === 'network-error') { @@ -284,11 +221,6 @@ export function useChatWidget() { previousResponseId: lastResponseId.value }; - // Include captcha response if available - if (captchaResponse.value) { - requestBody.captchaResponse = captchaResponse.value; - } - const response = await fetch('/api/chat/stream', { method: 'POST', headers: { @@ -301,72 +233,29 @@ export function useChatWidget() { if (response.status === 401) { throw new Error('Authentication required'); } else if (response.status === 429) { - // Handle rate limiting + // Handle rate limiting - simple error message without captcha let errorData; try { errorData = await response.json(); } catch (e) { errorData = { error: 'Rate limit exceeded. Please wait before sending another message.', - retryAfter: 60, - requiresCaptcha: true + retryAfter: 60 }; } - // Show captcha if required (rate limiting triggers captcha requirement) - if (errorData.requiresCaptcha) { - showCaptcha.value = true; - captchaResponse.value = ''; - - // Initialize hCaptcha - nextTick(() => { - if (window.hcaptcha && document.getElementById('hcaptcha-modal')) { - window.hcaptcha.render('hcaptcha-modal', { - sitekey: captchaSiteKey.value, - callback: 'onCaptchaSuccess', - 'expired-callback': 'onCaptchaExpired', - 'error-callback': 'onCaptchaError' - }); - } - }); - } - const retryAfter = errorData.retryAfter || 60; - const errorMessage = errorData.requiresCaptcha - ? `Rate limit exceeded. Please complete the captcha below and wait ${Math.ceil(retryAfter)} seconds before sending another message.` - : `Rate limit exceeded. Please wait ${Math.ceil(retryAfter)} seconds before sending another message.`; + const errorMessage = `Rate limit exceeded. Please wait ${Math.ceil(retryAfter)} seconds before sending another message.`; throw new Error(errorMessage); } else if (response.status === 400) { - // Handle captcha requirement + // Handle validation errors const errorData = await response.json(); - if (errorData.requiresCaptcha) { - showCaptcha.value = true; - captchaResponse.value = ''; - - // Initialize hCaptcha - nextTick(() => { - if (window.hcaptcha && document.getElementById('hcaptcha-modal')) { - window.hcaptcha.render('hcaptcha-modal', { - sitekey: captchaSiteKey.value, - callback: 'onCaptchaSuccess', - 'expired-callback': 'onCaptchaExpired', - 'error-callback': 'onCaptchaError' - }); - } - }); - - throw new Error(errorData.error || 'Captcha verification required'); - } throw new Error(errorData.error || 'Bad request'); } throw new Error(`HTTP error! status: ${response.status}`); } - // Reset captcha after successful request - captchaResponse.value = ''; - showCaptcha.value = false; - // Handle streaming response reader = response.body.getReader(); const decoder = new TextDecoder(); @@ -447,9 +336,6 @@ export function useChatWidget() { } else if (error.message?.includes('Rate limit exceeded')) { errorMessage = error.message; // Use the specific rate limit message with timing errorType = 'rate-limit'; - } else if (error.message?.includes('Captcha verification')) { - errorMessage = error.message; // Use the captcha-specific message - errorType = 'captcha-error'; } else if (error.message?.includes('HTTP error')) { errorMessage = 'Unable to connect to the chat service. Please check your connection and try again.'; errorType = 'connection-error'; @@ -518,18 +404,18 @@ export function useChatWidget() { isTyping, chatMessagesEl, chatInputField, + + // Captcha state showCaptcha, captchaSiteKey, - throttlingStatus, // Methods openChatDialog, closeChatDialog, - clearChat, + clearChatHistory, formatMessage, getErrorMessageClass, getErrorIconClass, - sendChatMessage, - checkThrottlingStatus + sendChatMessage }; } diff --git a/EssentialCSharp.Web/wwwroot/js/site.js b/EssentialCSharp.Web/wwwroot/js/site.js index 5f9ee835..c585ba21 100644 --- a/EssentialCSharp.Web/wwwroot/js/site.js +++ b/EssentialCSharp.Web/wwwroot/js/site.js @@ -7,6 +7,7 @@ import { watch, computed, } from "vue"; +import { createVuetify } from "vuetify"; import { useWindowSize } from "vue-window-size"; import { useChatWidget } from "./chat-module.js"; @@ -380,4 +381,14 @@ app.component("toc-tree", { template: "#toc-tree", }); +// Create and configure Vuetify +const vuetify = createVuetify({ + theme: { + defaultTheme: 'light' + } +}); + +// Use Vuetify with the Vue app +app.use(vuetify); + app.mount("#app"); From e44cb70368bef97889d411ba8a2d8877e52ceaea Mon Sep 17 00:00:00 2001 From: Benjamin Michaelis Date: Thu, 24 Jul 2025 00:07:08 -0700 Subject: [PATCH 18/22] Cleanup :) --- Directory.Packages.props | 25 +- .../EssentialCSharp.Chat.Common.csproj | 5 - .../Extensions/ServiceCollectionExtensions.cs | 80 + .../Models/AIOptions.cs | 1 + .../Services/AIChatService.cs | 12 +- .../Services/AISearchService.cs | 21 - .../Services/EmbeddingService.cs | 2 - .../Services/MarkdownChunkingService.cs | 8 +- .../EssentialCSharp.Chat.csproj | 5 - EssentialCSharp.Chat/Program.cs | 48 +- EssentialCSharp.Web.sln | 6 - .../AIServiceCollectionExtensions.cs | 75 - EssentialCSharp.Web/Program.cs | 14 +- EssentialCSharp.Web/appsettings.json | 4 +- .../wwwroot/css/chat-widget.css | 24 +- EssentialCSharp.Web/wwwroot/js/site.js | 29 +- EssentialCSharp.Web/wwwroot/lib/vue/LICENSE | 21 - EssentialCSharp.Web/wwwroot/lib/vue/README.md | 54 - .../wwwroot/lib/vue/compiler-sfc/index.d.ts | 1 - .../wwwroot/lib/vue/compiler-sfc/index.js | 1 - .../wwwroot/lib/vue/compiler-sfc/index.mjs | 1 - .../wwwroot/lib/vue/compiler-sfc/package.json | 5 - .../wwwroot/lib/vue/dist/vue.cjs.js | 81 - .../wwwroot/lib/vue/dist/vue.cjs.prod.js | 69 - .../wwwroot/lib/vue/dist/vue.d.ts | 9 - .../wwwroot/lib/vue/dist/vue.esm-browser.js | 16095 ---------------- .../lib/vue/dist/vue.esm-browser.prod.js | 1 - .../wwwroot/lib/vue/dist/vue.esm-bundler.js | 71 - .../wwwroot/lib/vue/dist/vue.global.js | 16081 --------------- .../wwwroot/lib/vue/dist/vue.global.prod.js | 1 - .../lib/vue/dist/vue.runtime.esm-browser.js | 10966 ----------- .../vue/dist/vue.runtime.esm-browser.prod.js | 1 - .../lib/vue/dist/vue.runtime.esm-bundler.js | 22 - .../lib/vue/dist/vue.runtime.global.js | 11100 ----------- .../lib/vue/dist/vue.runtime.global.prod.js | 1 - EssentialCSharp.Web/wwwroot/lib/vue/index.js | 7 - EssentialCSharp.Web/wwwroot/lib/vue/index.mjs | 1 - .../wwwroot/lib/vue/macros-global.d.ts | 19 - .../wwwroot/lib/vue/macros.d.ts | 112 - .../wwwroot/lib/vue/package.json | 77 - .../wwwroot/lib/vue/ref-macros.d.ts | 2 - .../lib/vue/server-renderer/index.d.ts | 1 - .../wwwroot/lib/vue/server-renderer/index.js | 1 - .../wwwroot/lib/vue/server-renderer/index.mjs | 1 - .../lib/vue/server-renderer/package.json | 5 - 45 files changed, 128 insertions(+), 55038 deletions(-) delete mode 100644 EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/LICENSE delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/README.md delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/compiler-sfc/index.d.ts delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/compiler-sfc/index.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/compiler-sfc/index.mjs delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/compiler-sfc/package.json delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.cjs.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.cjs.prod.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.d.ts delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.esm-browser.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.esm-browser.prod.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.esm-bundler.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.global.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.global.prod.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.runtime.esm-browser.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.runtime.esm-browser.prod.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.runtime.esm-bundler.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.runtime.global.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/dist/vue.runtime.global.prod.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/index.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/index.mjs delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/macros-global.d.ts delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/macros.d.ts delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/package.json delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/ref-macros.d.ts delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/server-renderer/index.d.ts delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/server-renderer/index.js delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/server-renderer/index.mjs delete mode 100644 EssentialCSharp.Web/wwwroot/lib/vue/server-renderer/package.json diff --git a/Directory.Packages.props b/Directory.Packages.props index 31429472..655e2411 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,7 +3,7 @@ true false 1.1.1.5540 - true + false https://api.nuget.org/v3/index.json; @@ -14,11 +14,6 @@ - - - - - @@ -41,22 +36,14 @@ + + - - - - - - - - - - - - - + + + diff --git a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj index ae22e34a..801fac80 100644 --- a/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj +++ b/EssentialCSharp.Chat.Shared/EssentialCSharp.Chat.Common.csproj @@ -7,15 +7,10 @@ - - - - - all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs b/EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs index e69de29b..d6eace31 100644 --- a/EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs +++ b/EssentialCSharp.Chat.Shared/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,80 @@ +using EssentialCSharp.Chat.Common.Services; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.SemanticKernel; + +namespace EssentialCSharp.Chat.Common.Extensions; + +public static class ServiceCollectionExtensions +{ + /// + /// Adds Azure OpenAI and related AI services to the service collection + /// + /// The service collection to add services to + /// The AI configuration options + /// The service collection for chaining + public static IServiceCollection AddAzureOpenAIServices(this IServiceCollection services, AIOptions aiOptions) + { + // Validate required configuration + if (aiOptions == null) + { + throw new InvalidOperationException("AIOptions cannot be null."); + } + + if (string.IsNullOrEmpty(aiOptions.Endpoint) || + string.IsNullOrEmpty(aiOptions.ApiKey)) + { + throw new InvalidOperationException("Azure OpenAI Endpoint and ApiKey must be properly configured in AIOptions. Please update your configuration with valid values."); + } + + if (string.IsNullOrEmpty(aiOptions.PostgresConnectionString) || + aiOptions.PostgresConnectionString.Contains("your-postgres-connection-string")) + { + throw new InvalidOperationException("PostgreSQL connection string must be properly configured in AIOptions for vector store. Please update your configuration with a valid connection string."); + } + +#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. + + // Register Azure OpenAI services + services.AddAzureOpenAIEmbeddingGenerator( + aiOptions.VectorGenerationDeploymentName, + aiOptions.Endpoint, + aiOptions.ApiKey); + + services.AddAzureOpenAIChatClient( + aiOptions.ChatDeploymentName, + aiOptions.Endpoint, + aiOptions.ApiKey); + + // Add PostgreSQL vector store + services.AddPostgresVectorStore(aiOptions.PostgresConnectionString); + +#pragma warning restore SKEXP0010 + + // Register shared AI services + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } + + /// + /// Adds Azure OpenAI and related AI services to the service collection using configuration + /// + /// The service collection to add services to + /// The configuration to read AIOptions from + /// The service collection for chaining + public static IServiceCollection AddAzureOpenAIServices(this IServiceCollection services, IConfiguration configuration) + { + // Configure AI options from configuration + services.Configure(configuration.GetSection("AIOptions")); + + var aiOptions = configuration.GetSection("AIOptions").Get(); + + return aiOptions == null + ? throw new InvalidOperationException("AIOptions section is missing from configuration.") + : services.AddAzureOpenAIServices(aiOptions); + } +} diff --git a/EssentialCSharp.Chat.Shared/Models/AIOptions.cs b/EssentialCSharp.Chat.Shared/Models/AIOptions.cs index 202d50f2..ab157f30 100644 --- a/EssentialCSharp.Chat.Shared/Models/AIOptions.cs +++ b/EssentialCSharp.Chat.Shared/Models/AIOptions.cs @@ -21,6 +21,7 @@ public class AIOptions /// The Azure OpenAI endpoint URL. /// public string Endpoint { get; set; } = string.Empty; + /// /// The API key for accessing Azure OpenAI services. /// diff --git a/EssentialCSharp.Chat.Shared/Services/AIChatService.cs b/EssentialCSharp.Chat.Shared/Services/AIChatService.cs index b57fa132..01164240 100644 --- a/EssentialCSharp.Chat.Shared/Services/AIChatService.cs +++ b/EssentialCSharp.Chat.Shared/Services/AIChatService.cs @@ -77,7 +77,7 @@ public AIChatService(IOptions options, AISearchService searchService) [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) { // Add logging to help debug the conversation state issue - System.Diagnostics.Debug.WriteLine($"GetChatCompletionStream called with previousResponseId: {previousResponseId}"); + Debug.WriteLine($"GetChatCompletionStream called with previousResponseId: {previousResponseId}"); var responseOptions = await CreateResponseOptionsAsync(previousResponseId, tools, reasoningEffortLevel, mcpClient: mcpClient, cancellationToken: cancellationToken); var enrichedPrompt = await EnrichPromptWithContext(prompt, enableContextualSearch, cancellationToken); @@ -137,7 +137,7 @@ private async Task EnrichPromptWithContext(string prompt, bool enableCon catch (Exception ex) { // Log the error but don't fail the request - System.Diagnostics.Debug.WriteLine($"Error enriching prompt with context: {ex.Message}"); + Debug.WriteLine($"Error enriching prompt with context: {ex.Message}"); return prompt; } } @@ -151,10 +151,9 @@ private async Task EnrichPromptWithContext(string prompt, bool enableCon IMcpClient? mcpClient, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) { - string? responseId = null; - await foreach (var update in streamingUpdates.WithCancellation(cancellationToken)) { + string? responseId; if (update is StreamingResponseCreatedUpdate created) { // Remember the response ID for later function calls @@ -182,8 +181,7 @@ private async Task EnrichPromptWithContext(string prompt, bool enableCon } else if (update is StreamingResponseCompletedUpdate completedUpdate) { - responseId = completedUpdate.Response.Id; - yield return (string.Empty, responseId); // Signal completion with response ID + yield return (string.Empty, responseId: completedUpdate.Response.Id); // Signal completion with response ID } } } @@ -198,7 +196,7 @@ private async Task EnrichPromptWithContext(string prompt, bool enableCon [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) { // A dictionary of arguments to pass to the tool. Each key represents a parameter name, and its associated value represents the argument value. - Dictionary arguments = new Dictionary(); + Dictionary arguments = []; // example JsonResponse: // "{\"question\":\"Azure OpenAI Responses API (Preview)\"}" var jsonResponse = functionCallItem.FunctionArguments.ToString(); diff --git a/EssentialCSharp.Chat.Shared/Services/AISearchService.cs b/EssentialCSharp.Chat.Shared/Services/AISearchService.cs index fb8eea61..915d1dc4 100644 --- a/EssentialCSharp.Chat.Shared/Services/AISearchService.cs +++ b/EssentialCSharp.Chat.Shared/Services/AISearchService.cs @@ -3,30 +3,9 @@ namespace EssentialCSharp.Chat.Common.Services; -#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. public class AISearchService(VectorStore vectorStore, EmbeddingService embeddingService) -#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. { // TODO: Implement Hybrid Search functionality, may need to switch db providers to support full text search? - //public async Task ExecuteHybridSearch(string query, string? collectionName = null) - //{ - // collectionName ??= EmbeddingService.CollectionName; - - // IKeywordHybridSearchable collection = (IKeywordHybridSearchable)vectorStore.GetCollection(collectionName); - - // ReadOnlyMemory searchVector = await embeddingService.GenerateEmbeddingAsync(query); - - // var hybridSearchOptions = new HybridSearchOptions - // { - - // }; - - // var searchResults = await collection.HybridSearchAsync (searchVector, ["C#"], top: 3); - // foreach (var result in results) - // { - // Console.WriteLine($"Found chunk: {result.Value.Heading} in file {result.Value.FileName}"); - // } - //} public async Task>> ExecuteVectorSearch(string query, string? collectionName = null) { diff --git a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs index 5839c8b4..b315f091 100644 --- a/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs +++ b/EssentialCSharp.Chat.Shared/Services/EmbeddingService.cs @@ -7,7 +7,6 @@ namespace EssentialCSharp.Chat.Common.Services; /// /// Service for generating embeddings for markdown chunks using Azure OpenAI /// -#pragma warning disable SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. public class EmbeddingService(VectorStore vectorStore, IEmbeddingGenerator> embeddingGenerator) { public static string CollectionName { get; } = "markdown_chunks"; @@ -54,4 +53,3 @@ await Parallel.ForEachAsync(bookContents, parallelOptions, async (chunk, cancell Console.WriteLine($"Successfully generated embeddings and uploaded {bookContents.Count()} chunks to collection '{collectionName}'."); } } -#pragma warning restore SKEXP0001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. diff --git a/EssentialCSharp.Chat.Shared/Services/MarkdownChunkingService.cs b/EssentialCSharp.Chat.Shared/Services/MarkdownChunkingService.cs index ee66e7ef..d50ee214 100644 --- a/EssentialCSharp.Chat.Shared/Services/MarkdownChunkingService.cs +++ b/EssentialCSharp.Chat.Shared/Services/MarkdownChunkingService.cs @@ -12,7 +12,7 @@ public partial class MarkdownChunkingService( int maxTokensPerChunk = 256, int overlapTokens = 25) { - private static readonly string[] _NewLineSeparators = new[] { "\r\n", "\n", "\r" }; + private static readonly string[] _NewLineSeparators = ["\r\n", "\n", "\r"]; private readonly int _MaxTokensPerChunk = maxTokensPerChunk; private readonly int _OverlapTokens = overlapTokens; @@ -73,14 +73,14 @@ public FileChunkingResult ProcessSingleMarkdownFile( int totalChunkCharacters = 0; int chunkCount = 0; - foreach (var section in sections) + foreach (var (Header, Content) in sections) { #pragma warning disable SKEXP0050 var chunks = TextChunker.SplitMarkdownParagraphs( - lines: section.Content, + lines: Content, maxTokensPerParagraph: _MaxTokensPerChunk, overlapTokens: _OverlapTokens, - chunkHeader: section.Header + " - " + chunkHeader: Header + " - " ); #pragma warning restore SKEXP0050 allChunks.AddRange(chunks); diff --git a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj index 54cae5f5..1f027998 100644 --- a/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj +++ b/EssentialCSharp.Chat/EssentialCSharp.Chat.csproj @@ -18,14 +18,9 @@ - - - - - diff --git a/EssentialCSharp.Chat/Program.cs b/EssentialCSharp.Chat/Program.cs index aadb5b40..2afa1d3f 100644 --- a/EssentialCSharp.Chat/Program.cs +++ b/EssentialCSharp.Chat/Program.cs @@ -1,11 +1,11 @@ using System.CommandLine; using System.Text.Json; +using EssentialCSharp.Chat.Common.Extensions; using EssentialCSharp.Chat.Common.Services; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; -using OpenAI.Responses; namespace EssentialCSharp.Chat; @@ -73,16 +73,8 @@ static int Main(string[] args) "AIOptions section is missing or not configured correctly in appsettings.json or environment variables."); builder.Services.Configure(config.GetRequiredSection("AIOptions")); - // Register Azure OpenAI text embedding generation service -#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - // Replace the obsolete method call with the new method - builder.AddAzureOpenAIEmbeddingGenerator(aiOptions.VectorGenerationDeploymentName, aiOptions.Endpoint, aiOptions.ApiKey); - - builder.AddAzureOpenAIChatClient( - aiOptions.ChatDeploymentName, - aiOptions.Endpoint, - aiOptions.ApiKey); -#pragma warning restore SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + // Use shared extension to register Azure OpenAI services + builder.Services.AddAzureOpenAIServices(aiOptions); builder.Services.AddLogging(loggingBuilder => { @@ -93,12 +85,6 @@ static int Main(string[] args) }); }); - builder.Services.AddPostgresVectorStore( - aiOptions.PostgresConnectionString); - - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - // Build the kernel and get the data uploader. var kernel = builder.Build(); var directory = parseResult.GetValue(directoryOption); @@ -147,11 +133,7 @@ static int Main(string[] args) //IMcpClient mcpClient = await McpClientFactory.CreateAsync(clientTransport: microsoftLearnMcp, cancellationToken: cancellationToken); var enableStreaming = parseResult.GetValue("--stream"); - var enableWebSearch = parseResult.GetValue("--web-search"); - enableWebSearch = false; var customSystemPrompt = parseResult.GetValue("--system-prompt"); - var enableContextualSearch = parseResult.GetValue("--contextual-search"); - enableContextualSearch = true; AIOptions aiOptions = config.GetRequiredSection("AIOptions").Get() ?? throw new InvalidOperationException( @@ -166,18 +148,8 @@ static int Main(string[] args) options.SingleLine = true; })); - // Add services for contextual search if enabled - if (enableContextualSearch) - { -#pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - services.AddAzureOpenAIEmbeddingGenerator(aiOptions.VectorGenerationDeploymentName, aiOptions.Endpoint, aiOptions.ApiKey); -#pragma warning restore SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - services.AddPostgresVectorStore(aiOptions.PostgresConnectionString); - services.AddSingleton(); - services.AddSingleton(); - } - - services.AddSingleton(); + // Use shared extension to register Azure OpenAI services + services.AddAzureOpenAIServices(aiOptions); var serviceProvider = services.BuildServiceProvider(); var aiChatService = serviceProvider.GetRequiredService(); @@ -185,8 +157,6 @@ static int Main(string[] args) Console.WriteLine("🤖 AI Chat Session Started!"); Console.WriteLine("Features enabled:"); Console.WriteLine($" • Streaming: {(enableStreaming ? "✅" : "❌")}"); - Console.WriteLine($" • Web Search: {(enableWebSearch ? "✅" : "❌")}"); - Console.WriteLine($" • Contextual Search: {(enableContextualSearch ? "✅" : "❌")}"); if (!string.IsNullOrEmpty(customSystemPrompt)) Console.WriteLine($" • Custom System Prompt: {customSystemPrompt}"); Console.WriteLine(); @@ -272,10 +242,8 @@ static int Main(string[] args) // Use streaming with optional tools and conversation context var fullResponse = new System.Text.StringBuilder(); - var tools = enableWebSearch ? new[] { ResponseTool.CreateWebSearchTool() } : null; - await foreach (var (text, responseId) in aiChatService.GetChatCompletionStream( - prompt: userInput/*, mcpClient: mcpClient*/, previousResponseId: previousResponseId, enableContextualSearch: enableContextualSearch, systemPrompt: customSystemPrompt, cancellationToken: cancellationToken)) + prompt: userInput/*, mcpClient: mcpClient*/, previousResponseId: previousResponseId, systemPrompt: customSystemPrompt, cancellationToken: cancellationToken)) { if (!string.IsNullOrEmpty(text)) { @@ -294,10 +262,8 @@ static int Main(string[] args) else { // Non-streaming response with optional tools and conversation context - var tools = enableWebSearch ? new[] { ResponseTool.CreateWebSearchTool() } : null; - var (response, responseId) = await aiChatService.GetChatCompletion( - prompt: userInput, previousResponseId: previousResponseId, enableContextualSearch: enableContextualSearch, systemPrompt: customSystemPrompt, cancellationToken: cancellationToken); + prompt: userInput, previousResponseId: previousResponseId, systemPrompt: customSystemPrompt, cancellationToken: cancellationToken); Console.WriteLine(response); conversationHistory.Add(("Assistant", response)); diff --git a/EssentialCSharp.Web.sln b/EssentialCSharp.Web.sln index 4eacae36..17425b1a 100644 --- a/EssentialCSharp.Web.sln +++ b/EssentialCSharp.Web.sln @@ -25,8 +25,6 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.Chat", "Ess EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.Chat.Common", "EssentialCSharp.Chat.Shared\EssentialCSharp.Chat.Common.csproj", "{1B9082D5-D325-42DB-9EC3-03A3953EA8EE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.VectorDbBuilder", "EssentialCSharp.VectorDbBuilder\EssentialCSharp.VectorDbBuilder.csproj", "{8F7A5E12-B234-4C8A-9D45-12F456789ABC}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EssentialCSharp.Chat.Tests", "EssentialCSharp.Chat.Tests\EssentialCSharp.Chat.Tests.csproj", "{05CC9D8A-D928-4537-AD09-737C43DFC00D}" EndProject Global @@ -51,10 +49,6 @@ Global {1B9082D5-D325-42DB-9EC3-03A3953EA8EE}.Debug|Any CPU.Build.0 = Debug|Any CPU {1B9082D5-D325-42DB-9EC3-03A3953EA8EE}.Release|Any CPU.ActiveCfg = Release|Any CPU {1B9082D5-D325-42DB-9EC3-03A3953EA8EE}.Release|Any CPU.Build.0 = Release|Any CPU - {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8F7A5E12-B234-4C8A-9D45-12F456789ABC}.Release|Any CPU.Build.0 = Release|Any CPU {05CC9D8A-D928-4537-AD09-737C43DFC00D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {05CC9D8A-D928-4537-AD09-737C43DFC00D}.Debug|Any CPU.Build.0 = Debug|Any CPU {05CC9D8A-D928-4537-AD09-737C43DFC00D}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs b/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs deleted file mode 100644 index a365b220..00000000 --- a/EssentialCSharp.Web/Extensions/AIServiceCollectionExtensions.cs +++ /dev/null @@ -1,75 +0,0 @@ -using EssentialCSharp.Chat.Common.Services; -using EssentialCSharp.Chat; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel; - -namespace EssentialCSharp.Web.Extensions; - -public static class AIServiceCollectionExtensions -{ - /// - /// Adds AI chat services to the dependency injection container - /// - public static IServiceCollection AddAIChatServices(this IServiceCollection services, IConfiguration configuration) - { - try - { - // Configure AI options from configuration - services.Configure(configuration.GetSection("AIOptions")); - - var aiOptions = configuration.GetSection("AIOptions").Get(); - - // If AI options are missing or incomplete, log warning and skip registration - if (aiOptions == null) - { - throw new InvalidOperationException("AIOptions section is missing from configuration."); - } - - // Validate required configuration - if (string.IsNullOrEmpty(aiOptions.Endpoint) || - aiOptions.Endpoint.Contains("your-azure-openai-endpoint") || - string.IsNullOrEmpty(aiOptions.ApiKey) || - aiOptions.ApiKey.Contains("your-azure-openai-api-key")) - { - throw new InvalidOperationException("Azure OpenAI Endpoint and ApiKey must be properly configured in AIOptions. Please update your configuration with valid values."); - } - - if (string.IsNullOrEmpty(aiOptions.PostgresConnectionString) || - aiOptions.PostgresConnectionString.Contains("your-postgres-connection-string")) - { - throw new InvalidOperationException("PostgreSQL connection string must be properly configured in AIOptions for vector store. Please update your configuration with a valid connection string."); - } - - #pragma warning disable SKEXP0010 // Type is for evaluation purposes only and is subject to change or removal in future updates. - // Register Azure OpenAI services - services.AddAzureOpenAIEmbeddingGenerator( - aiOptions.VectorGenerationDeploymentName, - aiOptions.Endpoint, - aiOptions.ApiKey); - - services.AddAzureOpenAIChatClient( - aiOptions.ChatDeploymentName, - aiOptions.Endpoint, - aiOptions.ApiKey); - - // Add PostgreSQL vector store - services.AddPostgresVectorStore(aiOptions.PostgresConnectionString); - #pragma warning restore SKEXP0010 - - // Register AI services - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - - return services; - } - catch (Exception) - { - // If AI services fail to register, don't register them at all - // The ChatController will handle the null service gracefully - throw; // Re-throw so Program.cs can log the specific error - } - } -} diff --git a/EssentialCSharp.Web/Program.cs b/EssentialCSharp.Web/Program.cs index d3fd49e3..c271977b 100644 --- a/EssentialCSharp.Web/Program.cs +++ b/EssentialCSharp.Web/Program.cs @@ -1,5 +1,6 @@ using System.Threading.RateLimiting; using Azure.Monitor.OpenTelemetry.AspNetCore; +using EssentialCSharp.Chat.Common.Extensions; using EssentialCSharp.Web.Areas.Identity.Data; using EssentialCSharp.Web.Areas.Identity.Services.PasswordValidators; using EssentialCSharp.Web.Data; @@ -95,7 +96,6 @@ private static void Main(string[] args) builder.Configuration .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddUserSecrets() - .AddEnvironmentVariables() .AddEnvironmentVariables(); builder.Services.ConfigureApplicationCookie(options => @@ -154,14 +154,9 @@ private static void Main(string[] args) builder.Services.AddScoped(); // Add AI Chat services - try - { - builder.Services.AddAIChatServices(builder.Configuration); - logger.LogInformation("AI Chat services registered successfully."); - } - catch (Exception ex) + if (!builder.Environment.IsDevelopment()) { - logger.LogWarning(ex, "AI Chat services could not be registered. Chat functionality will be unavailable."); + builder.Services.AddAzureOpenAIServices(configuration); } // Add Rate Limiting for API endpoints @@ -179,7 +174,7 @@ private static void Main(string[] args) factory: _ => new FixedWindowRateLimiterOptions { PermitLimit = 30, // requests per window - Window = TimeSpan.FromMinutes(1), // 1 minute window + Window = TimeSpan.FromMinutes(1), // minute window QueueProcessingOrder = QueueProcessingOrder.OldestFirst, QueueLimit = 0 // No queuing - immediate rejection for better UX }); @@ -302,7 +297,6 @@ await context.HttpContext.Response.WriteAsync( app.UseAuthentication(); app.UseAuthorization(); - // Enable rate limiting middleware (must be after UseAuthentication) app.UseRateLimiter(); app.UseMiddleware(); diff --git a/EssentialCSharp.Web/appsettings.json b/EssentialCSharp.Web/appsettings.json index c6ba7e39..72041284 100644 --- a/EssentialCSharp.Web/appsettings.json +++ b/EssentialCSharp.Web/appsettings.json @@ -19,8 +19,8 @@ "VectorGenerationDeploymentName": "text-embedding-ada-002", "ChatDeploymentName": "gpt-4o", "SystemPrompt": "You are a helpful AI assistant with expertise in C# programming and the Essential C# book content. You can help users understand C# concepts, answer programming questions, and provide guidance based on the Essential C# book materials. Be concise but thorough in your explanations.", - "Endpoint": "your-azure-openai-endpoint-here", - "ApiKey": "your-azure-openai-api-key-here", + "Endpoint": "", + "ApiKey": "", "PostgresConnectionString": "your-postgres-connection-string-here" } } \ No newline at end of file diff --git a/EssentialCSharp.Web/wwwroot/css/chat-widget.css b/EssentialCSharp.Web/wwwroot/css/chat-widget.css index b3b1e26c..6b6398ed 100644 --- a/EssentialCSharp.Web/wwwroot/css/chat-widget.css +++ b/EssentialCSharp.Web/wwwroot/css/chat-widget.css @@ -2,10 +2,21 @@ /* Screen reader only content */ .visually-hidden { - /* Optimized pulse animation */ + position: absolute !important; + width: 1px !important; + height: 1px !important; + padding: 0 !important; + margin: -1px !important; + overflow: hidden !important; + clip: rect(0, 0, 0, 0) !important; + white-space: nowrap !important; + border: 0 !important; +} + +/* Optimized pulse animation */ @keyframes pulse { 0% { - transform: scale(1); + position: absolute !important; opacity: 1; } 50% { @@ -16,15 +27,6 @@ transform: scale(1); opacity: 1; } -}olute !important; - width: 1px !important; - height: 1px !important; - padding: 0 !important; - margin: -1px !important; - overflow: hidden !important; - clip: rect(0, 0, 0, 0) !important; - white-space: nowrap !important; - border: 0 !important; } /* Main widget container */ diff --git a/EssentialCSharp.Web/wwwroot/js/site.js b/EssentialCSharp.Web/wwwroot/js/site.js index c585ba21..f2e943d7 100644 --- a/EssentialCSharp.Web/wwwroot/js/site.js +++ b/EssentialCSharp.Web/wwwroot/js/site.js @@ -25,6 +25,10 @@ const tocData = markRaw(TOC_DATA); //Add new content or features here: const featuresComingSoonList = [ + { + title: "AI Chat Assistant", + text: "Chat with an AI assistant that has access to Essential C# book content. Available as a floating widget on every page for contextual help while reading. Features streaming responses and markdown rendering.", + }, { title: "Client-side Compiler", text: "Write, compile, and run code snippets right from your browser. Enjoy hands-on experience with the code as you go through the site.", @@ -37,10 +41,6 @@ const featuresComingSoonList = [ title: "Hyperlinking", text: "Easily navigate to interesting and relevant sites as well as related sections in Essential C#.", }, - { - title: "Table of Contents Filtering", - text: "The Table of Contents filter will let you narrow down the list of topics to help you quickly and easily find your destination.", - }, ]; const contentComingSoonList = [ @@ -59,10 +59,6 @@ const contentComingSoonList = [ ]; const completedFeaturesList = [ - { - title: "AI Chat Assistant", - text: "Chat with an AI assistant that has access to Essential C# book content. Available as a floating widget on every page for contextual help while reading. Features streaming responses and markdown rendering.", - }, { title: "Copying Header Hyperlinks", text: "Easily copy a header URL to link to a book section.", @@ -131,19 +127,7 @@ const app = createApp({ const snackbarColor = ref(); function copyToClipboard(copyText) { - let url; - - // If copyText contains a #, it's a full path with anchor (e.g., 'page#anchor') - // If copyText doesn't contain a #, it's just an anchor for the current page - if (copyText.includes('#')) { - // Full path case: construct URL with origin + path - url = window.location.origin + "/" + copyText; - } else { - // Anchor only case: use current page URL + anchor - const currentUrl = window.location.href.split('#')[0]; // Remove any existing anchor - url = currentUrl + "#" + copyText; - } - + let url = window.location.origin + "/" + copyText; let referralId = REFERRAL_ID; if (referralId && referralId.trim()) { url = addQueryParam(url, 'rid', referralId); @@ -370,7 +354,6 @@ const app = createApp({ enableTocFilter, isContentPage, - // Chat functionality - spread the chat widget ...chatWidget }; }, @@ -391,4 +374,4 @@ const vuetify = createVuetify({ // Use Vuetify with the Vue app app.use(vuetify); -app.mount("#app"); +app.mount("#app"); \ No newline at end of file diff --git a/EssentialCSharp.Web/wwwroot/lib/vue/LICENSE b/EssentialCSharp.Web/wwwroot/lib/vue/LICENSE deleted file mode 100644 index 15f1f7e7..00000000 --- a/EssentialCSharp.Web/wwwroot/lib/vue/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018-present, Yuxi (Evan) You - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/EssentialCSharp.Web/wwwroot/lib/vue/README.md b/EssentialCSharp.Web/wwwroot/lib/vue/README.md deleted file mode 100644 index a98bd997..00000000 --- a/EssentialCSharp.Web/wwwroot/lib/vue/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# vue - -## Which dist file to use? - -### From CDN or without a Bundler - -- **`vue(.runtime).global(.prod).js`**: - - For direct use via `