From e49742b4a4767e31a22aee95c10396ef110990d9 Mon Sep 17 00:00:00 2001
From: Martijn Laarman <Mpdreamz@gmail.com>
Date: Wed, 11 Jun 2025 17:57:49 +0200
Subject: [PATCH 1/4] Refactor markdown and layout rendering by decoupling from
 `GlobalLayoutViewModel`. Streamline layout handling, enhance navigation item
 visibility logic, and rework documentation file structures for improved
 maintainability. (#1353)

* Refactor markdown and layout rendering by decoupling from `GlobalLayoutViewModel`. Streamline layout handling, enhance navigation item visibility logic, and rework documentation file structures for improved maintainability.

`Hidden` is a pure navigation property now as its removed from `MarkdownFile`.

`GetCurrentNavigation()` is now non nullable

* Ensure NavigationIndex is a navigation property.

* Update advertising of landing-page-path

* update landing page path output

* Add xmldocs for IsPhantom

(cherry picked from commit 8df9149187bb96b1478cea9fc69259148439ff15)
---
 .../Endpoints/ApiEndpoint.cs                  |   3 +
 .../Endpoints/EndpointView.cshtml             |   9 --
 .../Landing/LandingNavigationItem.cs          |   2 +
 .../Landing/LandingView.cshtml                |   9 --
 .../Operations/OperationNavigationItem.cs     |   4 +
 .../Operations/OperationView.cshtml           |   9 --
 .../TableOfContents/ITocItem.cs               |  16 +++
 .../Layout/_Head.cshtml                       |   7 --
 .../Layout/_SecondaryNav.cshtml               |   2 +-
 .../Navigation/INavigationItem.cs             |   4 +
 .../Navigation/_TocTreeNav.cshtml             |   4 +
 .../_GlobalLayout.cshtml                      |  42 +------
 src/Elastic.Documentation.Site/_ViewModels.cs |  34 +++--
 src/Elastic.Markdown/IO/DocumentationFile.cs  |   1 +
 src/Elastic.Markdown/IO/DocumentationSet.cs   | 119 ++++++++++--------
 src/Elastic.Markdown/IO/MarkdownFile.cs       |   6 +-
 .../IO/Navigation/DocumentationGroup.cs       |  34 +++--
 .../Links/CrossLinks/CrossLinkFetcher.cs      |   9 +-
 .../Myst/FrontMatter/FrontMatterParser.cs     |   1 +
 src/Elastic.Markdown/Slices/HtmlWriter.cs     |   2 +-
 src/Elastic.Markdown/Slices/Index.cshtml      |  21 +++-
 src/Elastic.Markdown/Slices/IndexViewModel.cs |   3 +-
 .../Slices}/Layout/_Archive.cshtml            |   2 +-
 .../Slices/Layout/_Breadcrumbs.cshtml         |   2 +-
 .../Slices}/Layout/_LandingPage.cshtml        |   2 +-
 .../Slices}/Layout/_NotFound.cshtml           |   2 +-
 .../Slices/Layout/_PrevNextNav.cshtml         |   2 +-
 .../Slices/Layout/_TableOfContents.cshtml     |   2 +-
 .../Slices}/LayoutName.cs                     |   2 +-
 .../Slices/MarkdownLayoutViewModel.cs         |  32 +++++
 src/Elastic.Markdown/Slices/_Layout.cshtml    |  87 +++++++++----
 .../Slices/_ViewImports.cshtml                |   1 +
 .../Elastic.Documentation.Refactor/Move.cs    |   2 +-
 src/tooling/docs-assembler/AssembleSources.cs |  11 +-
 .../docs-assembler/Cli/RepositoryCommands.cs  |  22 ++--
 .../Navigation/GlobalNavigation.cs            | 111 +++-------------
 .../Navigation/GlobalNavigationFile.cs        |   5 +-
 .../Navigation/GlobalNavigationHtmlWriter.cs  |   3 +
 src/tooling/docs-assembler/navigation.yml     |   1 +
 src/tooling/docs-builder/Cli/Commands.cs      |  11 +-
 .../docs-builder/Http/DocumentationWebHost.cs |  20 +--
 .../GlobalNavigationTests.cs                  |   5 +-
 42 files changed, 352 insertions(+), 314 deletions(-)
 rename src/{Elastic.Documentation.Site => Elastic.Markdown/Slices}/Layout/_Archive.cshtml (99%)
 rename src/{Elastic.Documentation.Site => Elastic.Markdown/Slices}/Layout/_LandingPage.cshtml (99%)
 rename src/{Elastic.Documentation.Site => Elastic.Markdown/Slices}/Layout/_NotFound.cshtml (88%)
 rename src/{Elastic.Documentation.Site => Elastic.Markdown/Slices}/LayoutName.cs (91%)
 create mode 100644 src/Elastic.Markdown/Slices/MarkdownLayoutViewModel.cs

diff --git a/src/Elastic.ApiExplorer/Endpoints/ApiEndpoint.cs b/src/Elastic.ApiExplorer/Endpoints/ApiEndpoint.cs
index 356e7651c..2c72f93d4 100644
--- a/src/Elastic.ApiExplorer/Endpoints/ApiEndpoint.cs
+++ b/src/Elastic.ApiExplorer/Endpoints/ApiEndpoint.cs
@@ -58,10 +58,13 @@ public EndpointNavigationItem(int depth, string url, ApiEndpoint apiEndpoint, La
 	public ApiEndpoint Index { get; }
 	public string Url { get; }
 	public string NavigationTitle { get; }
+	public bool Hidden => false;
 
 	public IReadOnlyCollection<OperationNavigationItem> NavigationItems { get; set; } = [];
 
 	public INodeNavigationItem<INavigationModel, INavigationItem> NavigationRoot { get; }
 
 	public INodeNavigationItem<INavigationModel, INavigationItem>? Parent { get; set; }
+
+	public int NavigationIndex { get; set; }
 }
diff --git a/src/Elastic.ApiExplorer/Endpoints/EndpointView.cshtml b/src/Elastic.ApiExplorer/Endpoints/EndpointView.cshtml
index 023545495..95bc519ec 100644
--- a/src/Elastic.ApiExplorer/Endpoints/EndpointView.cshtml
+++ b/src/Elastic.ApiExplorer/Endpoints/EndpointView.cshtml
@@ -7,24 +7,15 @@
 	{
 		DocSetName = "Api Explorer",
 		Description = "",
-		Layout = null,
-		PageTocItems = [],
 		CurrentNavigationItem = Model.CurrentNavigationItem,
 		Previous = null,
 		Next = null,
 		NavigationHtml = Model.NavigationHtml,
-		LegacyPage = null,
 		UrlPathPrefix = null,
-		GithubEditUrl = null,
-		ReportIssueUrl = null,
 		AllowIndexing = false,
 		CanonicalBaseUrl = null,
 		GoogleTagManager = new GoogleTagManagerConfiguration(),
 		Features = new FeatureFlags([]),
-		Parents =
-		[
-		],
-		Products = null,
 		StaticFileContentHashProvider = Model.StaticFileContentHashProvider
 	};
 }
diff --git a/src/Elastic.ApiExplorer/Landing/LandingNavigationItem.cs b/src/Elastic.ApiExplorer/Landing/LandingNavigationItem.cs
index 6a1f798a9..f472b58eb 100644
--- a/src/Elastic.ApiExplorer/Landing/LandingNavigationItem.cs
+++ b/src/Elastic.ApiExplorer/Landing/LandingNavigationItem.cs
@@ -34,8 +34,10 @@ public class LandingNavigationItem : INodeNavigationItem<ApiLanding, EndpointNav
 	public int Depth { get; }
 	public ApiLanding Index { get; }
 	public INodeNavigationItem<INavigationModel, INavigationItem>? Parent { get; set; }
+	public int NavigationIndex { get; set; }
 	public IReadOnlyCollection<EndpointNavigationItem> NavigationItems { get; set; } = [];
 	public string Url { get; }
+	public bool Hidden => false;
 
 	//TODO
 	public string NavigationTitle { get; } = "API Documentation";
diff --git a/src/Elastic.ApiExplorer/Landing/LandingView.cshtml b/src/Elastic.ApiExplorer/Landing/LandingView.cshtml
index ac0db7b18..d24553791 100644
--- a/src/Elastic.ApiExplorer/Landing/LandingView.cshtml
+++ b/src/Elastic.ApiExplorer/Landing/LandingView.cshtml
@@ -7,24 +7,15 @@
 	{
 		DocSetName = "Api Explorer",
 		Description = "",
-		Layout = null,
-		PageTocItems = [],
 		CurrentNavigationItem = Model.CurrentNavigationItem,
 		Previous = null,
 		Next = null,
 		NavigationHtml = Model.NavigationHtml,
-		LegacyPage = null,
 		UrlPathPrefix = null,
-		GithubEditUrl = null,
-		ReportIssueUrl = null,
 		AllowIndexing = false,
 		CanonicalBaseUrl = null,
 		GoogleTagManager = new GoogleTagManagerConfiguration(),
 		Features = new FeatureFlags([]),
-		Parents =
-		[
-		],
-		Products = null,
 		StaticFileContentHashProvider = Model.StaticFileContentHashProvider
 	};
 }
diff --git a/src/Elastic.ApiExplorer/Operations/OperationNavigationItem.cs b/src/Elastic.ApiExplorer/Operations/OperationNavigationItem.cs
index 81c040654..fe021ca1e 100644
--- a/src/Elastic.ApiExplorer/Operations/OperationNavigationItem.cs
+++ b/src/Elastic.ApiExplorer/Operations/OperationNavigationItem.cs
@@ -47,8 +47,12 @@ public OperationNavigationItem(int depth, string url, ApiOperation apiOperation,
 	public int Depth { get; }
 	public ApiOperation Model { get; }
 	public string Url { get; }
+	public bool Hidden => false;
 
 	public string NavigationTitle { get; }
 
 	public INodeNavigationItem<INavigationModel, INavigationItem>? Parent { get; set; }
+
+	public int NavigationIndex { get; set; }
+
 }
diff --git a/src/Elastic.ApiExplorer/Operations/OperationView.cshtml b/src/Elastic.ApiExplorer/Operations/OperationView.cshtml
index 8404f7fbe..12d821176 100644
--- a/src/Elastic.ApiExplorer/Operations/OperationView.cshtml
+++ b/src/Elastic.ApiExplorer/Operations/OperationView.cshtml
@@ -7,24 +7,15 @@
 	{
 		DocSetName = "Api Explorer",
 		Description = "",
-		Layout = null,
-		PageTocItems = [],
 		CurrentNavigationItem = Model.CurrentNavigationItem,
 		Previous = null,
 		Next = null,
 		NavigationHtml = Model.NavigationHtml,
-		LegacyPage = null,
 		UrlPathPrefix = null,
-		GithubEditUrl = null,
-		ReportIssueUrl = null,
 		AllowIndexing = false,
 		CanonicalBaseUrl = null,
 		GoogleTagManager = new GoogleTagManagerConfiguration(),
 		Features = new FeatureFlags([]),
-		Parents =
-		[
-		],
-		Products = null,
 		StaticFileContentHashProvider = Model.StaticFileContentHashProvider
 	};
 }
diff --git a/src/Elastic.Documentation.Configuration/TableOfContents/ITocItem.cs b/src/Elastic.Documentation.Configuration/TableOfContents/ITocItem.cs
index 4046135ad..a5f745150 100644
--- a/src/Elastic.Documentation.Configuration/TableOfContents/ITocItem.cs
+++ b/src/Elastic.Documentation.Configuration/TableOfContents/ITocItem.cs
@@ -22,4 +22,20 @@ public record TocReference(Uri Source, ITableOfContentsScope TableOfContentsScop
 {
 	public IReadOnlyDictionary<Uri, TocReference> TocReferences { get; } =
 		Children.OfType<TocReference>().ToDictionary(kv => kv.Source, kv => kv);
+
+	/// <summary>
+	/// A phantom table of contents is a table of contents that is not rendered in the UI but is used to generate the TOC.
+	/// This should be used sparingly and needs explicit configuration in navigation.yml.
+	/// It's typically used for container TOC that holds various other TOC's where its children are rehomed throughout the navigation.
+	/// <para>Examples of phantom toc's:</para>
+	/// <list type="">
+	///		<item> - toc: elasticsearch://reference</item>
+	///		<item> - toc: docs-content://</item>
+	/// </list>
+	/// <para>Because navigation.yml does exhaustive checks to ensure all toc.yml files are referenced, marking these containers as phantoms
+	/// ensures that these skip validation checks
+	/// </para>
+	/// </summary>
+	public bool IsPhantom { get; init; }
 }
+
diff --git a/src/Elastic.Documentation.Site/Layout/_Head.cshtml b/src/Elastic.Documentation.Site/Layout/_Head.cshtml
index 0080c521e..ef3904a60 100644
--- a/src/Elastic.Documentation.Site/Layout/_Head.cshtml
+++ b/src/Elastic.Documentation.Site/Layout/_Head.cshtml
@@ -1,6 +1,5 @@
 @inherits RazorSlice<Elastic.Documentation.Site.GlobalLayoutViewModel>
 @using FontPreloader = Elastic.Documentation.Site.FileProviders.FontPreloader
-<head hx-head="merge">
 	<meta charset="utf-8">
 	<title>@Model.Title</title>
 	<meta name="description" content="@Model.Description">
@@ -35,9 +34,3 @@
 	{
 		<meta property="og:url" content="@Model.CanonicalUrl" />
 	}
-	@if (!string.IsNullOrEmpty(Model.Products))
-	{	
-		<meta class="elastic" name="product_name" content="@(Model.Products)"/>
-		<meta name="DC.subject" content="@(Model.Products)"/>
-	}
-</head>
diff --git a/src/Elastic.Documentation.Site/Layout/_SecondaryNav.cshtml b/src/Elastic.Documentation.Site/Layout/_SecondaryNav.cshtml
index 60436170d..49ba93c0b 100644
--- a/src/Elastic.Documentation.Site/Layout/_SecondaryNav.cshtml
+++ b/src/Elastic.Documentation.Site/Layout/_SecondaryNav.cshtml
@@ -2,7 +2,7 @@
 <nav id="secondary-nav" class="bg-grey-10 border-t-1 border-grey-20 font-sans font-semibold text-sm text-ink-light md:text-base">
 	<div class="max-w-(--max-layout-width) flex mx-auto justify-between items-center p-6">
 		<div class="flex gap-2 flex-nowrap items-center">
-			@if (Model.Layout != LayoutName.LandingPage)
+			@if (Model.RenderHamburgerIcon)
 			{
 				@* ReSharper disable once Html.IdNotResolved *@
 				<label role="button" class="md:hidden cursor-pointer" for="pages-nav-hamburger">
diff --git a/src/Elastic.Documentation.Site/Navigation/INavigationItem.cs b/src/Elastic.Documentation.Site/Navigation/INavigationItem.cs
index 5cf0894bf..99a2a688a 100644
--- a/src/Elastic.Documentation.Site/Navigation/INavigationItem.cs
+++ b/src/Elastic.Documentation.Site/Navigation/INavigationItem.cs
@@ -30,6 +30,10 @@ public interface INavigationItem
 	/// TODO: This should be read-only however currently needs the setter in assembler.
 	/// </remarks>
 	INodeNavigationItem<INavigationModel, INavigationItem>? Parent { get; set; }
+
+	bool Hidden { get; }
+
+	int NavigationIndex { get; set; }
 }
 
 /// Represents a leaf node in the navigation tree with associated model data.
diff --git a/src/Elastic.Documentation.Site/Navigation/_TocTreeNav.cshtml b/src/Elastic.Documentation.Site/Navigation/_TocTreeNav.cshtml
index 31246dcd5..e67673c55 100644
--- a/src/Elastic.Documentation.Site/Navigation/_TocTreeNav.cshtml
+++ b/src/Elastic.Documentation.Site/Navigation/_TocTreeNav.cshtml
@@ -5,6 +5,10 @@
 }
 @foreach (var item in Model.SubTree.NavigationItems)
 {
+	if (item.Hidden)
+	{
+		continue;
+	}
 	if (item is INodeNavigationItem<INavigationModel, INavigationItem> { NavigationItems.Count: 0, Index: not null } group)
 	{
 		<li class="flex group/li pr-8 @(isTopLevel ? "font-semibold mt-6" : "mt-4")">
diff --git a/src/Elastic.Documentation.Site/_GlobalLayout.cshtml b/src/Elastic.Documentation.Site/_GlobalLayout.cshtml
index 361d920d4..a396783cf 100644
--- a/src/Elastic.Documentation.Site/_GlobalLayout.cshtml
+++ b/src/Elastic.Documentation.Site/_GlobalLayout.cshtml
@@ -1,11 +1,10 @@
 @inherits RazorLayoutSlice<GlobalLayoutViewModel>
 <!DOCTYPE html>
 <html lang="en" class="h-screen" xmlns="http://www.w3.org/1999/html">
-@await RenderPartialAsync(_Head.Create(Model))
-
-@{
-	var layout = Model.Layout;
-}
+<head hx-head="merge">
+	@await RenderPartialAsync(_Head.Create(Model))
+	@await RenderSectionAsync(GlobalSections.Head)
+</head>
 
 <body
 	class="group/body text-ink has-[#primary-nav-hamburger:checked]:overflow-hidden"
@@ -18,38 +17,9 @@
 }
 @(await RenderPartialAsync(_Header.Create(Model)))
 <div id="main-container" class="flex flex-col items-center border-t-1 border-grey-20">
-	@switch (layout)
-	{
-		case LayoutName.NotFound:
-			await RenderPartialAsync(_NotFound.Create(Model));
-			break;
-		case LayoutName.LandingPage:
-			await RenderPartialAsync(_LandingPage.Create(Model));
-			break;
-		case LayoutName.Archive:
-			await RenderPartialAsync(_Archive.Create(Model));
-			break;
-		default:
-			await RenderBodyAsync();
-			break;
-	}
+	@(await RenderBodyAsync())
 </div>
 @await RenderPartialAsync(_Footer.Create(Model))
-@if (layout is not LayoutName.Archive)
-{
-	<aside id="dismissible-banner" class="admonition tip">
-		<div class="container flex justify-between items-center mx-auto">
-			<p>
-				Welcome to the docs for the <a class="link text-base" href="/docs/get-started/versioning-availability#find-docs-for-your-product-version">latest Elastic product versions</a>, including Elastic Stack 9.0 and Elastic Cloud Serverless.
-				To view previous versions, go to <a class="link text-base" target="_blank" href="https://elastic.co/guide">elastic.co/guide</a>.
-			</p>
-			<button id="dismissible-button">
-				<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="size-6">
-					<path stroke-linecap="round" stroke-linejoin="round" d="M6 18 18 6M6 6l12 12"/>
-				</svg>
-			</button>
-		</div>
-	</aside>
-}
+@await RenderSectionAsync(GlobalSections.Footer)
 </body>
 </html>
diff --git a/src/Elastic.Documentation.Site/_ViewModels.cs b/src/Elastic.Documentation.Site/_ViewModels.cs
index bf080e779..f171e8885 100644
--- a/src/Elastic.Documentation.Site/_ViewModels.cs
+++ b/src/Elastic.Documentation.Site/_ViewModels.cs
@@ -4,38 +4,41 @@
 
 using Elastic.Documentation.Configuration.Assembler;
 using Elastic.Documentation.Configuration.Builder;
-using Elastic.Documentation.Legacy;
 using Elastic.Documentation.Site.FileProviders;
 using Elastic.Documentation.Site.Navigation;
 
 namespace Elastic.Documentation.Site;
 
+public static class GlobalSections
+{
+	public const string Head = "head";
+	public const string Footer = "footer";
+}
+
 public class GlobalLayoutViewModel
 {
 	public required string DocSetName { get; init; }
 	public string Title { get; set; } = "Elastic Documentation";
 	public required string Description { get; init; }
-	public required LayoutName? Layout { get; init; }
 
-	public required IReadOnlyCollection<PageTocItem> PageTocItems { get; init; }
-	public required INavigationItem? CurrentNavigationItem { get; init; }
+	public required INavigationItem CurrentNavigationItem { get; init; }
 	public required INavigationItem? Previous { get; init; }
 	public required INavigationItem? Next { get; init; }
+
 	public required string NavigationHtml { get; init; }
-	public required LegacyPageMapping? LegacyPage { get; init; }
 	public required string? UrlPathPrefix { get; init; }
-	public required string? GithubEditUrl { get; init; }
-	public required string? ReportIssueUrl { get; init; }
-	public required bool AllowIndexing { get; init; }
 	public required Uri? CanonicalBaseUrl { get; init; }
-	public required GoogleTagManagerConfiguration GoogleTagManager { get; init; }
 	public string? CanonicalUrl => CanonicalBaseUrl is not null ?
 		new Uri(CanonicalBaseUrl, CurrentNavigationItem?.Url).ToString().TrimEnd('/') : null;
+
 	public required FeatureFlags Features { get; init; }
 
-	public required INavigationItem[] Parents { get; init; }
+	// TODO move to @inject
+	public required GoogleTagManagerConfiguration GoogleTagManager { get; init; }
+	public required bool AllowIndexing { get; init; }
+	public required StaticFileContentHashProvider StaticFileContentHashProvider { get; init; }
 
-	public required string? Products { get; init; }
+	public bool RenderHamburgerIcon { get; init; } = true;
 
 	public string Static(string path)
 	{
@@ -46,9 +49,6 @@ public string Static(string path)
 			: $"{UrlPathPrefix}/{staticPath}?v={contentHash}";
 	}
 
-	// TODO move to @inject
-	public required StaticFileContentHashProvider StaticFileContentHashProvider { get; init; }
-
 	public string Link(string path)
 	{
 		path = path.AsSpan().TrimStart('/').ToString();
@@ -56,9 +56,3 @@ public string Link(string path)
 	}
 }
 
-public record PageTocItem
-{
-	public required string Heading { get; init; }
-	public required string Slug { get; init; }
-	public required int Level { get; init; }
-}
diff --git a/src/Elastic.Markdown/IO/DocumentationFile.cs b/src/Elastic.Markdown/IO/DocumentationFile.cs
index e867af7d7..f1dcbaff1 100644
--- a/src/Elastic.Markdown/IO/DocumentationFile.cs
+++ b/src/Elastic.Markdown/IO/DocumentationFile.cs
@@ -5,6 +5,7 @@
 using Elastic.Documentation.Site;
 using Elastic.Markdown.Myst;
 using Elastic.Markdown.Myst.FrontMatter;
+using Elastic.Markdown.Slices;
 
 namespace Elastic.Markdown.IO;
 
diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs
index 2d7c5b842..8129c9477 100644
--- a/src/Elastic.Markdown/IO/DocumentationSet.cs
+++ b/src/Elastic.Markdown/IO/DocumentationSet.cs
@@ -32,10 +32,42 @@ public interface INavigationLookups
 public interface IPositionalNavigation
 {
 	FrozenDictionary<string, INavigationItem> MarkdownNavigationLookup { get; }
+	FrozenDictionary<int, INavigationItem> NavigationIndexedByOrder { get; }
 
-	INavigationItem? GetPrevious(MarkdownFile current);
-	INavigationItem? GetNext(MarkdownFile current);
-	INavigationItem? GetCurrent(MarkdownFile file) => MarkdownNavigationLookup.GetValueOrDefault(file.CrossLink);
+	INavigationItem? GetPrevious(MarkdownFile current)
+	{
+		if (!MarkdownNavigationLookup.TryGetValue(current.CrossLink, out var currentNavigation))
+			return null;
+		var index = currentNavigation.NavigationIndex;
+		do
+		{
+			var previous = NavigationIndexedByOrder.GetValueOrDefault(index - 1);
+			if (previous is not null && !previous.Hidden)
+				return previous;
+			index--;
+		} while (index > 0);
+
+		return null;
+	}
+
+	INavigationItem? GetNext(MarkdownFile current)
+	{
+		if (!MarkdownNavigationLookup.TryGetValue(current.CrossLink, out var currentNavigation))
+			return null;
+		var index = currentNavigation.NavigationIndex;
+		do
+		{
+			var next = NavigationIndexedByOrder.GetValueOrDefault(index + 1);
+			if (next is not null && !next.Hidden)
+				return next;
+			index++;
+		} while (index <= NavigationIndexedByOrder.Count - 1);
+
+		return null;
+	}
+
+	INavigationItem GetCurrent(MarkdownFile file) =>
+		MarkdownNavigationLookup.GetValueOrDefault(file.CrossLink) ?? throw new InvalidOperationException($"Could not find {file.CrossLink} in navigation");
 
 	INavigationItem[] GetParents(INavigationItem current)
 	{
@@ -152,24 +184,45 @@ public DocumentationSet(
 			FilesGroupedByFolder = FilesGroupedByFolder
 		};
 
-		Tree = new TableOfContentsTree(this, Source, Context, lookups, treeCollector, ref fileIndex);
+		Tree = new TableOfContentsTree(Source, Context, lookups, treeCollector, ref fileIndex);
 
 		var markdownFiles = Files.OfType<MarkdownFile>().ToArray();
 
-		var excludedChildren = markdownFiles.Where(f => f.NavigationIndex == -1).ToArray();
+		var excludedChildren = markdownFiles.Where(f => !f.PartOfNavigation).ToArray();
 		foreach (var excludedChild in excludedChildren)
 			Context.EmitError(Context.ConfigurationPath, $"{excludedChild.RelativePath} is unreachable in the TOC because one of its parents matches exclusion glob");
 
-		MarkdownFiles = markdownFiles.Where(f => f.NavigationIndex > -1).ToDictionary(i => i.NavigationIndex, i => i).ToFrozenDictionary();
+		MarkdownFiles = markdownFiles.Where(f => f.PartOfNavigation).ToFrozenSet();
+		NavigationIndexedByOrder = CreateNavigationLookup(Tree)
+			.ToDictionary(n => n.NavigationIndex, n => n)
+			.ToFrozenDictionary();
 
 		MarkdownNavigationLookup = Tree.NavigationItems
 			.SelectMany(Pairs)
+			.Concat(Pairs(Tree))
+			.DistinctBy(kv => kv.Item1)
 			.ToDictionary(kv => kv.Item1, kv => kv.Item2)
 			.ToFrozenDictionary();
 
 		ValidateRedirectsExists();
 	}
 
+	public FrozenDictionary<int, INavigationItem> NavigationIndexedByOrder { get; }
+
+	private static IReadOnlyCollection<INavigationItem> CreateNavigationLookup(INavigationItem item)
+	{
+		if (item is ILeafNavigationItem<INavigationModel> leaf)
+			return [leaf];
+
+		if (item is INodeNavigationItem<INavigationModel, INavigationItem> node)
+		{
+			var items = node.NavigationItems.SelectMany(CreateNavigationLookup);
+			return items.Concat([node]).ToArray();
+		}
+
+		return [];
+	}
+
 	public static (string, INavigationItem)[] Pairs(INavigationItem item)
 	{
 		if (item is FileNavigationItem f)
@@ -274,7 +327,10 @@ void ValidateExists(string from, string to, IReadOnlyDictionary<string, string?>
 		}
 	}
 
-	public FrozenDictionary<int, MarkdownFile> MarkdownFiles { get; }
+	public FrozenSet<MarkdownFile> MarkdownFiles { get; }
+
+	public string FirstInterestingUrl =>
+		NavigationIndexedByOrder.Values.OfType<DocumentationGroup>().First().Url;
 
 	public DocumentationFile? DocumentationFileLookup(IFileInfo sourceFile)
 	{
@@ -282,44 +338,6 @@ void ValidateExists(string from, string to, IReadOnlyDictionary<string, string?>
 		return FlatMappedFiles.GetValueOrDefault(relativePath);
 	}
 
-	public INavigationItem? GetPrevious(MarkdownFile current)
-	{
-		var index = current.NavigationIndex;
-		do
-		{
-			var previous = MarkdownFiles.GetValueOrDefault(index - 1);
-			if (previous is null)
-				return null;
-			if (!previous.Hidden)
-			{
-				if (MarkdownNavigationLookup.TryGetValue(previous.CrossLink, out var navigationItem))
-					return navigationItem;
-			}
-			index--;
-		} while (index > 0);
-
-		return null;
-	}
-
-	public INavigationItem? GetNext(MarkdownFile current)
-	{
-		var index = current.NavigationIndex;
-		do
-		{
-			var previous = MarkdownFiles.GetValueOrDefault(index + 1);
-			if (previous is null)
-				return null;
-			if (!previous.Hidden)
-			{
-				if (MarkdownNavigationLookup.TryGetValue(previous.CrossLink, out var navigationItem))
-					return navigationItem;
-			}
-			index++;
-		} while (index <= MarkdownFiles.Count - 1);
-
-		return null;
-	}
-
 	public async Task ResolveDirectoryTree(Cancel ctx) =>
 		await Tree.Resolve(ctx);
 
@@ -362,14 +380,17 @@ public RepositoryLinks CreateLinkReference()
 	{
 		var redirects = Configuration.Redirects;
 		var crossLinks = Context.Collector.CrossLinks.ToHashSet().ToArray();
-		var links = MarkdownFiles.Values
-			.Select(m => (m.LinkReferenceRelativePath, File: m))
+		var links = NavigationIndexedByOrder
+			.Select(kv => kv.Value)
+			.OfType<FileNavigationItem>()
+			.Select(m => (m.Model.LinkReferenceRelativePath, Navigation: m))
 			.ToDictionary(k => RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
 			? k.LinkReferenceRelativePath.Replace('\\', '/')
 			: k.LinkReferenceRelativePath, v =>
 			{
-				var anchors = v.File.Anchors.Count == 0 ? null : v.File.Anchors.ToArray();
-				return new LinkMetadata { Anchors = anchors, Hidden = v.File.Hidden };
+				var md = v.Navigation.Model;
+				var anchors = md.Anchors.Count == 0 ? null : md.Anchors.ToArray();
+				return new LinkMetadata { Anchors = anchors, Hidden = v.Navigation.Hidden };
 			});
 
 		return new RepositoryLinks
diff --git a/src/Elastic.Markdown/IO/MarkdownFile.cs b/src/Elastic.Markdown/IO/MarkdownFile.cs
index b9f5cd08b..fa4b99f29 100644
--- a/src/Elastic.Markdown/IO/MarkdownFile.cs
+++ b/src/Elastic.Markdown/IO/MarkdownFile.cs
@@ -15,6 +15,7 @@
 using Elastic.Markdown.Myst.Directives;
 using Elastic.Markdown.Myst.FrontMatter;
 using Elastic.Markdown.Myst.InlineParsers;
+using Elastic.Markdown.Slices;
 using Markdig;
 using Markdig.Extensions.Yaml;
 using Markdig.Renderers.Roundtrip;
@@ -58,6 +59,8 @@ DocumentationSet set
 		NavigationSource = set.Source;
 	}
 
+	public bool PartOfNavigation { get; set; }
+
 	public IDirectoryInfo ScopeDirectory { get; set; }
 
 	public INodeNavigationItem<INavigationModel, INavigationItem> NavigationRoot { get; set; }
@@ -66,7 +69,6 @@ DocumentationSet set
 
 	private IDiagnosticsCollector Collector { get; }
 
-	public bool Hidden { get; internal set; }
 	public string? UrlPathPrefix { get; }
 	protected MarkdownParser MarkdownParser { get; }
 	public YamlFrontMatter? YamlFrontMatter { get; private set; }
@@ -136,7 +138,7 @@ public string Url
 		}
 	}
 
-	public int NavigationIndex { get; set; } = -1;
+	//public int NavigationIndex { get; set; } = -1;
 
 	private bool _instructionsParsed;
 	private string? _title;
diff --git a/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs b/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs
index bd239ce1b..c5f42fdbc 100644
--- a/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs
+++ b/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs
@@ -13,12 +13,13 @@
 namespace Elastic.Markdown.IO.Navigation;
 
 [DebuggerDisplay("Current: {Model.RelativePath}")]
-public record FileNavigationItem(MarkdownFile Model, DocumentationGroup Group) : ILeafNavigationItem<MarkdownFile>
+public record FileNavigationItem(MarkdownFile Model, DocumentationGroup Group, bool Hidden = false) : ILeafNavigationItem<MarkdownFile>
 {
 	public INodeNavigationItem<INavigationModel, INavigationItem>? Parent { get; set; } = Group;
 	public INodeNavigationItem<INavigationModel, INavigationItem> NavigationRoot { get; } = Group.NavigationRoot;
 	public string Url => Model.Url;
 	public string NavigationTitle => Model.NavigationTitle;
+	public int NavigationIndex { get; set; }
 }
 
 public class TableOfContentsTreeCollector
@@ -47,7 +48,6 @@ public class TableOfContentsTree : DocumentationGroup
 	//TODO remove documentation set argument once navigation.yml fully bootstraps.
 	//See GlobalNavigation.BuildNavigation which has fallback logic that needs to be removed
 	public TableOfContentsTree(
-		DocumentationSet documentationSet,
 		Uri source,
 		BuildContext context,
 		NavigationLookups lookups,
@@ -60,7 +60,6 @@ public TableOfContentsTree(
 
 		Source = source;
 		TreeCollector.Collect(source, this);
-		DocumentationSet = documentationSet;
 
 		//edge case if a tree only holds a single group, ensure we collapse it down to the root (this)
 		if (NavigationItems.Count == 1 && NavigationItems.First() is DocumentationGroup { NavigationItems.Count: 0 })
@@ -108,6 +107,10 @@ public class DocumentationGroup : INodeNavigationItem<MarkdownFile, INavigationI
 
 	public string NavigationTitle => Index.NavigationTitle;
 
+	public bool Hidden { get; set; }
+
+	public int NavigationIndex { get; set; }
+
 	private IReadOnlyCollection<MarkdownFile> FilesInOrder { get; }
 
 	private IReadOnlyCollection<DocumentationGroup> GroupsInOrder { get; }
@@ -185,6 +188,15 @@ private MarkdownFile ProcessTocItems(BuildContext context,
 		navigationItems = [];
 		files = [];
 		var indexFile = virtualIndexFile;
+
+		var list = navigationItems;
+
+		void AddToNavigationItems(INavigationItem item, ref int fileIndex)
+		{
+			item.NavigationIndex = Interlocked.Increment(ref fileIndex);
+			list.Add(item);
+		}
+
 		foreach (var tocItem in lookups.TableOfContents)
 		{
 			if (tocItem is FileReference file)
@@ -209,9 +221,9 @@ private MarkdownFile ProcessTocItems(BuildContext context,
 					continue;
 				}
 
-				md.Hidden = file.Hidden;
-				var navigationIndex = Interlocked.Increment(ref fileIndex);
-				md.NavigationIndex = navigationIndex;
+				md.PartOfNavigation = true;
+
+				// TODO these have to be refactor to be pure navigational properties
 				md.ScopeDirectory = file.TableOfContentsScope.ScopeDirectory;
 				md.NavigationRoot = topLevelGroup;
 				md.NavigationSource = NavigationSource;
@@ -229,7 +241,7 @@ private MarkdownFile ProcessTocItems(BuildContext context,
 							TableOfContents = file.Children,
 						}, NavigationSource, ref fileIndex, depth + 1, topLevelGroup, this, md);
 					groups.Add(group);
-					navigationItems.Add(group);
+					AddToNavigationItems(group, ref fileIndex);
 					indexFile ??= md;
 					continue;
 				}
@@ -242,8 +254,8 @@ private MarkdownFile ProcessTocItems(BuildContext context,
 				// the index file can either be the discovered `index.md` or the parent group's
 				// explicit index page. E.g., when grouping related files together.
 				// If the page is referenced as hidden in the TOC do not include it in the navigation
-				if (indexFile != md && !md.Hidden)
-					navigationItems.Add(new FileNavigationItem(md, this));
+				if (indexFile != md)
+					AddToNavigationItems(new FileNavigationItem(md, this, file.Hidden), ref fileIndex);
 			}
 			else if (tocItem is FolderReference folder)
 			{
@@ -266,7 +278,7 @@ .. documentationFiles
 					}, ref fileIndex, depth + 1, topLevelGroup, this);
 
 					group = toc;
-					navigationItems.Add(toc);
+					AddToNavigationItems(toc, ref fileIndex);
 				}
 				else
 				{
@@ -274,7 +286,7 @@ .. documentationFiles
 					{
 						TableOfContents = children
 					}, NavigationSource, ref fileIndex, depth + 1, topLevelGroup, this);
-					navigationItems.Add(group);
+					AddToNavigationItems(group, ref fileIndex);
 				}
 
 				groups.Add(group);
diff --git a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs
index 4923ab188..5afd44e7d 100644
--- a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs
+++ b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs
@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Collections.Concurrent;
 using System.Collections.Frozen;
 using System.Text.Json;
 using Elastic.Documentation.Configuration;
@@ -119,16 +120,22 @@ private void WriteLinksJsonCachedFile(string repository, LinkRegistryEntry linkR
 		}
 	}
 
+	private readonly ConcurrentDictionary<string, RepositoryLinks> _cachedLinkReferences = new();
+
 	private async Task<RepositoryLinks?> TryGetCachedLinkReference(string repository, LinkRegistryEntry linkRegistryEntry)
 	{
-		var cachedFileName = $"links-elastic-{repository}-main-{linkRegistryEntry.ETag}.json";
+		var cachedFileName = $"links-elastic-{repository}-{linkRegistryEntry.Branch}-{linkRegistryEntry.ETag}.json";
 		var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "links", cachedFileName);
+		if (_cachedLinkReferences.TryGetValue(cachedFileName, out var cachedLinkReference))
+			return cachedLinkReference;
+
 		if (File.Exists(cachedPath))
 		{
 			try
 			{
 				var json = await File.ReadAllTextAsync(cachedPath);
 				var linkReference = Deserialize(json);
+				_ = _cachedLinkReferences.TryAdd(cachedFileName, linkReference);
 				return linkReference;
 			}
 			catch (Exception e)
diff --git a/src/Elastic.Markdown/Myst/FrontMatter/FrontMatterParser.cs b/src/Elastic.Markdown/Myst/FrontMatter/FrontMatterParser.cs
index e0b591779..d9e31f60f 100644
--- a/src/Elastic.Markdown/Myst/FrontMatter/FrontMatterParser.cs
+++ b/src/Elastic.Markdown/Myst/FrontMatter/FrontMatterParser.cs
@@ -4,6 +4,7 @@
 
 using Elastic.Documentation.Configuration.Builder;
 using Elastic.Documentation.Site;
+using Elastic.Markdown.Slices;
 using YamlDotNet.Serialization;
 
 namespace Elastic.Markdown.Myst.FrontMatter;
diff --git a/src/Elastic.Markdown/Slices/HtmlWriter.cs b/src/Elastic.Markdown/Slices/HtmlWriter.cs
index f123958ac..821d7f121 100644
--- a/src/Elastic.Markdown/Slices/HtmlWriter.cs
+++ b/src/Elastic.Markdown/Slices/HtmlWriter.cs
@@ -104,7 +104,7 @@ private async Task<string> RenderLayout(MarkdownFile markdown, MarkdownDocument
 			UrlPathPrefix = markdown.UrlPathPrefix,
 			AppliesTo = markdown.YamlFrontMatter?.AppliesTo,
 			GithubEditUrl = editUrl,
-			AllowIndexing = DocumentationSet.Context.AllowIndexing && (markdown is DetectionRuleFile || !markdown.Hidden),
+			AllowIndexing = DocumentationSet.Context.AllowIndexing && (markdown is DetectionRuleFile || !current.Hidden),
 			CanonicalBaseUrl = DocumentationSet.Context.CanonicalBaseUrl,
 			GoogleTagManager = DocumentationSet.Context.GoogleTagManager,
 			Features = DocumentationSet.Configuration.Features,
diff --git a/src/Elastic.Markdown/Slices/Index.cshtml b/src/Elastic.Markdown/Slices/Index.cshtml
index 258b35b39..70b19e8f9 100644
--- a/src/Elastic.Markdown/Slices/Index.cshtml
+++ b/src/Elastic.Markdown/Slices/Index.cshtml
@@ -1,11 +1,12 @@
 @using Elastic.Markdown.Slices.Components
 @using Markdig
 @inherits RazorSliceHttpResult<IndexViewModel>
-@implements IUsesLayout<Elastic.Markdown.Slices._Layout, GlobalLayoutViewModel>
+@implements IUsesLayout<Elastic.Markdown.Slices._Layout, MarkdownLayoutViewModel>
 @functions {
-	public GlobalLayoutViewModel LayoutModel => new()
+	public MarkdownLayoutViewModel LayoutModel => new()
 	{
 		Layout = Model.CurrentDocument.YamlFrontMatter?.Layout,
+		RenderHamburgerIcon = Model.CurrentDocument.YamlFrontMatter?.Layout != LayoutName.LandingPage,
 		DocSetName = Model.DocSetName,
 		Title = $"{Model.Title} | {Model.SiteName}",
 		Description = Model.Description,
@@ -23,9 +24,19 @@
 		Features = Model.Features,
 		StaticFileContentHashProvider = Model.StaticFileContentHashProvider,
 		ReportIssueUrl = Model.ReportIssueUrl,
-		LegacyPage = Model.LegacyPage,
-		Products = Model.Products is { Count: > 0} products ? string.Join(",", products.Select(p => p.DisplayName)) : null,
+		LegacyPage = Model.LegacyPage
 	};
+	protected override Task ExecuteSectionAsync(string name)
+	{
+		if (name == GlobalSections.Head && Model.Products is { Count: > 0 })
+		{
+			var products = string.Join(",", Model.Products.Select(p => p.DisplayName));
+			<meta class="elastic" name="product_name" content="@(products)"/>
+			<meta name="DC.subject" content="@(products)"/>
+		}
+
+		return Task.CompletedTask;
+	}
 }
 <section id="elastic-docs-v3">
 	@* This way it's correctly rendered as <h1>text</h1> instead of <h1><p>text</p></h1> *@
@@ -41,4 +52,4 @@
 		</p>
 	}
 	@(new HtmlString(Model.MarkdownHtml))
-</section>
+</section>
\ No newline at end of file
diff --git a/src/Elastic.Markdown/Slices/IndexViewModel.cs b/src/Elastic.Markdown/Slices/IndexViewModel.cs
index 6334d5684..856da876a 100644
--- a/src/Elastic.Markdown/Slices/IndexViewModel.cs
+++ b/src/Elastic.Markdown/Slices/IndexViewModel.cs
@@ -5,7 +5,6 @@
 using Elastic.Documentation.Configuration.Assembler;
 using Elastic.Documentation.Configuration.Builder;
 using Elastic.Documentation.Legacy;
-using Elastic.Documentation.Site;
 using Elastic.Documentation.Site.FileProviders;
 using Elastic.Documentation.Site.Navigation;
 using Elastic.Markdown.IO;
@@ -26,7 +25,7 @@ public class IndexViewModel
 	public required IReadOnlyCollection<PageTocItem> PageTocItems { get; init; }
 	public required MarkdownFile CurrentDocument { get; init; }
 
-	public required INavigationItem? CurrentNavigationItem { get; init; }
+	public required INavigationItem CurrentNavigationItem { get; init; }
 	public required INavigationItem? PreviousDocument { get; init; }
 	public required INavigationItem? NextDocument { get; init; }
 	public required INavigationItem[] Parents { get; init; }
diff --git a/src/Elastic.Documentation.Site/Layout/_Archive.cshtml b/src/Elastic.Markdown/Slices/Layout/_Archive.cshtml
similarity index 99%
rename from src/Elastic.Documentation.Site/Layout/_Archive.cshtml
rename to src/Elastic.Markdown/Slices/Layout/_Archive.cshtml
index 6e3c6f3ee..06d26704c 100644
--- a/src/Elastic.Documentation.Site/Layout/_Archive.cshtml
+++ b/src/Elastic.Markdown/Slices/Layout/_Archive.cshtml
@@ -1,4 +1,4 @@
-@inherits RazorSlice<Elastic.Documentation.Site.GlobalLayoutViewModel>
+@inherits RazorSlice<MarkdownLayoutViewModel>
 <div class="bg-[#F5F7FA] w-full px-8 py-16 text-center">
 	<div class="container mx-auto mt-20">
 		<h1 class="text-4xl md:text-6xl">Archive</h1>
diff --git a/src/Elastic.Markdown/Slices/Layout/_Breadcrumbs.cshtml b/src/Elastic.Markdown/Slices/Layout/_Breadcrumbs.cshtml
index e282ec055..a0e9c9f46 100644
--- a/src/Elastic.Markdown/Slices/Layout/_Breadcrumbs.cshtml
+++ b/src/Elastic.Markdown/Slices/Layout/_Breadcrumbs.cshtml
@@ -1,4 +1,4 @@
-@inherits RazorSlice<Elastic.Documentation.Site.GlobalLayoutViewModel>
+@inherits RazorSlice<MarkdownLayoutViewModel>
 @{
 	var targets =
 		Model.Features.IsPrimaryNavEnabled
diff --git a/src/Elastic.Documentation.Site/Layout/_LandingPage.cshtml b/src/Elastic.Markdown/Slices/Layout/_LandingPage.cshtml
similarity index 99%
rename from src/Elastic.Documentation.Site/Layout/_LandingPage.cshtml
rename to src/Elastic.Markdown/Slices/Layout/_LandingPage.cshtml
index 16bbe559c..929d9e0a5 100644
--- a/src/Elastic.Documentation.Site/Layout/_LandingPage.cshtml
+++ b/src/Elastic.Markdown/Slices/Layout/_LandingPage.cshtml
@@ -1,4 +1,4 @@
-@inherits RazorSlice<Elastic.Documentation.Site.GlobalLayoutViewModel>
+@inherits RazorSlice<MarkdownLayoutViewModel>
 <div class="w-full text-ink relative text-pretty">
 	<div class="w-full absolute top-0 left-0 right-0 htmx-indicator" id="htmx-indicator" role="status">
 		<div class="h-[2px] w-full overflow-hidden">
diff --git a/src/Elastic.Documentation.Site/Layout/_NotFound.cshtml b/src/Elastic.Markdown/Slices/Layout/_NotFound.cshtml
similarity index 88%
rename from src/Elastic.Documentation.Site/Layout/_NotFound.cshtml
rename to src/Elastic.Markdown/Slices/Layout/_NotFound.cshtml
index 6512a60c1..c8503bc9e 100644
--- a/src/Elastic.Documentation.Site/Layout/_NotFound.cshtml
+++ b/src/Elastic.Markdown/Slices/Layout/_NotFound.cshtml
@@ -1,4 +1,4 @@
-@inherits RazorSlice<Elastic.Documentation.Site.GlobalLayoutViewModel>
+@inherits RazorSlice<MarkdownLayoutViewModel>
 <div class="p-12 font-body text-center flex flex-col items-center justify-center">
 	<span class="text-4xl font-sans font-semibold">Page not found</span>
 	<p class="mt-2">The page you are looking for could not be found.</p>
diff --git a/src/Elastic.Markdown/Slices/Layout/_PrevNextNav.cshtml b/src/Elastic.Markdown/Slices/Layout/_PrevNextNav.cshtml
index 903e363ad..3be23cc97 100644
--- a/src/Elastic.Markdown/Slices/Layout/_PrevNextNav.cshtml
+++ b/src/Elastic.Markdown/Slices/Layout/_PrevNextNav.cshtml
@@ -1,4 +1,4 @@
-@inherits RazorSlice<Elastic.Documentation.Site.GlobalLayoutViewModel>
+@inherits RazorSlice<MarkdownLayoutViewModel>
 <footer id="prev-next-nav" class="content-container mt-20 px-4">
 	<div class="flex flex-wrap lg:flex-nowrap gap-2 mt-2">
 		<div class="w-full">
diff --git a/src/Elastic.Markdown/Slices/Layout/_TableOfContents.cshtml b/src/Elastic.Markdown/Slices/Layout/_TableOfContents.cshtml
index 9a5840816..22bbebaa0 100644
--- a/src/Elastic.Markdown/Slices/Layout/_TableOfContents.cshtml
+++ b/src/Elastic.Markdown/Slices/Layout/_TableOfContents.cshtml
@@ -1,4 +1,4 @@
-@inherits RazorSlice<Elastic.Documentation.Site.GlobalLayoutViewModel>
+@inherits RazorSlice<MarkdownLayoutViewModel>
 <aside class="sidebar hidden lg:block max-w-65 md:hidden">
 	<nav id="toc-nav" class="sidebar-nav">
 		<div id="page-version-dropdown" tabindex="1"
diff --git a/src/Elastic.Documentation.Site/LayoutName.cs b/src/Elastic.Markdown/Slices/LayoutName.cs
similarity index 91%
rename from src/Elastic.Documentation.Site/LayoutName.cs
rename to src/Elastic.Markdown/Slices/LayoutName.cs
index 1db706cbc..92b50d404 100644
--- a/src/Elastic.Documentation.Site/LayoutName.cs
+++ b/src/Elastic.Markdown/Slices/LayoutName.cs
@@ -4,7 +4,7 @@
 
 using System.Runtime.Serialization;
 
-namespace Elastic.Documentation.Site;
+namespace Elastic.Markdown.Slices;
 
 public enum LayoutName
 {
diff --git a/src/Elastic.Markdown/Slices/MarkdownLayoutViewModel.cs b/src/Elastic.Markdown/Slices/MarkdownLayoutViewModel.cs
new file mode 100644
index 000000000..6e3d60eda
--- /dev/null
+++ b/src/Elastic.Markdown/Slices/MarkdownLayoutViewModel.cs
@@ -0,0 +1,32 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Documentation.Legacy;
+using Elastic.Documentation.Site;
+using Elastic.Documentation.Site.Navigation;
+
+namespace Elastic.Markdown.Slices;
+
+public class MarkdownLayoutViewModel : GlobalLayoutViewModel
+{
+	public required string? GithubEditUrl { get; init; }
+
+	public required string? ReportIssueUrl { get; init; }
+
+	public required INavigationItem[] Parents { get; init; }
+
+	public required LegacyPageMapping? LegacyPage { get; init; }
+
+	public required IReadOnlyCollection<PageTocItem> PageTocItems { get; init; }
+
+	public required LayoutName? Layout { get; init; }
+}
+
+public record PageTocItem
+{
+	public required string Heading { get; init; }
+	public required string Slug { get; init; }
+	public required int Level { get; init; }
+}
+
diff --git a/src/Elastic.Markdown/Slices/_Layout.cshtml b/src/Elastic.Markdown/Slices/_Layout.cshtml
index 6ca8a80cc..4094c2275 100644
--- a/src/Elastic.Markdown/Slices/_Layout.cshtml
+++ b/src/Elastic.Markdown/Slices/_Layout.cshtml
@@ -1,37 +1,76 @@
-@inherits RazorLayoutSlice<GlobalLayoutViewModel>
+@inherits RazorLayoutSlice<MarkdownLayoutViewModel>
 @implements IUsesLayout<Elastic.Documentation.Site._GlobalLayout, GlobalLayoutViewModel>
 @functions {
 	public GlobalLayoutViewModel LayoutModel => Model;
-}
-			<div class="max-w-(--max-layout-width) w-full h-full grid
+	protected override Task ExecuteSectionAsync(string name)
+	{
+		if (name == GlobalSections.Footer && Model.Layout is not LayoutName.Archive)
+		{
+			<aside id="dismissible-banner" class="admonition tip">
+				<div class="container flex justify-between items-center mx-auto">
+					<p>
+						Welcome to the docs for the <a class="link text-base" href="/docs/get-started/versioning-availability#find-docs-for-your-product-version">latest Elastic product versions</a>, including Elastic Stack 9.0 and Elastic Cloud Serverless.
+						To view previous versions, go to <a class="link text-base" target="_blank" href="https://elastic.co/guide">elastic.co/guide</a>.
+					</p>
+					<button id="dismissible-button">
+						<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="size-6">
+							<path stroke-linecap="round" stroke-linejoin="round" d="M6 18 18 6M6 6l12 12"/>
+						</svg>
+					</button>
+				</div>
+			</aside>
+		}
+		return Task.CompletedTask;
+	}
+	private async Task RenderDefault()
+	{
+		<div class="max-w-(--max-layout-width) w-full h-full grid
 					grid-cols-1
 					md:grid-cols-[var(--max-sidebar-width)_1fr]
 					lg:grid-cols-[var(--max-sidebar-width)_1fr_var(--max-sidebar-width)]
 				">
-				@await RenderPartialAsync(_PagesNav.Create(Model))
-				<div class="justify-center grid
+			@await RenderPartialAsync(_PagesNav.Create<GlobalLayoutViewModel>(Model))
+			<div class="justify-center grid
 					grid-cols-1
 					px-6 lg:px-0
 					lg:grid-cols-[1fr_var(--max-text-width)_1fr]
 					">
-					<div class="spacer"></div>
-					<main id="content-container" class="w-full flex flex-col relative pb-30 overflow-x-hidden">
-						<div class="w-full absolute top-0 left-0 right-0 htmx-indicator" id="htmx-indicator" role="status">
-							<div class="h-[2px] w-full overflow-hidden">
-								<div class="progress w-full h-full bg-pink-70 left-right"></div>
-							</div>
-							<div class="sr-only">Loading</div>
-						</div>
-						<div class="content-container md:px-4">
-							@await RenderPartialAsync(Elastic.Markdown.Slices.Layout._Breadcrumbs.Create(Model))
+				<div class="spacer"></div>
+				<main id="content-container" class="w-full flex flex-col relative pb-30 overflow-x-hidden">
+					<div class="w-full absolute top-0 left-0 right-0 htmx-indicator" id="htmx-indicator" role="status">
+						<div class="h-[2px] w-full overflow-hidden">
+							<div class="progress w-full h-full bg-pink-70 left-right"></div>
 						</div>
-						<article id="markdown-content" class="content-container markdown-content md:px-4">
-							<input type="checkbox" class="hidden" id="pages-nav-hamburger">
-							@await RenderBodyAsync()
-						</article>
-						@await RenderPartialAsync(Elastic.Markdown.Slices.Layout._PrevNextNav.Create(Model))
-					</main>
-					<div class="spacer"></div>
-				</div>
-				@await RenderPartialAsync(Elastic.Markdown.Slices.Layout._TableOfContents.Create(Model))
+						<div class="sr-only">Loading</div>
+					</div>
+					<div class="content-container md:px-4">
+						@await RenderPartialAsync(_Breadcrumbs.Create(Model))
+					</div>
+					<article id="markdown-content" class="content-container markdown-content md:px-4">
+						<input type="checkbox" class="hidden" id="pages-nav-hamburger">
+						@await RenderBodyAsync()
+					</article>
+					@await RenderPartialAsync(_PrevNextNav.Create(Model))
+				</main>
+				<div class="spacer"></div>
 			</div>
+			@await RenderPartialAsync(_TableOfContents.Create(Model))
+		</div>
+	}
+}
+
+@switch (Model.Layout)
+{
+	case LayoutName.NotFound:
+		await RenderPartialAsync(_NotFound.Create(Model));
+		break;
+	case LayoutName.LandingPage:
+		await RenderPartialAsync(_LandingPage.Create(Model));
+		break;
+	case LayoutName.Archive:
+		await RenderPartialAsync(_Archive.Create(Model));
+		break;
+	default:
+		await RenderDefault();
+		break;
+}
diff --git a/src/Elastic.Markdown/Slices/_ViewImports.cshtml b/src/Elastic.Markdown/Slices/_ViewImports.cshtml
index e9cd9ad8c..b92793cb6 100644
--- a/src/Elastic.Markdown/Slices/_ViewImports.cshtml
+++ b/src/Elastic.Markdown/Slices/_ViewImports.cshtml
@@ -6,6 +6,7 @@
 @using Microsoft.AspNetCore.Http.HttpResults
 @using RazorSlices
 @using Elastic.Markdown.Slices
+@using Elastic.Markdown.Slices.Layout
 @using Elastic.Markdown.Slices.Directives
 @using Elastic.Documentation.Site.Layout
 @using Elastic.Documentation.Site
diff --git a/src/authoring/Elastic.Documentation.Refactor/Move.cs b/src/authoring/Elastic.Documentation.Refactor/Move.cs
index 1d6ff9bd3..97fb0ad41 100644
--- a/src/authoring/Elastic.Documentation.Refactor/Move.cs
+++ b/src/authoring/Elastic.Documentation.Refactor/Move.cs
@@ -90,7 +90,7 @@ private async Task SetupChanges(ChangeSet changeSet, Cancel ctx)
 
 		_changes[changeSet] = [new Change(changeSet.From, sourceContent, change)];
 
-		foreach (var (_, markdownFile) in documentationSet.MarkdownFiles)
+		foreach (var markdownFile in documentationSet.MarkdownFiles)
 		{
 			await ProcessMarkdownFile(
 				changeSet,
diff --git a/src/tooling/docs-assembler/AssembleSources.cs b/src/tooling/docs-assembler/AssembleSources.cs
index 3f53d0a0b..da15fa044 100644
--- a/src/tooling/docs-assembler/AssembleSources.cs
+++ b/src/tooling/docs-assembler/AssembleSources.cs
@@ -14,6 +14,7 @@
 using Elastic.Documentation.LinkIndex;
 using Elastic.Markdown.IO.Navigation;
 using Elastic.Markdown.Links.CrossLinks;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using YamlDotNet.RepresentationModel;
 
@@ -48,26 +49,26 @@ public class AssembleSources
 
 	public PublishEnvironmentUriResolver UriResolver { get; }
 
-	public static async Task<AssembleSources> AssembleAsync(AssembleContext context, Checkout[] checkouts, Cancel ctx)
+	public static async Task<AssembleSources> AssembleAsync(ILoggerFactory logger, AssembleContext context, Checkout[] checkouts, Cancel ctx)
 	{
-		var sources = new AssembleSources(context, checkouts);
+		var sources = new AssembleSources(logger, context, checkouts);
 		foreach (var (_, set) in sources.AssembleSets)
 			await set.DocumentationSet.ResolveDirectoryTree(ctx);
 		return sources;
 	}
 
-	private AssembleSources(AssembleContext assembleContext, Checkout[] checkouts)
+	private AssembleSources(ILoggerFactory logger, AssembleContext assembleContext, Checkout[] checkouts)
 	{
 		AssembleContext = assembleContext;
 		TocTopLevelMappings = GetConfiguredSources(assembleContext);
 		HistoryMappings = GetHistoryMapping(assembleContext);
 		var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous();
-		var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration, assembleContext.Environment, linkIndexProvider);
+		var crossLinkFetcher = new AssemblerCrossLinkFetcher(logger, assembleContext.Configuration, assembleContext.Environment, linkIndexProvider);
 		UriResolver = new PublishEnvironmentUriResolver(TocTopLevelMappings, assembleContext.Environment);
 		var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, UriResolver);
 		AssembleSets = checkouts
 			.Where(c => c.Repository is { Skip: false })
-			.Select(c => new AssemblerDocumentationSet(NullLoggerFactory.Instance, assembleContext, c, crossLinkResolver, TreeCollector))
+			.Select(c => new AssemblerDocumentationSet(logger, assembleContext, c, crossLinkResolver, TreeCollector))
 			.ToDictionary(s => s.Checkout.Repository.Name, s => s)
 			.ToFrozenDictionary();
 
diff --git a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs
index 735400030..cdc5f25b7 100644
--- a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs
+++ b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs
@@ -25,20 +25,19 @@ namespace Documentation.Assembler.Cli;
 
 internal sealed class RepositoryCommands(ICoreService githubActionsService, ILoggerFactory logger)
 {
+	private readonly ILogger<Program> _log = logger.CreateLogger<Program>();
+
 	[SuppressMessage("Usage", "CA2254:Template should be a static expression")]
 	private void AssignOutputLogger()
 	{
-		var log = logger.CreateLogger<Program>();
-		ConsoleApp.Log = msg => log.LogInformation(msg);
-		ConsoleApp.LogError = msg => log.LogError(msg);
+		ConsoleApp.Log = msg => _log.LogInformation(msg);
+		ConsoleApp.LogError = msg => _log.LogError(msg);
 	}
 
-	// would love to use libgit2 so there is no git dependency but
-	// libgit2 is magnitudes slower to clone repositories https://github.com/libgit2/libgit2/issues/4674
 	/// <summary> Clones all repositories </summary>
 	/// <param name="strict"> Treat warnings as errors and fail the build on warnings</param>
 	/// <param name="environment"> The environment to build</param>
-	/// <param name="fetchLatest"> If true fetch the latest commit of the branch instead of the link registry entry ref</param>
+	/// <param name="fetchLatest"> If true, fetch the latest commit of the branch instead of the link registry entry ref</param>
 	/// <param name="ctx"></param>
 	[Command("clone-all")]
 	public async Task<int> CloneAll(
@@ -88,23 +87,30 @@ public async Task<int> BuildAll(
 		var githubEnvironmentInput = githubActionsService.GetInput("environment");
 		environment ??= !string.IsNullOrEmpty(githubEnvironmentInput) ? githubEnvironmentInput : "dev";
 
+		_log.LogInformation("Building all repositories for environment {Environment}", environment);
+
 		await using var collector = new ConsoleDiagnosticsCollector(logger, githubActionsService)
 		{
 			NoHints = true
 		}.StartAsync(ctx);
 
+		_log.LogInformation("Creating assemble context");
+
 		var assembleContext = new AssembleContext(environment, collector, new FileSystem(), new FileSystem(), null, null)
 		{
 			Force = force ?? false,
 			AllowIndexing = allowIndexing ?? false
 		};
 
+		_log.LogInformation("Validating navigation.yml does not contain colliding path prefixes");
 		// this validates all path prefixes are unique, early exit if duplicates are detected
 		if (!GlobalNavigationFile.ValidatePathPrefixes(assembleContext) || assembleContext.Collector.Errors > 0)
 		{
 			await assembleContext.Collector.StopAsync(ctx);
 			return 1;
 		}
+
+		_log.LogInformation("Get all clone directory information");
 		var cloner = new AssemblerRepositorySourcer(logger, assembleContext);
 		var checkoutResult = cloner.GetAll();
 		var checkouts = checkoutResult.Checkouts.ToArray();
@@ -112,9 +118,11 @@ public async Task<int> BuildAll(
 		if (checkouts.Length == 0)
 			throw new Exception("No checkouts found");
 
-		var assembleSources = await AssembleSources.AssembleAsync(assembleContext, checkouts, ctx);
+		_log.LogInformation("Preparing all assemble sources for build");
+		var assembleSources = await AssembleSources.AssembleAsync(logger, assembleContext, checkouts, ctx);
 		var navigationFile = new GlobalNavigationFile(assembleContext, assembleSources);
 
+		_log.LogInformation("Create global navigation");
 		var navigation = new GlobalNavigation(assembleSources, navigationFile);
 
 		var pathProvider = new GlobalNavigationPathProvider(navigationFile, assembleSources, assembleContext);
diff --git a/src/tooling/docs-assembler/Navigation/GlobalNavigation.cs b/src/tooling/docs-assembler/Navigation/GlobalNavigation.cs
index 401891497..14b921ff0 100644
--- a/src/tooling/docs-assembler/Navigation/GlobalNavigation.cs
+++ b/src/tooling/docs-assembler/Navigation/GlobalNavigation.cs
@@ -23,25 +23,20 @@ public record GlobalNavigation : IPositionalNavigation
 
 	public FrozenDictionary<string, INavigationItem> MarkdownNavigationLookup { get; }
 
-	public FrozenDictionary<int, MarkdownFile> MarkdownFiles { get; set; }
-
+	public FrozenDictionary<int, INavigationItem> NavigationIndexedByOrder { get; }
 
 	public GlobalNavigation(AssembleSources assembleSources, GlobalNavigationFile navigationFile)
 	{
 		_assembleSources = assembleSources;
 		_navigationFile = navigationFile;
-		NavigationItems = BuildNavigation(navigationFile.TableOfContents, 0);
+		NavigationItems = BuildNavigation(navigationFile.TableOfContents.Concat(navigationFile.Phantoms).ToArray(), 0);
 		var navigationIndex = 0;
-		var markdownFiles = new HashSet<MarkdownFile>();
-		UpdateNavigationIndex(markdownFiles, NavigationItems, null, ref navigationIndex);
-		TopLevelItems = NavigationItems.OfType<TableOfContentsTree>().ToList();
+		var allNavigationItems = new HashSet<INavigationItem>();
+		UpdateNavigationIndex(allNavigationItems, NavigationItems, null, ref navigationIndex);
+		TopLevelItems = NavigationItems.OfType<TableOfContentsTree>().Where(t => !t.Hidden).ToList();
 		NavigationLookup = TopLevelItems.ToDictionary(kv => kv.Source, kv => kv);
-		var grouped = markdownFiles.GroupBy(f => f.NavigationIndex).ToList();
-		var files = grouped
-			.Select(g => g.First())
-			.ToList();
 
-		MarkdownFiles = files.Where(f => f.NavigationIndex > -1).ToDictionary(i => i.NavigationIndex, i => i).ToFrozenDictionary();
+		NavigationIndexedByOrder = allNavigationItems.ToDictionary(i => i.NavigationIndex, i => i).ToFrozenDictionary();
 
 		MarkdownNavigationLookup = NavigationItems
 			.SelectMany(DocumentationSet.Pairs)
@@ -50,7 +45,7 @@ public GlobalNavigation(AssembleSources assembleSources, GlobalNavigationFile na
 	}
 
 	private void UpdateNavigationIndex(
-		HashSet<MarkdownFile> markdownFiles,
+		HashSet<INavigationItem> allNavigationItems,
 		IReadOnlyCollection<INavigationItem> navigationItems,
 		INodeNavigationItem<INavigationModel, INavigationItem>? parent,
 		ref int navigationIndex
@@ -62,18 +57,18 @@ ref int navigationIndex
 			{
 				case FileNavigationItem fileNavigationItem:
 					var fileIndex = Interlocked.Increment(ref navigationIndex);
-					fileNavigationItem.Model.NavigationIndex = fileIndex;
+					fileNavigationItem.NavigationIndex = fileIndex;
 					if (parent is not null)
 						fileNavigationItem.Parent = parent;
-					_ = markdownFiles.Add(fileNavigationItem.Model);
+					_ = allNavigationItems.Add(fileNavigationItem);
 					break;
 				case DocumentationGroup documentationGroup:
 					var groupIndex = Interlocked.Increment(ref navigationIndex);
-					documentationGroup.Index.NavigationIndex = groupIndex;
+					documentationGroup.NavigationIndex = groupIndex;
 					if (parent is not null)
 						documentationGroup.Parent = parent;
-					_ = markdownFiles.Add(documentationGroup.Index);
-					UpdateNavigationIndex(markdownFiles, documentationGroup.NavigationItems, documentationGroup, ref navigationIndex);
+					_ = allNavigationItems.Add(documentationGroup);
+					UpdateNavigationIndex(allNavigationItems, documentationGroup.NavigationItems, documentationGroup, ref navigationIndex);
 					break;
 				default:
 					_navigationFile.EmitError($"Unhandled navigation item type: {item.GetType()}");
@@ -89,43 +84,8 @@ private IReadOnlyCollection<INavigationItem> BuildNavigation(IReadOnlyCollection
 		{
 			if (!_assembleSources.TreeCollector.TryGetTableOfContentsTree(toc.Source, out var tree))
 			{
-				_navigationFile.EmitWarning($"No {nameof(TableOfContentsTree)} found for {toc.Source}");
-				if (!_assembleSources.TocTopLevelMappings.TryGetValue(toc.Source, out var topLevel))
-				{
-					_navigationFile.EmitError(
-						$"Can not create temporary {nameof(TableOfContentsTree)} for {toc.Source} since no top level source could be located for it"
-					);
-					continue;
-				}
-
-				// TODO passing DocumentationSet to TableOfContentsTree constructor is temporary
-				// We only build this fallback in order to aid with bootstrapping the navigation
-				if (!_assembleSources.TreeCollector.TryGetTableOfContentsTree(topLevel.TopLevelSource, out tree))
-				{
-					_navigationFile.EmitError(
-						$"Can not create temporary {nameof(TableOfContentsTree)} for {topLevel.TopLevelSource} since no top level source could be located for it"
-					);
-					continue;
-				}
-
-				var documentationSet = tree.DocumentationSet ?? (tree.Parent as TableOfContentsTree)?.DocumentationSet
-					?? throw new InvalidOperationException($"Can not fall back for {toc.Source} because no documentation set is available");
-
-				var lookups = new NavigationLookups
-				{
-					FlatMappedFiles = new Dictionary<string, DocumentationFile>().ToFrozenDictionary(),
-					TableOfContents = [],
-					EnabledExtensions = documentationSet.EnabledExtensions,
-					FilesGroupedByFolder = new Dictionary<string, DocumentationFile[]>().ToFrozenDictionary(),
-				};
-
-				var fileIndex = 0;
-				tree = new TableOfContentsTree(
-					documentationSet,
-					toc.Source,
-					documentationSet.Context,
-					lookups,
-					_assembleSources.TreeCollector, ref fileIndex);
+				_navigationFile.EmitError($"{toc.Source} does not define a toc.yml or docset.yml file");
+				continue;
 			}
 
 			var navigationItem = tree;
@@ -163,48 +123,11 @@ private IReadOnlyCollection<INavigationItem> BuildNavigation(IReadOnlyCollection
 
 			tree.NavigationItems = cleanNavigationItems.ToArray();
 			list.Add(navigationItem);
+
+			if (toc.IsPhantom)
+				navigationItem.Hidden = true;
 		}
 
 		return list.ToArray().AsReadOnly();
 	}
-
-	public INavigationItem? GetPrevious(MarkdownFile current)
-	{
-		var index = current.NavigationIndex;
-		do
-		{
-			var previous = MarkdownFiles.GetValueOrDefault(index - 1);
-			if (previous is null)
-				return null;
-			if (!previous.Hidden)
-			{
-				if (MarkdownNavigationLookup.TryGetValue(previous.CrossLink, out var navigationItem))
-					return navigationItem;
-			}
-			index--;
-		} while (index > 0);
-
-		return null;
-	}
-
-	public INavigationItem? GetNext(MarkdownFile current)
-	{
-		var index = current.NavigationIndex;
-		do
-		{
-			var previous = MarkdownFiles.GetValueOrDefault(index + 1);
-			if (previous is null)
-				return null;
-			if (!previous.Hidden)
-			{
-				if (MarkdownNavigationLookup.TryGetValue(previous.CrossLink, out var navigationItem))
-					return navigationItem;
-			}
-			index++;
-		} while (index <= MarkdownFiles.Count - 1);
-
-		return null;
-	}
-
-
 }
diff --git a/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs
index 7afb05de0..d78f4f070 100644
--- a/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs
+++ b/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs
@@ -210,7 +210,10 @@ private IReadOnlyCollection<TocReference> ReadChildren(string key, YamlStreamRea
 					if (source != null && !source.Contains("://"))
 						source = ContentSourceMoniker.CreateString(NarrativeRepository.RepositoryName, source);
 					var sourceUri = new Uri(source!);
-					var tocReference = new TocReference(sourceUri, this, "", []);
+					var tocReference = new TocReference(sourceUri, this, "", [])
+					{
+						IsPhantom = true
+					};
 					return tocReference;
 			}
 		}
diff --git a/src/tooling/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs b/src/tooling/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs
index d0f8d0f6a..2963a3309 100644
--- a/src/tooling/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs
+++ b/src/tooling/docs-assembler/Navigation/GlobalNavigationHtmlWriter.cs
@@ -46,6 +46,9 @@ private bool TryGetNavigationRoot(
 
 	public async Task<string> RenderNavigation(INodeNavigationItem<INavigationModel, INavigationItem> currentRootNavigation, Uri navigationSource, Cancel ctx = default)
 	{
+		if (Phantoms.Contains(navigationSource))
+			return string.Empty;
+
 		if (!TryGetNavigationRoot(navigationSource, out var navigationRoot, out var navigationRootSource))
 			return string.Empty;
 
diff --git a/src/tooling/docs-assembler/navigation.yml b/src/tooling/docs-assembler/navigation.yml
index ffd42a667..3469c04c7 100644
--- a/src/tooling/docs-assembler/navigation.yml
+++ b/src/tooling/docs-assembler/navigation.yml
@@ -11,6 +11,7 @@
 phantoms:
   - toc: elasticsearch://reference
   - toc: docs-content://release-notes
+  - toc: docs-content://
   - toc: cloud://release-notes
 
 toc:
diff --git a/src/tooling/docs-builder/Cli/Commands.cs b/src/tooling/docs-builder/Cli/Commands.cs
index 242e51a19..f4338a323 100644
--- a/src/tooling/docs-builder/Cli/Commands.cs
+++ b/src/tooling/docs-builder/Cli/Commands.cs
@@ -22,12 +22,12 @@ namespace Documentation.Builder.Cli;
 
 internal sealed class Commands(ILoggerFactory logger, ICoreService githubActionsService)
 {
+	private readonly ILogger<Program> _log = logger.CreateLogger<Program>();
 	[SuppressMessage("Usage", "CA2254:Template should be a static expression")]
 	private void AssignOutputLogger()
 	{
-		var log = logger.CreateLogger<Program>();
-		ConsoleApp.Log = msg => log.LogInformation(msg);
-		ConsoleApp.LogError = msg => log.LogError(msg);
+		ConsoleApp.Log = msg => _log.LogInformation(msg);
+		ConsoleApp.LogError = msg => _log.LogError(msg);
 	}
 
 	/// <summary>
@@ -45,6 +45,9 @@ public async Task Serve(string? path = null, int port = 3000, Cancel ctx = defau
 	{
 		AssignOutputLogger();
 		var host = new DocumentationWebHost(path, port, logger, new FileSystem(), new MockFileSystem());
+		_log.LogInformation("Find your documentation at http://localhost:{Port}/{Path}", port,
+			host.GeneratorState.Generator.DocumentationSet.FirstInterestingUrl.TrimStart('/')
+		);
 		await host.RunAsync(ctx);
 		await host.StopAsync(ctx);
 	}
@@ -162,7 +165,7 @@ public async Task<int> Generate(
 		await openApiGenerator.Generate(ctx);
 
 		if (runningOnCi)
-			await githubActionsService.SetOutputAsync("landing-page-path", set.MarkdownFiles.First().Value.Url);
+			await githubActionsService.SetOutputAsync("landing-page-path", set.FirstInterestingUrl);
 
 		await collector.StopAsync(ctx);
 		if (bool.TryParse(githubActionsService.GetInput("strict"), out var strictValue) && strictValue)
diff --git a/src/tooling/docs-builder/Http/DocumentationWebHost.cs b/src/tooling/docs-builder/Http/DocumentationWebHost.cs
index 4ac96aa57..721f0416b 100644
--- a/src/tooling/docs-builder/Http/DocumentationWebHost.cs
+++ b/src/tooling/docs-builder/Http/DocumentationWebHost.cs
@@ -24,7 +24,6 @@ public class DocumentationWebHost
 {
 	private readonly WebApplication _webApplication;
 
-	private readonly BuildContext _context;
 	private readonly IHostedService _hostedService;
 	private readonly IFileSystem _writeFileSystem;
 
@@ -44,18 +43,18 @@ public DocumentationWebHost(string? path, int port, ILoggerFactory logger, IFile
 		var hostUrl = $"http://localhost:{port}";
 
 		_hostedService = collector;
-		_context = new BuildContext(collector, readFs, writeFs, path, null)
+		Context = new BuildContext(collector, readFs, writeFs, path, null)
 		{
 			CanonicalBaseUrl = new Uri(hostUrl),
 		};
-		var generatorState = new ReloadableGeneratorState(_context.DocumentationSourceDirectory, _context.DocumentationOutputDirectory, _context, logger);
+		GeneratorState = new ReloadableGeneratorState(Context.DocumentationSourceDirectory, Context.DocumentationOutputDirectory, Context, logger);
 		_ = builder.Services
 			.AddAotLiveReload(s =>
 			{
-				s.FolderToMonitor = _context.DocumentationSourceDirectory.FullName;
+				s.FolderToMonitor = Context.DocumentationSourceDirectory.FullName;
 				s.ClientFileExtensions = ".md,.yml";
 			})
-			.AddSingleton<ReloadableGeneratorState>(_ => generatorState)
+			.AddSingleton<ReloadableGeneratorState>(_ => GeneratorState)
 			.AddHostedService<ReloadGeneratorService>();
 
 		if (IsDotNetWatchBuild())
@@ -67,6 +66,10 @@ public DocumentationWebHost(string? path, int port, ILoggerFactory logger, IFile
 		SetUpRoutes();
 	}
 
+	public ReloadableGeneratorState GeneratorState { get; }
+
+	public BuildContext Context { get; }
+
 	private static bool IsDotNetWatchBuild() => Environment.GetEnvironmentVariable("DOTNET_WATCH") is not null;
 
 	public async Task RunAsync(Cancel ctx)
@@ -141,7 +144,7 @@ await context.Response.WriteAsync(@"
 			.UseStaticFiles(
 				new StaticFileOptions
 				{
-					FileProvider = new EmbeddedOrPhysicalFileProvider(_context),
+					FileProvider = new EmbeddedOrPhysicalFileProvider(Context),
 					RequestPath = "/_static"
 				})
 			.UseRouting();
@@ -202,11 +205,14 @@ private static async Task<IResult> ServeDocumentationFile(ReloadableGeneratorSta
 				return Results.File(image.SourceFile.FullName, image.MimeType);
 			default:
 				if (s == "index.md")
-					return Results.Redirect(generator.DocumentationSet.MarkdownFiles.First().Value.Url);
+					return Results.Redirect(generator.DocumentationSet.MarkdownFiles.First().Url);
 
 				if (!generator.DocumentationSet.FlatMappedFiles.TryGetValue("404.md", out var notFoundDocumentationFile))
 					return Results.NotFound();
 
+				if (Path.GetExtension(s) is "" or not ".md")
+					return Results.NotFound();
+
 				var renderedNotFound = await generator.RenderLayout((notFoundDocumentationFile as MarkdownFile)!, ctx);
 				return Results.Content(renderedNotFound, "text/html", null, (int)HttpStatusCode.NotFound);
 		}
diff --git a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs
index c715c3b98..9d7117b90 100644
--- a/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs
+++ b/tests/docs-assembler.Tests/src/docs-assembler.Tests/GlobalNavigationTests.cs
@@ -12,6 +12,7 @@
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.Navigation;
 using FluentAssertions;
+using Microsoft.Extensions.Logging.Abstractions;
 
 namespace Documentation.Assembler.Tests;
 
@@ -61,7 +62,7 @@ private async Task<AssembleSources> Setup()
 			.ToArray();
 		var checkouts = repos.Select(r => CreateCheckout(FileSystem, r)).ToArray();
 
-		var assembleSources = await AssembleSources.AssembleAsync(Context, checkouts, TestContext.Current.CancellationToken);
+		var assembleSources = await AssembleSources.AssembleAsync(NullLoggerFactory.Instance, Context, checkouts, TestContext.Current.CancellationToken);
 		return assembleSources;
 	}
 
@@ -267,7 +268,7 @@ public async Task UriResolving()
 			.ToArray();
 		var checkouts = repos.Select(r => CreateCheckout(fs, r)).ToArray();
 
-		var assembleSources = await AssembleSources.AssembleAsync(assembleContext, checkouts, TestContext.Current.CancellationToken);
+		var assembleSources = await AssembleSources.AssembleAsync(NullLoggerFactory.Instance, assembleContext, checkouts, TestContext.Current.CancellationToken);
 		var globalNavigationFile = new GlobalNavigationFile(assembleContext, assembleSources);
 
 		globalNavigationFile.TableOfContents.Should().NotBeNull().And.NotBeEmpty();

From 8cc610bd72ad906fadbc29c8fdba1b6c7b98d8dc Mon Sep 17 00:00:00 2001
From: Martijn Laarman <Mpdreamz@gmail.com>
Date: Wed, 11 Jun 2025 20:10:25 +0200
Subject: [PATCH 2/4] Ensure links.json considers all navigation items

(cherry picked from commit 40f1332d45593f7c0bbbaf08ac540807ad5b5395)
---
 src/Elastic.Markdown/IO/DocumentationSet.cs | 38 +++++++++++++++------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs
index 8129c9477..23ef45b0e 100644
--- a/src/Elastic.Markdown/IO/DocumentationSet.cs
+++ b/src/Elastic.Markdown/IO/DocumentationSet.cs
@@ -380,18 +380,36 @@ public RepositoryLinks CreateLinkReference()
 	{
 		var redirects = Configuration.Redirects;
 		var crossLinks = Context.Collector.CrossLinks.ToHashSet().ToArray();
-		var links = NavigationIndexedByOrder
-			.Select(kv => kv.Value)
+		// reference/aggregations/metrics.md
+		var markdownInNavigation = NavigationIndexedByOrder.Values
 			.OfType<FileNavigationItem>()
-			.Select(m => (m.Model.LinkReferenceRelativePath, Navigation: m))
-			.ToDictionary(k => RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-			? k.LinkReferenceRelativePath.Replace('\\', '/')
-			: k.LinkReferenceRelativePath, v =>
+			.Select(m => (Markdown: m.Model, Navigation: (INavigationItem)m))
+			.Concat(NavigationIndexedByOrder.Values
+				.OfType<DocumentationGroup>()
+				.Select(g => (Markdown: g.Index, Navigation: (INavigationItem)g))
+			)
+			.ToList();
+
+		var links = markdownInNavigation
+			.Select(tuple =>
 			{
-				var md = v.Navigation.Model;
-				var anchors = md.Anchors.Count == 0 ? null : md.Anchors.ToArray();
-				return new LinkMetadata { Anchors = anchors, Hidden = v.Navigation.Hidden };
-			});
+				var path = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+					? tuple.Markdown.LinkReferenceRelativePath.Replace('\\', '/')
+					: tuple.Markdown.LinkReferenceRelativePath;
+				return (Path: path, tuple.Markdown, tuple.Navigation);
+			})
+			.DistinctBy(tuple => tuple.Path)
+			.ToDictionary(
+				tuple => tuple.Path,
+				tuple =>
+				{
+					var anchors = tuple.Markdown.Anchors.Count == 0 ? null : tuple.Markdown.Anchors.ToArray();
+					return new LinkMetadata
+					{
+						Anchors = anchors,
+						Hidden = tuple.Navigation.Hidden
+					};
+				});
 
 		return new RepositoryLinks
 		{

From a462b14d16d2242fecf613cb81277aef6ce8c2fa Mon Sep 17 00:00:00 2001
From: Martijn Laarman <Mpdreamz@gmail.com>
Date: Wed, 11 Jun 2025 20:15:06 +0200
Subject: [PATCH 3/4] remove commented line

---
 src/Elastic.Markdown/IO/DocumentationSet.cs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs
index 23ef45b0e..c8ab6c77a 100644
--- a/src/Elastic.Markdown/IO/DocumentationSet.cs
+++ b/src/Elastic.Markdown/IO/DocumentationSet.cs
@@ -380,7 +380,6 @@ public RepositoryLinks CreateLinkReference()
 	{
 		var redirects = Configuration.Redirects;
 		var crossLinks = Context.Collector.CrossLinks.ToHashSet().ToArray();
-		// reference/aggregations/metrics.md
 		var markdownInNavigation = NavigationIndexedByOrder.Values
 			.OfType<FileNavigationItem>()
 			.Select(m => (Markdown: m.Model, Navigation: (INavigationItem)m))

From 173e1eb56f07be173437e6ed2fd9fce3d02f3e77 Mon Sep 17 00:00:00 2001
From: Martijn Laarman <Mpdreamz@gmail.com>
Date: Wed, 11 Jun 2025 20:29:08 +0200
Subject: [PATCH 4/4] update test file output order

---
 tests/authoring/Generator/LinkReferenceFile.fs | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/authoring/Generator/LinkReferenceFile.fs b/tests/authoring/Generator/LinkReferenceFile.fs
index cf2f06f96..f8fbe3c24 100644
--- a/tests/authoring/Generator/LinkReferenceFile.fs
+++ b/tests/authoring/Generator/LinkReferenceFile.fs
@@ -64,13 +64,6 @@ Through various means $$$including-this-inline-syntax$$$
          "url_path_prefix": "",
          "links": {
            "file.md": {},
-           "index.md": {
-             "anchors": [
-               "including-this-inline-syntax",
-              "this-anchor-is-autogenerated",
-              "and-anchored"
-            ]
-           },
            "testing/redirects/5th-page.md": {
              "anchors": [ "bb" ]
            },
@@ -82,6 +75,13 @@ Through various means $$$including-this-inline-syntax$$$
            },
            "testing/redirects/first-page.md": {
              "anchors": [ "has-an-anchor-as-well" ]
+           },
+           "index.md": {
+             "anchors": [
+               "including-this-inline-syntax",
+              "this-anchor-is-autogenerated",
+              "and-anchored"
+            ]
            }
          },
          "cross_links": [],