feat(cookbook): add /cookbook and export all cookbooks to it (langfuse#376)

Author: marcklingen

components/MainContentWrapper.tsx

@@ -18,19 +18,28 @@ import { ProductUpdateSignup } from "./productUpdateSignup";
 
 const pathsWithoutFooterWidgets = ["/imprint", "/blog"];
 
+const cleanedCookbookRoutes = cookbookRoutes.flatMap(
+  ({ notebook, destinations }) => {
+    return [
+      ...destinations,
+      "cookbook/" + notebook.replace(".ipynb", ".md"), // add cookbook path that all notebooks are published to
+    ].map((d) => ({
+      notebook,
+      destination: "/" + d,
+    }));
+  }
+);
+
 export const MainContentWrapper = (props) => {
   const router = useRouter();
-  const notebook = cookbookRoutes.find(
+  const notebook = cleanedCookbookRoutes.find(
     ({ destination }) => destination === router.pathname + ".md"
   );
 
   return (
     <>
       {notebook ? (
-        <NotebookBanner
-          src={notebook.source.replace(".md", ".ipynb")}
-          className="mb-4"
-        />
+        <NotebookBanner src={notebook.notebook} className="mb-4" />
       ) : null}
       {props.children}
       {!pathsWithoutFooterWidgets.includes(router.pathname) ? (

components/NotebookBanner.tsx

@@ -11,7 +11,7 @@ export const NotebookBanner: React.FC<{ src: string; className?: string }> = ({
         <span>This is a Jupyter notebook</span>
         <div className="flex gap-2 flex-wrap">
           <a
-            href={`https://github.com/langfuse/langfuse-docs/blob/main/${src}`}
+            href={`https://github.com/langfuse/langfuse-docs/blob/main/cookbook/${src}`}
             target="_blank"
             rel="noopener noreferrer"
           >
@@ -20,7 +20,7 @@ export const NotebookBanner: React.FC<{ src: string; className?: string }> = ({
             </Button>
           </a>
           <a
-            href={`https://colab.research.google.com/github/langfuse/langfuse-docs/blob/main/${src}`}
+            href={`https://colab.research.google.com/github/langfuse/langfuse-docs/blob/main/cookbook/${src}`}
             target="_blank"
             rel="noopener noreferrer"
           >

cookbook/_routes.json

@@ -1,26 +1,30 @@
 [
   {
-    "source": "cookbook/datasets.md",
-    "destination": "/docs/datasets/python-cookbook.md"
+    "notebook": "datasets.ipynb",
+    "destinations": ["docs/datasets/python-cookbook.md"]
   },
   {
-    "source": "cookbook/integration_langchain.md",
-    "destination": "/docs/integrations/langchain/example-python.md"
+    "notebook": "integration_langchain.ipynb",
+    "destinations": ["docs/integrations/langchain/example-python.md"]
   },
   {
-    "source": "cookbook/integration_openai_sdk.md",
-    "destination": "/docs/integrations/openai.md"
+    "notebook": "integration_openai_sdk.ipynb",
+    "destinations": ["docs/integrations/openai.md"]
   },
   {
-    "source": "cookbook/evaluation_with_langchain.md",
-    "destination": "/docs/scores/model-based-evals/langchain.md"
+    "notebook": "evaluation_with_langchain.ipynb",
+    "destinations": ["docs/scores/model-based-evals/langchain.md"]
   },
   {
-    "source": "cookbook/python_sdk.md",
-    "destination": "/docs/sdk/python.md"
+    "notebook": "integration_azure_openai_langchain.ipynb",
+    "destinations": []
   },
   {
-    "source": "cookbook/evaluation_of_rag_with_ragas.md",
-    "destination": "/docs/scores/model-based-evals/ragas.md"
+    "notebook": "python_sdk.ipynb",
+    "destinations": ["docs/sdk/python.md"]
+  },
+  {
+    "notebook": "evaluation_of_rag_with_ragas.ipynb",
+    "destinations": ["docs/scores/model-based-evals/ragas.md"]
   }
 ]

pages/_meta.json

@@ -29,6 +29,11 @@
     "type": "page",
     "title": "Docs"
   },
+  "cookbook": {
+    "type": "page",
+    "title": "Cookbook",
+    "display": "hidden"
+  },
   "pricing": {
     "title": "Pricing",
     "type": "page",

pages/cookbook/_meta.json

@@ -0,0 +1,7 @@
+{
+  "index": "Overview",
+  "-- Cookbook": {
+    "type": "separator",
+    "title": "Cookbook"
+  }
+}

pages/cookbook/datasets.md

@@ -0,0 +1,239 @@
+# Langfuse Datasets Cookbook
+
+In this cookbook, we'll iterate on systems prompts with the goal of getting only the capital of a given country. We use Langfuse datasets, to store a list of example inputs and expected outputs.
+
+This is a very simple example, you can run experiments on any LLM application that you either trace with the [Langfuse SDKs](https://langfuse.com/docs/sdk) (Python, JS/TS) or via one of our [integrations](https://langfuse.com/docs/integrations) (e.g. Langchain).
+
+_Simple example application_
+
+- **Model**: gpt-3.5-turbo
+- **Input**: country name
+- **Output**: capital
+- **Evaluation**: exact match of completion and ground truth
+- **Experiment on**: system prompt
+
+## Setup
+
+
+```python
+%pip install langfuse openai langchain --upgrade
+```
+
+
+```python
+import os
+
+# get keys for your project from https://cloud.langfuse.com
+os.environ["LANGFUSE_PUBLIC_KEY"] = ""
+os.environ["LANGFUSE_SECRET_KEY"] = ""
+
+# your openai key
+os.environ["OPENAI_API_KEY"] = ""
+
+# Your host, defaults to https://cloud.langfuse.com
+# For US data region, set to "https://us.cloud.langfuse.com"
+# os.environ["LANGFUSE_HOST"] = "http://localhost:3000"
+```
+
+
+```python
+# import
+from langfuse import Langfuse
+import openai
+
+# init
+langfuse = Langfuse()
+```
+
+## Create a dataset
+
+
+```python
+langfuse.create_dataset(name="capital_cities");
+```
+
+### Items
+
+Load local items into the Langfuse dataset. Alternatively you can add items from production via the Langfuse UI.
+
+
+```python
+# example items, could also be json instead of strings
+local_items = [
+    {"input": {"country": "Italy"}, "expected_output": "Rome"},
+    {"input": {"country": "Spain"}, "expected_output": "Madrid"},
+    {"input": {"country": "Brazil"}, "expected_output": "Brasília"},
+    {"input": {"country": "Japan"}, "expected_output": "Tokyo"},
+    {"input": {"country": "India"}, "expected_output": "New Delhi"},
+    {"input": {"country": "Canada"}, "expected_output": "Ottawa"},
+    {"input": {"country": "South Korea"}, "expected_output": "Seoul"},
+    {"input": {"country": "Argentina"}, "expected_output": "Buenos Aires"},
+    {"input": {"country": "South Africa"}, "expected_output": "Pretoria"},
+    {"input": {"country": "Egypt"}, "expected_output": "Cairo"},
+]
+```
+
+
+```python
+# Upload to Langfuse
+for item in local_items:
+  langfuse.create_dataset_item(
+      dataset_name="capital_cities",
+      # any python object or value
+      input=item["input"],
+      # any python object or value, optional
+      expected_output=item["expected_output"]
+)
+```
+
+## Define application and run experiments
+
+We implement the application in two ways to demonstrate how it's done
+
+1. Custom LLM app using e.g. OpenAI SDK, traced with Langfuse Python SDK
+2. Langchain Application, traced via native Langfuse integration
+
+
+```python
+# we use a very simple eval here, you can use any eval library
+# see https://langfuse.com/docs/scores/model-based-evals for details
+def simple_evaluation(output, expected_output):
+  return output == expected_output
+```
+
+### Custom app
+
+
+```python
+from datetime import datetime
+
+def run_my_custom_llm_app(input, system_prompt):
+  messages = [
+      {"role":"system", "content": system_prompt},
+      {"role":"user", "content": input["country"]}
+  ]
+
+  generationStartTime = datetime.now()
+
+  openai_completion = openai.chat.completions.create(
+      model="gpt-3.5-turbo",
+      messages=messages
+  ).choices[0].message.content
+
+  langfuse_generation = langfuse.generation(
+    name="guess-countries",
+    input=messages,
+    output=openai_completion,
+    model="gpt-3.5-turbo",
+    start_time=generationStartTime,
+    end_time=datetime.now()
+  )
+
+  return openai_completion, langfuse_generation
+```
+
+
+```python
+def run_experiment(experiment_name, system_prompt):
+  dataset = langfuse.get_dataset("capital_cities")
+
+  for item in dataset.items:
+    completion, langfuse_generation = run_my_custom_llm_app(item.input, system_prompt)
+
+    item.link(langfuse_generation, experiment_name) # pass the observation/generation object or the id
+
+    langfuse_generation.score(
+      name="exact_match",
+      value=simple_evaluation(completion, item.expected_output)
+    )
+```
+
+
+```python
+run_experiment(
+    "famous_city",
+    "The user will input countries, respond with the most famous city in this country"
+)
+run_experiment(
+    "directly_ask",
+    "What is the capital of the following country?"
+)
+run_experiment(
+    "asking_specifically",
+    "The user will input countries, respond with only the name of the capital"
+)
+run_experiment(
+    "asking_specifically_2nd_try",
+    "The user will input countries, respond with only the name of the capital. State only the name of the city."
+)
+```
+
+### Langchain application
+
+
+```python
+from datetime import datetime
+from langchain.chat_models import ChatOpenAI
+from langchain.chains import LLMChain
+from langchain.schema import AIMessage, HumanMessage, SystemMessage
+
+def run_my_langchain_llm_app(input, system_message, callback_handler):
+
+  # needs to include {country}
+  messages = [
+    SystemMessage(
+      content=system_message
+    ),
+    HumanMessage(
+      content=input
+    ),
+  ]
+  chat = ChatOpenAI(callbacks=[callback_handler])
+  completion = chat(messages)
+
+  return completion.content
+```
+
+
+```python
+def run_langchain_experiment(experiment_name, system_message):
+  dataset = langfuse.get_dataset("capital_cities")
+
+  for item in dataset.items:
+    handler = item.get_langchain_handler(run_name=experiment_name)
+
+    completion = run_my_langchain_llm_app(item.input["country"], system_message, handler)
+
+    handler.root_span.score(
+      name="exact_match",
+      value=simple_evaluation(completion, item.expected_output)
+    )
+```
+
+
+```python
+run_langchain_experiment(
+    "langchain_famous_city",
+    "The user will input countries, respond with the most famous city in this country"
+)
+run_langchain_experiment(
+    "langchain_directly_ask",
+    "What is the capital of the following country?"
+)
+run_langchain_experiment(
+    "langchain_asking_specifically",
+    "The user will input countries, respond with only the name of the capital"
+)
+run_langchain_experiment(
+    "langchain_asking_specifically_2nd_try",
+    "The user will input countries, respond with only the name of the capital. State only the name of the city."
+)
+```
+
+## Evaluate experiments in Langfuse UI
+
+- Average scores per experiment run
+- Browse each run for an individual item
+- Look at traces to debug issues
+
+![Experiment runs in Langfuse](https://langfuse.com/images/docs/dataset-runs-cookbook.jpg)