chore(ci): Fix tests for new envs (#118)

* Restrict permissions

* More perms

* Fix tests for new env

* Fix

Author: jacoblee93

.github/workflows/integration_tests.yml

@@ -9,6 +9,10 @@ on:
       - main
   workflow_dispatch:
 
+permissions:
+  contents: read
+  pull-requests: read
+
 jobs:
   changed_files:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -27,6 +27,9 @@ on:
         default: false
         description: "Release from a non-main branch (danger!)"
 
+permissions:
+  contents: read
+
 env:
   PYTHON_VERSION: "3.11"
   UV_FROZEN: "true"

.github/workflows/release_js.yml

@@ -14,6 +14,8 @@ on:
         default: false
         description: "Release from a non-main branch (danger!)"
 
+permissions:
+  contents: read
 
 jobs:
   if_release:

js/src/string/tests/embedding_similarity.test.ts

@@ -37,25 +37,4 @@ ls.describe("Embedding Similarity Tests", () => {
       expect(res.score).toBeLessThan(1.0);
     }
   );
-
-  ls.test(
-    "test works with evaluate",
-    {
-      inputs: { dataset: "exact match" },
-    },
-    async ({ inputs }) => {
-      const evaluator = createEmbeddingSimilarityEvaluator({
-        embeddings: new OpenAIEmbeddings({ model: "text-embedding-3-small" }),
-      });
-      const result = await evaluate((inputs) => inputs, {
-        data: inputs.dataset,
-        evaluators: [evaluator],
-      });
-      expect(result).toBeDefined();
-      expect(result.results.length).toBeGreaterThan(0);
-      expect(
-        result.results[0].evaluationResults.results[0].score
-      ).toBeDefined();
-    }
-  );
 });

js/src/string/tests/levenshtein.test.ts

@@ -22,23 +22,4 @@ ls.describe("Levenshtein Distance Tests", () => {
     expect(res.key).toBe("levenshtein_distance");
     expect(res.score).toBeLessThan(1.0);
   });
-
-  ls.test(
-    "test works with evaluate",
-    {
-      inputs: { dataset: "exact match" },
-    },
-    async ({ inputs }) => {
-      const evaluator = levenshteinDistance;
-      const result = await evaluate((inputs) => inputs, {
-        data: inputs.dataset,
-        evaluators: [evaluator],
-      });
-      expect(result).toBeDefined();
-      expect(result.results.length).toBeGreaterThan(0);
-      expect(
-        result.results[0].evaluationResults.results[0].score
-      ).toBeDefined();
-    }
-  );
 });

js/src/tests/exact.test.ts

@@ -49,14 +49,3 @@ ls.describe("exact match", () => {
     }
   );
 });
-
-test("test works with evaluate", async () => {
-  const evaluator = exactMatch;
-  const result = await evaluate((inputs) => inputs, {
-    data: "exact match",
-    evaluators: [evaluator],
-  });
-  expect(result).toBeDefined();
-  expect(result.results.length).toBeGreaterThan(0);
-  expect(result.results[0].evaluationResults.results[0].score).toBeDefined();
-});

js/src/tests/json.test.ts

@@ -983,19 +983,6 @@ ls.describe("json", () => {
     }
   );
 
-  test("test json works with evaluate", async () => {
-    const evaluator = createJsonMatchEvaluator({
-      aggregator: "average",
-    });
-    const result = await evaluate((inputs) => inputs, {
-      data: "exact match",
-      evaluators: [evaluator],
-    });
-    expect(result).toBeDefined();
-    expect(result.results.length).toBeGreaterThan(0);
-    expect(result.results[0].evaluationResults.results[0].score).toBeDefined();
-  });
-
   ls.test(
     "test json throws error no rubric",
     {

js/src/tests/llm.test.ts

@@ -1,6 +1,5 @@
 import * as ls from "langsmith/vitest";
-import { expect, test, expectTypeOf } from "vitest";
-import { evaluate } from "langsmith/evaluation";
+import { expect, expectTypeOf, beforeAll } from "vitest";
 import { OpenAI } from "openai";
 import { ChatOpenAI } from "@langchain/openai";
 
@@ -9,9 +8,89 @@ import * as hub from "langchain/hub";
 import { HumanMessage } from "@langchain/core/messages";
 
 import { z } from "zod";
-import { ChatPromptTemplate } from "@langchain/core/prompts";
+import { ChatPromptTemplate, HumanMessagePromptTemplate, StructuredPrompt } from "@langchain/core/prompts";
+import { Client } from "langsmith";
 
 ls.describe("llm as judge", () => {
+  beforeAll(async () => {
+    // Setup required prompts in LangChain Hub before running tests
+    const client = new Client();
+
+    // Create test-equality prompt
+    const testEqualityPrompt = ChatPromptTemplate.fromMessages([
+      ["system", "You are an expert LLM as judge."],
+      ["human", "Are these two equal? {inputs} {outputs}"],
+    ]);
+
+    try {
+      await client.pushPrompt("test-equality", { object: testEqualityPrompt });
+      console.log("Created test-equality prompt");
+    } catch (error) {
+      console.log(`test-equality prompt may already exist: ${error}`);
+    }
+
+    // Create equality-1-message prompt
+    const equality1MessagePrompt = ChatPromptTemplate.fromMessages([
+      ["human", "Are these two equal? {inputs} {outputs}"],
+    ]);
+
+    try {
+      await client.pushPrompt("equality-1-message", {
+        object: equality1MessagePrompt,
+      });
+      console.log("Created equality-1-message prompt");
+    } catch (error) {
+      console.log(`equality-1-message prompt may already exist: ${error}`);
+    }
+
+    // Create simple-equality-structured prompt
+    const structuredEqualityPrompt = new StructuredPrompt({
+      inputVariables: ["inputs", "outputs"],
+      promptMessages: [
+        HumanMessagePromptTemplate.fromTemplate(
+          `Are these equal?
+
+<item1>
+{inputs}
+</item1>
+
+<item2>
+{outputs}
+</item2>`,
+        ),
+      ],
+      schema: {
+        title: "score",
+        description: "Get a score",
+        type: "object",
+        properties: {
+          equality: {
+            type: "boolean",
+            description: "Whether the two items are equal",
+          },
+          justification: {
+            type: "string",
+            description: "Justification for your decision above",
+          },
+        },
+        required: ["equality", "justification"],
+        strict: true,
+        additionalProperties: false,
+      },
+    });
+
+    try {
+      await client.pushPrompt("simple-equality-structured", {
+        object: structuredEqualityPrompt,
+      });
+      console.log("Created simple-equality-structured prompt");
+    } catch (error) {
+      console.log(
+        `simple-equality-structured prompt may already exist: ${error}`
+      );
+    }
+  });
+
   ls.test(
     "prompt hub prompt",
     {
@@ -21,7 +100,7 @@ ls.describe("llm as judge", () => {
       const outputs = { a: 1, b: 2 };
       const client = new OpenAI();
       const evaluator = createLLMAsJudge({
-        prompt: await hub.pull("langchain-ai/equality-1-message"),
+        prompt: await hub.pull("equality-1-message"),
         judge: client,
         model: "openai:gpt-4o-mini",
       });
@@ -42,7 +121,7 @@ ls.describe("llm as judge", () => {
     async ({ inputs }) => {
       const outputs = { a: 1, b: 2 };
       const evaluator = createLLMAsJudge({
-        prompt: await hub.pull("jacob/simple-equality-structured"),
+        prompt: await hub.pull("simple-equality-structured"),
         model: "openai:gpt-4o-mini",
       });
       const result = await evaluator({ inputs, outputs });
@@ -313,19 +392,6 @@ ls.describe("llm as judge", () => {
     }
   );
 
-  test("test llm as judge works with evaluate", async () => {
-    const evaluator = createLLMAsJudge({
-      prompt: "Are these two foo? {inputs} {outputs}",
-      model: "openai:o3-mini",
-    });
-    const result = await evaluate((inputs) => inputs, {
-      data: "exact match",
-      evaluators: [evaluator],
-    });
-    expect(result).toBeDefined();
-    expect(result.results.length).toBeGreaterThan(0);
-  }, 60000);
-
   ls.test(
     "llm as judge with mustache prompt",
     {

python/tests/test_llm.py

@@ -11,6 +11,97 @@
 from langchain import hub as prompts
 from langchain_core.messages import HumanMessage
 from langchain_core.prompts.chat import ChatPromptTemplate
+from langchain_core.prompts.structured import StructuredPrompt
+
+
+@pytest.fixture(scope="session", autouse=True)
+def setup_prompts():
+    """Setup required prompts in LangChain Hub before running tests."""
+    client = Client()
+
+    # Create test-equality prompt
+    test_equality_prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                "You are an expert LLM as judge.",
+            ),
+            (
+                "human",
+                "Are these two equal? {inputs} {outputs}",
+            ),
+        ]
+    )
+
+    try:
+        client.push_prompt("test-equality", object=test_equality_prompt)
+        print("Created test-equality prompt")
+    except Exception as e:
+        print(f"test-equality prompt may already exist: {e}")
+
+    # Create equality-1-message prompt
+    equality_1_message_prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "human",
+                "Are these two equal? {inputs} {outputs}",
+            )
+        ]
+    )
+
+    try:
+        client.push_prompt("equality-1-message", object=equality_1_message_prompt)
+        print("Created equality-1-message prompt")
+    except Exception as e:
+        print(f"equality-1-message prompt may already exist: {e}")
+
+    # Create simple-equality-structured prompt
+    structured_equality_prompt = StructuredPrompt(
+        messages=[
+            (
+                "human",
+                """
+Are these equal?
+
+<item1>
+{inputs}
+</item1>
+
+<item2>
+{outputs}
+</item2>
+""",
+            ),
+        ],
+        schema={
+            "title": "score",
+            "description": "Get a score",
+            "type": "object",
+            "properties": {
+                "equality": {
+                    "type": "boolean",
+                    "description": "Whether the two items are equal",
+                },
+                "justification": {
+                    "type": "string",
+                    "description": "Justification for your decision above",
+                },
+            },
+            "required": ["equality", "justification"],
+            "strict": True,
+            "additionalProperties": False,
+        },
+    )
+
+    try:
+        client.push_prompt(
+            "simple-equality-structured", object=structured_equality_prompt
+        )
+        print("Created simple-equality-structured prompt")
+    except Exception as e:
+        print(f"simple-equality-structured prompt may already exist: {e}")
+
+    return True
 
 
 @pytest.mark.langsmith
@@ -19,7 +110,7 @@ def test_prompt_hub_works():
     outputs = {"a": 1, "b": 2}
     client = OpenAI()
     llm_as_judge = create_llm_as_judge(
-        prompt=prompts.pull("langchain-ai/test-equality"),
+        prompt=prompts.pull("test-equality"),
         judge=client,
         model="gpt-4o-mini",
     )
@@ -34,7 +125,7 @@ def test_prompt_hub_works_one_message():
     outputs = {"a": 1, "b": 2}
     client = OpenAI()
     llm_as_judge = create_llm_as_judge(
-        prompt=prompts.pull("langchain-ai/equality-1-message"),
+        prompt=prompts.pull("equality-1-message"),
         judge=client,
         model="gpt-4o-mini",
     )
@@ -48,7 +139,7 @@ def test_structured_prompt():
     inputs = {"a": 1, "b": 2}
     outputs = {"a": 1, "b": 2}
     client = Client()
-    prompt = client.pull_prompt("jacob/simple-equality-structured")
+    prompt = client.pull_prompt("simple-equality-structured")
     llm_as_judge = create_llm_as_judge(
         prompt=prompt,
         model="openai:gpt-4o-mini",
@@ -272,18 +363,6 @@ class EqualityResult(BaseModel):
     assert eval_result.justification is not None
 
 
-@pytest.mark.langsmith
-def test_llm_as_judge_with_evaluate():
-    client = Client()
-    evaluator = create_llm_as_judge(
-        prompt="Are these two equal? {inputs} {outputs}",
-        model="openai:gpt-4o-mini",
-    )
-    res = client.evaluate(lambda x: x, data="exact match", evaluators=[evaluator])
-    for r in res:
-        assert r["evaluation_results"]["results"][0].score is not None
-
-
 @pytest.mark.langsmith
 def test_llm_as_judge_mustache_prompt():
     inputs = {"a": 1, "b": 2}