From f74151b4e4f3a44c8e51ef251652c5c64b1ea7cd Mon Sep 17 00:00:00 2001
From: Nuno Campos <nuno@langchain.dev>
Date: Thu, 28 Dec 2023 14:48:53 -0800
Subject: [PATCH] Make all json parsing less strict by default (#15287)

- Enables strict=False by default
- Uses partial json recovery logic by default

<!-- Thank you for contributing to LangChain!

Please title your PR "<package>: <description>", where <package> is
whichever of langchain, community, core, experimental, etc. is being
modified.

Replace this entire comment with:
  - **Description:** a description of the change,
  - **Issue:** the issue # it fixes if applicable,
  - **Dependencies:** any dependencies required for this change,
- **Twitter handle:** we announce bigger features on Twitter. If your PR
gets announced, and you'd like a mention, we'll gladly shout you out!

Please make sure your PR is passing linting and testing before
submitting. Run `make format`, `make lint` and `make test` from the root
of the package you've modified to check this locally.

See contribution guidelines for more information on how to write/run
tests, lint, etc: https://python.langchain.com/docs/contributing/

If you're adding a new integration, please include:
1. a test for the integration, preferably unit tests that do not rely on
network access,
2. an example notebook showing its use. It lives in
`docs/docs/integrations` directory.

If no one reviews your PR within a few days, please @-mention one of
@baskaryan, @eyurtsev, @hwchase17.
 -->
---
 libs/core/langchain_core/output_parsers/json.py     | 4 ++--
 libs/langchain/langchain/evaluation/parsing/base.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/libs/core/langchain_core/output_parsers/json.py b/libs/core/langchain_core/output_parsers/json.py
index 14d70a5c7b42be..caed447068f451 100644
--- a/libs/core/langchain_core/output_parsers/json.py
+++ b/libs/core/langchain_core/output_parsers/json.py
@@ -114,7 +114,7 @@ def parse_partial_json(s: str, *, strict: bool = False) -> Any:
 
 
 def parse_json_markdown(
-    json_string: str, *, parser: Callable[[str], Any] = json.loads
+    json_string: str, *, parser: Callable[[str], Any] = parse_partial_json
 ) -> dict:
     """
     Parse a JSON string from a Markdown string.
@@ -190,7 +190,7 @@ def _diff(self, prev: Optional[Any], next: Any) -> Any:
     def parse(self, text: str) -> Any:
         text = text.strip()
         try:
-            return parse_json_markdown(text.strip(), parser=parse_partial_json)
+            return parse_json_markdown(text.strip())
         except JSONDecodeError as e:
             raise OutputParserException(f"Invalid json output: {text}") from e
 
diff --git a/libs/langchain/langchain/evaluation/parsing/base.py b/libs/langchain/langchain/evaluation/parsing/base.py
index c3a28f7ce586f2..6a5a49ef45dbad 100644
--- a/libs/langchain/langchain/evaluation/parsing/base.py
+++ b/libs/langchain/langchain/evaluation/parsing/base.py
@@ -1,4 +1,5 @@
 """Evaluators for parsing strings."""
+import json
 from operator import eq
 from typing import Any, Callable, Optional, Union, cast
 
@@ -68,7 +69,7 @@ def _evaluate_strings(
 
         """
         try:
-            parse_json_markdown(prediction)
+            parse_json_markdown(prediction, parser=json.loads)
             return {"score": 1}
         except Exception as e:
             return {"score": 0, "reasoning": str(e)}