From 5b6c7aba57f661ad36da912f4be52df0da500bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Messias=20Lima=20Pereira?= Date: Tue, 29 Oct 2024 19:42:12 -0300 Subject: [PATCH 1/9] Add Generate of LlamaIndex docs and models | Changes of evaluate to new Testset used of ragas --- src/ragas/integrations/llama_index.py | 30 +++---- src/ragas/testset/synthesizers/generate.py | 91 +++++++++++++++++++++- 2 files changed, 106 insertions(+), 15 deletions(-) diff --git a/src/ragas/integrations/llama_index.py b/src/ragas/integrations/llama_index.py index 3a5b6e8b5..1e35a1ad2 100644 --- a/src/ragas/integrations/llama_index.py +++ b/src/ragas/integrations/llama_index.py @@ -1,17 +1,17 @@ from __future__ import annotations import logging +import datasets import typing as t from uuid import uuid4 -from datasets import Dataset - from ragas.embeddings import LlamaIndexEmbeddingsWrapper from ragas.evaluation import evaluate as ragas_evaluate from ragas.exceptions import ExceptionInRunner from ragas.executor import Executor from ragas.llms import LlamaIndexLLMWrapper from ragas.run_config import RunConfig +from ragas.testset.synthesizers.testset_schema import Testset if t.TYPE_CHECKING: from llama_index.core.base.embeddings.base import ( @@ -28,7 +28,7 @@ def evaluate( query_engine, - dataset: Dataset, + dataset: Testset, metrics: list[Metric], llm: t.Optional[LlamaindexLLM] = None, embeddings: t.Optional[LlamaIndexEmbeddings] = None, @@ -41,15 +41,17 @@ def evaluate( # wrap llms and embeddings li_llm = None if llm is not None: - li_llm = LlamaIndexLLMWrapper(llm) + li_llm = LlamaIndexLLMWrapper(llm, run_config=run_config) li_embeddings = None if embeddings is not None: - li_embeddings = LlamaIndexEmbeddingsWrapper(embeddings) + li_embeddings = LlamaIndexEmbeddingsWrapper(embeddings, run_config=run_config) # validate and transform dataset if dataset is None: raise ValueError("Provide dataset!") + dataset = datasets.Dataset.from_list(dataset.to_hf_dataset()['eval_sample']) + exec = Executor( desc="Running Query Engine", keep_progress_bar=True, @@ -58,7 +60,7 @@ def evaluate( ) # get query - queries = dataset["question"] + queries = dataset["user_input"] for i, q in enumerate(queries): exec.submit(query_engine.aquery, q, name=f"query-{i}") @@ -76,19 +78,21 @@ def evaluate( contexts.append([n.node.text for n in r.source_nodes]) # create HF dataset - hf_dataset = Dataset.from_dict( + hf_dataset = datasets.Dataset.from_dict( { - "question": queries, - "contexts": contexts, - "answer": answers, + "user_input": queries, + "retrieved_contexts": contexts, + "response": answers, } ) - if "ground_truth" in dataset.column_names: + if "reference" in dataset.column_names: hf_dataset = hf_dataset.add_column( - name="ground_truth", - column=dataset["ground_truth"], + name="reference", + column=dataset["reference"], new_fingerprint=str(uuid4()), ) + + print(hf_dataset) results = ragas_evaluate( dataset=hf_dataset, diff --git a/src/ragas/testset/synthesizers/generate.py b/src/ragas/testset/synthesizers/generate.py index d36de35a3..78e918379 100644 --- a/src/ragas/testset/synthesizers/generate.py +++ b/src/ragas/testset/synthesizers/generate.py @@ -9,9 +9,9 @@ from ragas._analytics import TestsetGenerationEvent, track from ragas.callbacks import new_group from ragas.cost import TokenUsageParser -from ragas.embeddings.base import BaseRagasEmbeddings, LangchainEmbeddingsWrapper +from ragas.embeddings.base import BaseRagasEmbeddings, LangchainEmbeddingsWrapper, LlamaIndexEmbeddingsWrapper from ragas.executor import Executor -from ragas.llms import BaseRagasLLM, LangchainLLMWrapper +from ragas.llms import BaseRagasLLM, LangchainLLMWrapper, LlamaIndexLLMWrapper from ragas.run_config import RunConfig from ragas.testset.graph import KnowledgeGraph, Node, NodeType from ragas.testset.synthesizers import default_query_distribution @@ -25,6 +25,11 @@ from langchain_core.embeddings.embeddings import Embeddings as LangchainEmbeddings from langchain_core.language_models import BaseLanguageModel as LangchainLLM + from llama_index.core.base.llms.base import BaseLLM as LlamaIndexLLM + from llama_index.core.base.embeddings.base import BaseEmbedding as LlamaIndexEmbedding + + from llama_index.core.schema import Document as LlamaindexDocument + from ragas.embeddings.base import BaseRagasEmbeddings from ragas.llms.base import BaseRagasLLM from ragas.testset.synthesizers import QueryDistribution @@ -71,6 +76,20 @@ def from_langchain( knowledge_graph, ) + @classmethod + def from_llama_index( + cls, + llm: LlamaIndexLLM, + embedding_model: LlamaIndexEmbedding, + knowledge_graph: t.Optional[KnowledgeGraph] = None + ) -> "TestsetGenerator": + knowledge_graph = knowledge_graph or KnowledgeGraph() + return cls( + LlamaIndexLLMWrapper(llm), + LlamaIndexEmbeddingsWrapper(embedding_model), + knowledge_graph, + ) + def generate_with_langchain_docs( self, documents: t.Sequence[LCDocument], @@ -135,6 +154,71 @@ def generate_with_langchain_docs( raise_exceptions=raise_exceptions, ) + def generate_with_llamaindex_docs( + self, + documents: t.Sequence[LCDocument], + testset_size: int, + transforms: t.Optional[Transforms] = None, + transforms_llm: t.Optional[BaseRagasLLM] = None, + transforms_embedding_model: t.Optional[BaseRagasEmbeddings] = None, + query_distribution: t.Optional[QueryDistribution] = None, + run_config: t.Optional[RunConfig] = None, + callbacks: t.Optional[Callbacks] = None, + with_debugging_logs=False, + raise_exceptions: bool = True, + ): + """ + Generates an evaluation dataset based on given scenarios and parameters. + """ + + # force the user to provide an llm and embedding client to prevent use of default LLMs + if not self.llm and not transforms_llm: + raise ValueError( + """An llm client was not provided. + Provide an LLM on TestsetGenerator instantiation or as an argument for transforms_llm parameter. + Alternatively you can provide your own transforms through the `transforms` parameter.""" + ) + if not self.embedding_model and not transforms_embedding_model: + raise ValueError( + """An embedding client was not provided. + Provide an embedding model on TestsetGenerator instantiation or as an argument for transforms_llm parameter. + Alternatively you can provide your own transforms through the `transforms` parameter.""" + ) + + if not transforms: + transforms = default_transforms( + llm=transforms_llm or self.llm, + embedding_model=transforms_embedding_model or self.embedding_model, + ) + + # convert the documents to Ragas nodes + nodes = [] + for doc in documents: + if doc.text is not None and doc.text.strip() != "": + node = Node( + type=NodeType.DOCUMENT, + properties={ + "page_content": doc.text, + "document_metadata": doc.metadata, + }, + ) + nodes.append(node) + + kg = KnowledgeGraph(nodes=nodes) + + # apply transforms and update the knowledge graph + apply_transforms(kg, transforms, run_config) + self.knowledge_graph = kg + + return self.generate( + testset_size=testset_size, + query_distribution=query_distribution, + run_config=run_config, + callbacks=callbacks, + with_debugging_logs=with_debugging_logs, + raise_exceptions=raise_exceptions, + ) + def generate( self, testset_size: int, @@ -182,6 +266,9 @@ def generate( 4. Generate samples for each scenario. 5. Compile the results into an EvaluationDataset. """ + if run_config is not None: + self.llm.set_run_config(run_config) + query_distribution = query_distribution or default_query_distribution(self.llm) callbacks = callbacks or [] From e22679caec0f6a152aa266e5f5c8f3542b4d4942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Messias=20Lima=20Pereira?= Date: Tue, 29 Oct 2024 20:21:09 -0300 Subject: [PATCH 2/9] Update llama_index.py --- src/ragas/integrations/llama_index.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ragas/integrations/llama_index.py b/src/ragas/integrations/llama_index.py index 1e35a1ad2..cc1553d34 100644 --- a/src/ragas/integrations/llama_index.py +++ b/src/ragas/integrations/llama_index.py @@ -50,7 +50,7 @@ def evaluate( if dataset is None: raise ValueError("Provide dataset!") - dataset = datasets.Dataset.from_list(dataset.to_hf_dataset()['eval_sample']) + dataset = dataset.to_hf_dataset() exec = Executor( desc="Running Query Engine", @@ -92,8 +92,6 @@ def evaluate( new_fingerprint=str(uuid4()), ) - print(hf_dataset) - results = ragas_evaluate( dataset=hf_dataset, metrics=metrics, From 5d386ea24605008b6433f57afdce1b8ed292f3ed Mon Sep 17 00:00:00 2001 From: jjmachan Date: Thu, 31 Oct 2024 15:55:46 +0530 Subject: [PATCH 3/9] style: lint fixes --- src/ragas/evaluation.py | 2 +- src/ragas/integrations/llama_index.py | 7 ++++--- src/ragas/metrics/_factual_correctness.py | 2 -- src/ragas/metrics/_tool_call_accuracy.py | 4 +++- src/ragas/testset/synthesizers/generate.py | 19 +++++++++++-------- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/ragas/evaluation.py b/src/ragas/evaluation.py index 5a874762f..d681e7132 100644 --- a/src/ragas/evaluation.py +++ b/src/ragas/evaluation.py @@ -114,7 +114,7 @@ def evaluate( Returns ------- EvaluationResult - EvaluationResult object containing the scores of each metric. + EvaluationResult object containing the scores of each metric. You can use this do analysis later. Raises diff --git a/src/ragas/integrations/llama_index.py b/src/ragas/integrations/llama_index.py index cc1553d34..815feb5ad 100644 --- a/src/ragas/integrations/llama_index.py +++ b/src/ragas/integrations/llama_index.py @@ -1,10 +1,11 @@ from __future__ import annotations import logging -import datasets import typing as t from uuid import uuid4 +import datasets + from ragas.embeddings import LlamaIndexEmbeddingsWrapper from ragas.evaluation import evaluate as ragas_evaluate from ragas.exceptions import ExceptionInRunner @@ -51,7 +52,7 @@ def evaluate( raise ValueError("Provide dataset!") dataset = dataset.to_hf_dataset() - + exec = Executor( desc="Running Query Engine", keep_progress_bar=True, @@ -91,7 +92,7 @@ def evaluate( column=dataset["reference"], new_fingerprint=str(uuid4()), ) - + results = ragas_evaluate( dataset=hf_dataset, metrics=metrics, diff --git a/src/ragas/metrics/_factual_correctness.py b/src/ragas/metrics/_factual_correctness.py index 1d887cfab..3f027a4a4 100644 --- a/src/ragas/metrics/_factual_correctness.py +++ b/src/ragas/metrics/_factual_correctness.py @@ -271,7 +271,6 @@ async def _single_turn_ascore( reference_response = await self.verify_claims( premise=reference, hypothesis_list=response_claims, callbacks=callbacks ) - if self.mode != "precision": response_reference = await self.verify_claims( @@ -286,7 +285,6 @@ async def _single_turn_ascore( fn = sum(~response_reference) else: fn = 0 - if self.mode == "precision": score = tp / (tp + fp + 1e-8) diff --git a/src/ragas/metrics/_tool_call_accuracy.py b/src/ragas/metrics/_tool_call_accuracy.py index 8fe416d6a..b5a0e0dc5 100644 --- a/src/ragas/metrics/_tool_call_accuracy.py +++ b/src/ragas/metrics/_tool_call_accuracy.py @@ -61,7 +61,9 @@ def is_sequence_aligned( async def _multi_turn_ascore( self, sample: MultiTurnSample, callbacks: Callbacks ) -> float: - assert sample.reference_tool_calls is not None, "Reference tool calls is not set" + assert ( + sample.reference_tool_calls is not None + ), "Reference tool calls is not set" pred_tool_calls = [] for item in sample.user_input: diff --git a/src/ragas/testset/synthesizers/generate.py b/src/ragas/testset/synthesizers/generate.py index 78e918379..123222f95 100644 --- a/src/ragas/testset/synthesizers/generate.py +++ b/src/ragas/testset/synthesizers/generate.py @@ -9,7 +9,11 @@ from ragas._analytics import TestsetGenerationEvent, track from ragas.callbacks import new_group from ragas.cost import TokenUsageParser -from ragas.embeddings.base import BaseRagasEmbeddings, LangchainEmbeddingsWrapper, LlamaIndexEmbeddingsWrapper +from ragas.embeddings.base import ( + BaseRagasEmbeddings, + LangchainEmbeddingsWrapper, + LlamaIndexEmbeddingsWrapper, +) from ragas.executor import Executor from ragas.llms import BaseRagasLLM, LangchainLLMWrapper, LlamaIndexLLMWrapper from ragas.run_config import RunConfig @@ -24,12 +28,11 @@ from langchain_core.documents import Document as LCDocument from langchain_core.embeddings.embeddings import Embeddings as LangchainEmbeddings from langchain_core.language_models import BaseLanguageModel as LangchainLLM - + from llama_index.core.base.embeddings.base import ( + BaseEmbedding as LlamaIndexEmbedding, + ) from llama_index.core.base.llms.base import BaseLLM as LlamaIndexLLM - from llama_index.core.base.embeddings.base import BaseEmbedding as LlamaIndexEmbedding - from llama_index.core.schema import Document as LlamaindexDocument - from ragas.embeddings.base import BaseRagasEmbeddings from ragas.llms.base import BaseRagasLLM from ragas.testset.synthesizers import QueryDistribution @@ -81,7 +84,7 @@ def from_llama_index( cls, llm: LlamaIndexLLM, embedding_model: LlamaIndexEmbedding, - knowledge_graph: t.Optional[KnowledgeGraph] = None + knowledge_graph: t.Optional[KnowledgeGraph] = None, ) -> "TestsetGenerator": knowledge_graph = knowledge_graph or KnowledgeGraph() return cls( @@ -190,7 +193,7 @@ def generate_with_llamaindex_docs( llm=transforms_llm or self.llm, embedding_model=transforms_embedding_model or self.embedding_model, ) - + # convert the documents to Ragas nodes nodes = [] for doc in documents: @@ -268,7 +271,7 @@ def generate( """ if run_config is not None: self.llm.set_run_config(run_config) - + query_distribution = query_distribution or default_query_distribution(self.llm) callbacks = callbacks or [] From 2dc57dfa5d02bd07eed3af0e78d57f982d9b5724 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Thu, 31 Oct 2024 16:54:09 +0530 Subject: [PATCH 4/9] feat: added llamaindex evaluate --- src/ragas/dataset_schema.py | 3 + src/ragas/integrations/llama_index.py | 80 +++++++++++++-------------- src/ragas/llms/base.py | 3 + 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/src/ragas/dataset_schema.py b/src/ragas/dataset_schema.py index c08ddd5dd..2403ec014 100644 --- a/src/ragas/dataset_schema.py +++ b/src/ragas/dataset_schema.py @@ -316,6 +316,9 @@ def __getitem__( else: raise TypeError("Index must be int or slice") + def is_multi_turn(self) -> bool: + return self.get_sample_type() == MultiTurnSample + def to_list(self) -> t.List[t.Dict]: rows = [sample.to_dict() for sample in self.samples] diff --git a/src/ragas/integrations/llama_index.py b/src/ragas/integrations/llama_index.py index 815feb5ad..d833670f4 100644 --- a/src/ragas/integrations/llama_index.py +++ b/src/ragas/integrations/llama_index.py @@ -2,24 +2,22 @@ import logging import typing as t -from uuid import uuid4 - -import datasets +from ragas.dataset_schema import EvaluationDataset, SingleTurnSample from ragas.embeddings import LlamaIndexEmbeddingsWrapper from ragas.evaluation import evaluate as ragas_evaluate -from ragas.exceptions import ExceptionInRunner from ragas.executor import Executor from ragas.llms import LlamaIndexLLMWrapper from ragas.run_config import RunConfig -from ragas.testset.synthesizers.testset_schema import Testset if t.TYPE_CHECKING: + from langchain_core.callbacks import Callbacks from llama_index.core.base.embeddings.base import ( BaseEmbedding as LlamaIndexEmbeddings, ) from llama_index.core.base.llms.base import BaseLLM as LlamaindexLLM + from ragas.cost import TokenUsageParser from ragas.evaluation import EvaluationResult from ragas.metrics.base import Metric @@ -29,13 +27,17 @@ def evaluate( query_engine, - dataset: Testset, + dataset: EvaluationDataset, metrics: list[Metric], llm: t.Optional[LlamaindexLLM] = None, embeddings: t.Optional[LlamaIndexEmbeddings] = None, + callbacks: t.Optional[Callbacks] = None, + in_ci: bool = False, + run_config: t.Optional[RunConfig] = None, + token_usage_parser: t.Optional[TokenUsageParser] = None, raise_exceptions: bool = False, column_map: t.Optional[t.Dict[str, str]] = None, - run_config: t.Optional[RunConfig] = None, + show_progress: bool = True, ) -> EvaluationResult: column_map = column_map or {} @@ -48,57 +50,53 @@ def evaluate( li_embeddings = LlamaIndexEmbeddingsWrapper(embeddings, run_config=run_config) # validate and transform dataset - if dataset is None: - raise ValueError("Provide dataset!") - - dataset = dataset.to_hf_dataset() + if dataset is None or not isinstance(dataset, EvaluationDataset): + raise ValueError("Please provide a dataset that is of type EvaluationDataset") exec = Executor( desc="Running Query Engine", keep_progress_bar=True, + show_progress=show_progress, raise_exceptions=raise_exceptions, run_config=run_config, ) - # get query - queries = dataset["user_input"] + # check if multi-turn + if dataset.is_multi_turn(): + raise NotImplementedError( + "Multi-turn evaluation is not implemented yet. Please do raise an issue on GitHub if you need this feature and we will prioritize it" + ) + samples = t.cast(t.List[SingleTurnSample], dataset.samples) + + # get query and make jobs + queries = [sample.user_input for sample in samples] for i, q in enumerate(queries): exec.submit(query_engine.aquery, q, name=f"query-{i}") - answers: t.List[str] = [] - contexts: t.List[t.List[str]] = [] - try: - results = exec.results() - if results == []: - raise ExceptionInRunner() - except Exception as e: - raise e - else: - for r in results: - answers.append(r.response) - contexts.append([n.node.text for n in r.source_nodes]) - - # create HF dataset - hf_dataset = datasets.Dataset.from_dict( - { - "user_input": queries, - "retrieved_contexts": contexts, - "response": answers, - } - ) - if "reference" in dataset.column_names: - hf_dataset = hf_dataset.add_column( - name="reference", - column=dataset["reference"], - new_fingerprint=str(uuid4()), - ) + # get responses and retrieved contexts + responses: t.List[str] = [] + retrieved_contexts: t.List[t.List[str]] = [] + results = exec.results() + for r in results: + responses.append(r.response) + retrieved_contexts.append([n.node.text for n in r.source_nodes]) + + # append the extra information to the dataset + for i, sample in enumerate(samples): + sample.response = responses[i] + sample.retrieved_contexts = retrieved_contexts[i] results = ragas_evaluate( - dataset=hf_dataset, + dataset=dataset, metrics=metrics, llm=li_llm, embeddings=li_embeddings, raise_exceptions=raise_exceptions, + callbacks=callbacks, + show_progress=show_progress, + run_config=run_config or RunConfig(), + in_ci=in_ci, + token_usage_parser=token_usage_parser, ) return results diff --git a/src/ragas/llms/base.py b/src/ragas/llms/base.py index ecbf6279e..20856deef 100644 --- a/src/ragas/llms/base.py +++ b/src/ragas/llms/base.py @@ -299,6 +299,9 @@ def check_args( "stop": stop, } + def is_finished(self, response: LLMResult) -> bool: + return True + def generate_text( self, prompt: PromptValue, From f4582c560459a74e86d844cb6d5a96fc66fc52f8 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Thu, 31 Oct 2024 17:04:21 +0530 Subject: [PATCH 5/9] feat: tweaks to generate --- src/ragas/testset/synthesizers/generate.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/ragas/testset/synthesizers/generate.py b/src/ragas/testset/synthesizers/generate.py index 123222f95..7cb50f889 100644 --- a/src/ragas/testset/synthesizers/generate.py +++ b/src/ragas/testset/synthesizers/generate.py @@ -32,6 +32,7 @@ BaseEmbedding as LlamaIndexEmbedding, ) from llama_index.core.base.llms.base import BaseLLM as LlamaIndexLLM + from llama_index.core.schema import Document as LlamaIndexDocument from ragas.embeddings.base import BaseRagasEmbeddings from ragas.llms.base import BaseRagasLLM @@ -159,7 +160,7 @@ def generate_with_langchain_docs( def generate_with_llamaindex_docs( self, - documents: t.Sequence[LCDocument], + documents: t.Sequence[LlamaIndexDocument], testset_size: int, transforms: t.Optional[Transforms] = None, transforms_llm: t.Optional[BaseRagasLLM] = None, @@ -174,18 +175,16 @@ def generate_with_llamaindex_docs( Generates an evaluation dataset based on given scenarios and parameters. """ + run_config = run_config or RunConfig() + # force the user to provide an llm and embedding client to prevent use of default LLMs if not self.llm and not transforms_llm: raise ValueError( - """An llm client was not provided. - Provide an LLM on TestsetGenerator instantiation or as an argument for transforms_llm parameter. - Alternatively you can provide your own transforms through the `transforms` parameter.""" + "An llm client was not provided. Provide an LLM on TestsetGenerator instantiation or as an argument for transforms_llm parameter. Alternatively you can provide your own transforms through the `transforms` parameter." ) if not self.embedding_model and not transforms_embedding_model: raise ValueError( - """An embedding client was not provided. - Provide an embedding model on TestsetGenerator instantiation or as an argument for transforms_llm parameter. - Alternatively you can provide your own transforms through the `transforms` parameter.""" + "An embedding client was not provided. Provide an embedding model on TestsetGenerator instantiation or as an argument for transforms_llm parameter. Alternatively you can provide your own transforms through the `transforms` parameter." ) if not transforms: From c29d50bfd279903dc637bf68b56a3200a2532142 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Fri, 1 Nov 2024 11:10:31 +0530 Subject: [PATCH 6/9] docs: llamaindex integration docs --- docs/howtos/integrations/llamaindex.ipynb | 570 +++++++++++++--------- 1 file changed, 339 insertions(+), 231 deletions(-) diff --git a/docs/howtos/integrations/llamaindex.ipynb b/docs/howtos/integrations/llamaindex.ipynb index c20371443..7ccef2ed8 100644 --- a/docs/howtos/integrations/llamaindex.ipynb +++ b/docs/howtos/integrations/llamaindex.ipynb @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "096e5af0", "metadata": {}, "outputs": [], @@ -47,25 +47,23 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "id": "e2107b62", "metadata": {}, "outputs": [], "source": [ - "from ragas.testset.generator import TestsetGenerator\n", - "from ragas.testset.evolutions import simple, reasoning, multi_context\n", + "from ragas.testset import TestsetGenerator\n", + "\n", "from llama_index.llms.openai import OpenAI\n", "from llama_index.embeddings.openai import OpenAIEmbedding\n", "\n", "# generator with openai models\n", - "generator_llm = OpenAI(model=\"gpt-3.5-turbo-16k\")\n", - "critic_llm = OpenAI(model=\"gpt-4\")\n", - "embeddings = OpenAIEmbedding()\n", + "generator_llm = OpenAI(model=\"gpt-4o\")\n", + "embeddings = OpenAIEmbedding(model=\"text-embedding-3-large\")\n", "\n", "generator = TestsetGenerator.from_llama_index(\n", - " generator_llm=generator_llm,\n", - " critic_llm=critic_llm,\n", - " embeddings=embeddings,\n", + " llm=generator_llm,\n", + " embedding_model=embeddings,\n", ")" ] }, @@ -79,19 +77,61 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "id": "fe03839d", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e555d31a1f8f494a9533605c03ec4140", + "model_id": "c79d70c876624617a0d8245bb57dfa34", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Applying [SummaryExtractor, HeadlinesExtractor]: 0%| | 0/2 [00:00\n", " \n", " \n", - " question\n", - " contexts\n", - " ground_truth\n", - " evolution_type\n", - " metadata\n", - " episode_done\n", + " user_input\n", + " reference_contexts\n", + " reference\n", + " synthesizer_name\n", " \n", " \n", " \n", " \n", " 0\n", - " What cultural movement began in New York City ...\n", - " [ Others cite the end of the crack epidemic an...\n", - " The Harlem Renaissance\n", - " simple\n", - " [{'file_path': '/home/jjmachan/jjmachan/explod...\n", - " True\n", + " What events led to New York being named after ...\n", + " [Etymology ==\\n\\nIn 1664, New York was named i...\n", + " New York was named after the Duke of York in 1...\n", + " AbstractQuerySynthesizer\n", " \n", " \n", " 1\n", - " What is the significance of New York City's tr...\n", - " [ consisting of 51 council members whose distr...\n", - " New York City's transportation system is both ...\n", - " simple\n", - " [{'file_path': '/home/jjmachan/jjmachan/explod...\n", - " True\n", + " How early European explorers and Native Americ...\n", + " [History ==\\n\\n\\n=== Early history ===\\nIn the...\n", + " Early European explorers and Native Americans ...\n", + " AbstractQuerySynthesizer\n", " \n", " \n", " 2\n", - " What factors led to the creation of Central Pa...\n", - " [ next ten years with British troops stationed...\n", - " Public-minded members of the contemporaneous b...\n", - " reasoning\n", - " [{'file_path': '/home/jjmachan/jjmachan/explod...\n", - " True\n", + " New York City population economy challenges\n", + " [New York City, the most populous city in the ...\n", + " New York City, as the most populous city in th...\n", + " ComparativeAbstractQuerySynthesizer\n", " \n", " \n", " 3\n", - " What was the impact of the Treaty of Breda on ...\n", - " [ British raids. In 1626, the Dutch colonial D...\n", - " The Treaty of Breda confirmed the transfer of ...\n", - " multi_context\n", - " [{'file_path': '/home/jjmachan/jjmachan/explod...\n", - " True\n", + " How do the economic aspects of New York City, ...\n", + " [New York City, the most populous city in the ...\n", + " New York City's economic aspects as a global c...\n", + " ComparativeAbstractQuerySynthesizer\n", " \n", " \n", " 4\n", - " What role did New York play in the American Re...\n", - " [ British raids. In 1626, the Dutch colonial D...\n", - " New York played a significant role in the Amer...\n", - " simple\n", - " [{'file_path': '/home/jjmachan/jjmachan/explod...\n", - " True\n", + " What are some of the cultural and architectura...\n", + " [Geography ==\\n\\nDuring the Wisconsin glaciati...\n", + " Brooklyn is distinct within New York City due ...\n", + " SpecificQuerySynthesizer\n", " \n", " \n", "\n", "" ], "text/plain": [ - " question \\\n", - "0 What cultural movement began in New York City ... \n", - "1 What is the significance of New York City's tr... \n", - "2 What factors led to the creation of Central Pa... \n", - "3 What was the impact of the Treaty of Breda on ... \n", - "4 What role did New York play in the American Re... \n", + " user_input \\\n", + "0 What events led to New York being named after ... \n", + "1 How early European explorers and Native Americ... \n", + "2 New York City population economy challenges \n", + "3 How do the economic aspects of New York City, ... \n", + "4 What are some of the cultural and architectura... \n", "\n", - " contexts \\\n", - "0 [ Others cite the end of the crack epidemic an... \n", - "1 [ consisting of 51 council members whose distr... \n", - "2 [ next ten years with British troops stationed... \n", - "3 [ British raids. In 1626, the Dutch colonial D... \n", - "4 [ British raids. In 1626, the Dutch colonial D... \n", + " reference_contexts \\\n", + "0 [Etymology ==\\n\\nIn 1664, New York was named i... \n", + "1 [History ==\\n\\n\\n=== Early history ===\\nIn the... \n", + "2 [New York City, the most populous city in the ... \n", + "3 [New York City, the most populous city in the ... \n", + "4 [Geography ==\\n\\nDuring the Wisconsin glaciati... \n", "\n", - " ground_truth evolution_type \\\n", - "0 The Harlem Renaissance simple \n", - "1 New York City's transportation system is both ... simple \n", - "2 Public-minded members of the contemporaneous b... reasoning \n", - "3 The Treaty of Breda confirmed the transfer of ... multi_context \n", - "4 New York played a significant role in the Amer... simple \n", + " reference \\\n", + "0 New York was named after the Duke of York in 1... \n", + "1 Early European explorers and Native Americans ... \n", + "2 New York City, as the most populous city in th... \n", + "3 New York City's economic aspects as a global c... \n", + "4 Brooklyn is distinct within New York City due ... \n", "\n", - " metadata episode_done \n", - "0 [{'file_path': '/home/jjmachan/jjmachan/explod... True \n", - "1 [{'file_path': '/home/jjmachan/jjmachan/explod... True \n", - "2 [{'file_path': '/home/jjmachan/jjmachan/explod... True \n", - "3 [{'file_path': '/home/jjmachan/jjmachan/explod... True \n", - "4 [{'file_path': '/home/jjmachan/jjmachan/explod... True " + " synthesizer_name \n", + "0 AbstractQuerySynthesizer \n", + "1 AbstractQuerySynthesizer \n", + "2 ComparativeAbstractQuerySynthesizer \n", + "3 ComparativeAbstractQuerySynthesizer \n", + "4 SpecificQuerySynthesizer " ] }, - "execution_count": 6, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -275,14 +381,13 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "id": "37c4a1cb", "metadata": {}, "outputs": [], "source": [ "# build query engine\n", - "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n", - "from llama_index.core.settings import Settings\n", + "from llama_index.core import VectorStoreIndex\n", "\n", "vector_index = VectorStoreIndex.from_documents(documents)\n", "\n", @@ -299,17 +404,17 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "id": "895d95b2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'What cultural movement began in New York City and established the African-American literary canon in the United States?'" + "'What events led to New York being named after the Duke of York?'" ] }, - "execution_count": 9, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -317,12 +422,12 @@ "source": [ "# convert it to pandas dataset\n", "df = testset.to_pandas()\n", - "df[\"question\"][0]" + "df[\"user_input\"][0]" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "id": "a25026c2", "metadata": {}, "outputs": [ @@ -330,12 +435,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "The Harlem Renaissance was the cultural movement that began in New York City and established the African-American literary canon in the United States.\n" + "New York was named in honor of the Duke of York because King Charles II appointed the Duke as proprietor of the former territory of New Netherland, which included the city of New Amsterdam, when England seized it from Dutch control.\n" ] } ], "source": [ - "response_vector = query_engine.query(df[\"question\"][0])\n", + "response_vector = query_engine.query(df[\"user_input\"][0])\n", "\n", "print(response_vector)" ] @@ -374,50 +479,30 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "id": "9875132a", "metadata": {}, "outputs": [], "source": [ + "# import metrics\n", "from ragas.metrics import (\n", - " faithfulness,\n", - " answer_relevancy,\n", - " context_precision,\n", - " context_recall,\n", + " Faithfulness,\n", + " AnswerRelevancy,\n", + " ContextPrecision,\n", + " ContextRecall,\n", ")\n", - "from ragas.metrics.critique import harmfulness\n", "\n", + "# init metrics with evaluator LLM\n", + "from ragas.llms import LlamaIndexLLMWrapper\n", + "evaluator_llm = LlamaIndexLLMWrapper(OpenAI(model=\"gpt-4o\"))\n", "metrics = [\n", - " faithfulness,\n", - " answer_relevancy,\n", - " context_precision,\n", - " context_recall,\n", - " harmfulness,\n", + " Faithfulness(llm=evaluator_llm),\n", + " AnswerRelevancy(llm=evaluator_llm),\n", + " ContextPrecision(llm=evaluator_llm),\n", + " ContextRecall(llm=evaluator_llm),\n", "]" ] }, - { - "cell_type": "markdown", - "id": "8230a307", - "metadata": {}, - "source": [ - "now lets init the evaluator model" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "f8049166", - "metadata": {}, - "outputs": [], - "source": [ - "from llama_index.llms.openai import OpenAI\n", - "from llama_index.embeddings.openai import OpenAIEmbedding\n", - "\n", - "# using GPT 3.5, use GPT 4 / 4-turbo for better accuracy\n", - "evaluator_llm = OpenAI(model=\"gpt-3.5-turbo\")" - ] - }, { "cell_type": "markdown", "id": "605e5d96", @@ -428,32 +513,25 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 17, "id": "4b2a81ed", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['The Harlem Renaissance',\n", - " \"New York City's transportation system is both complex and extensive, with a comprehensive mass transit system that accounts for one in every three users of mass transit in the United States. The New York City Subway system is the largest rapid transit system in the world, and the city has a high usage of public transport, with a majority of households not owning a car. Due to their reliance on mass transit, New Yorkers spend less of their household income on transportation compared to the national average.\",\n", - " 'Public-minded members of the contemporaneous business elite lobbied for the establishment of Central Park',\n", - " 'The Treaty of Breda confirmed the transfer of New Amsterdam to English control and the renaming of the settlement as New York. The Duke of York, who would later become King James II and VII, played a significant role in the naming of New York City.',\n", - " 'New York played a significant role in the American Revolution. The Stamp Act Congress met in New York in October 1765, and the city became a center for the Sons of Liberty organization. Skirmishes and battles took place in and around New York, including the Battle of Long Island and the Battle of Saratoga. The city was occupied by British forces for much of the war, but it was eventually liberated by American troops in 1783.']" + "EvaluationDataset(samples=[SingleTurnSample(user_input='What events led to New York being named after the Duke of York?', retrieved_contexts=None, reference_contexts=[\"Etymology ==\\n\\nIn 1664, New York was named in honor of the Duke of York, who would become King James II of England. James's elder brother, King Charles II, appointed the Duke as proprietor of the former territory of New Netherland, including the city of New Amsterdam, when England seized it from Dutch control.\\n\\n\\n== \"], response=None, multi_responses=None, reference='New York was named after the Duke of York in 1664 when King Charles II appointed him as proprietor of the former territory of New Netherland, including the city of New Amsterdam, after England seized it from Dutch control.', rubric=None), SingleTurnSample(user_input='How early European explorers and Native Americans shape New York City?', retrieved_contexts=None, reference_contexts=['History ==\\n\\n\\n=== Early history ===\\nIn the pre-Columbian era, the area of present-day New York City was inhabited by Algonquian Native Americans, including the Lenape. Their homeland, known as Lenapehoking, included the present-day areas of Staten Island, Manhattan, the Bronx, the western portion of Long Island (including the areas that would later become the boroughs of Brooklyn and Queens), and the Lower Hudson Valley.The first documented visit into New York Harbor by a European was in 1524 by Italian Giovanni da Verrazzano, an explorer from Florence in the service of the French crown. He claimed the area for France and named it Nouvelle Angoulême (New Angoulême). A Spanish expedition, led by the Portuguese captain Estêvão Gomes sailing for Emperor Charles V, arrived in New York Harbor in January 1525 and charted the mouth of the Hudson River, which he named Río de San Antonio (\\'Saint Anthony\\'s River\\'). The Padrón Real of 1527, the first scientific map to show the East Coast of North America continuously, was informed by Gomes\\' expedition and labeled the northeastern United States as Tierra de Esteban Gómez in his honor.In 1609, the English explorer Henry Hudson rediscovered New York Harbor while searching for the Northwest Passage to the Orient for the Dutch East India Company. He proceeded to sail up what the Dutch would name the North River (now the Hudson River), named first by Hudson as the Mauritius after Maurice, Prince of Orange. Hudson\\'s first mate described the harbor as \"a very good Harbour for all windes\" and the river as \"a mile broad\" and \"full of fish\". Hudson sailed roughly 150 miles (240 km) north, past the site of the present-day New York State capital city of Albany, in the belief that it might be an oceanic tributary before the river became too shallow to continue. He made a ten-day exploration of the area and claimed the region for the Dutch East India Company. In 1614, the area between Cape Cod and Delaware Bay was claimed by the Netherlands and called Nieuw-Nederland (\\'New Netherland\\').\\nThe first non–Native American inhabitant of what would eventually become New York City was Juan Rodriguez (transliterated to the Dutch language as Jan Rodrigues), a merchant from Santo Domingo. Born in Santo Domingo of Portuguese and African descent, he arrived in Manhattan during the winter of 1613–14, trapping for pelts and trading with the local population as a representative of the Dutch. Broadway, from 159th Street to 218th Street in Upper Manhattan, is named Juan Rodriguez Way in his honor.\\n\\n\\n=== Dutch rule ===\\n\\nA permanent European presence near New York Harbor was established in 1624, making New York the 12th-oldest continuously occupied European-established settlement in the continental United States—with the founding of a Dutch fur trading settlement on Governors Island. In 1625, construction was started on a citadel and Fort Amsterdam, later called Nieuw Amsterdam (New Amsterdam), on present-day Manhattan Island. The colony of New Amsterdam was centered on what would ultimately be known as Lower Manhattan. Its area extended from the southern tip of Manhattan to modern day Wall Street, where a 12-foot wooden stockade was built in 1653 to protect against Native American and British raids. In 1626, the Dutch colonial Director-General Peter Minuit, acting as charged by the Dutch West India Company, purchased the island of Manhattan from the Canarsie, a small Lenape band, for \"the value of 60 guilders\" (about $900 in 2018). A disproved legend claims that Manhattan was purchased for $24 worth of glass beads.Following the purchase, New Amsterdam grew slowly. To attract settlers, the Dutch instituted the patroon system in 1628, whereby wealthy Dutchmen (patroons, or patrons) who brought 50 colonists to New Netherland would be awarded swaths of land, along with local political autonomy and rights to participate in the lucrative fur trade. This program had little success.Since 1621, the Dutch West India Company had operated as a monopoly in New Netherland, on authority granted by the Dutch States General. In 1639–1640, in an effort to bolster economic growth, the Dutch West India Company relinquished its monopoly over the fur trade, leading to growth in the production and trade of food, timber, tobacco, and slaves (particularly with the Dutch West Indies).In 1647, Peter Stuyvesant began his tenure as the last Director-General of New Netherland. During his tenure, the population of New Netherland grew from 2,000 to 8,000. Stuyvesant has been credited with improving law and order in the colony; however, he also earned a reputation as a despotic leader. He instituted regulations on liquor sales, attempted to assert control over the Dutch Reformed Church, and blocked other religious groups (including Quakers, Jews, and Lutherans) from establishing houses of worship. The Dutch West India Company would eventually attempt to ease tensions between Stuyvesant and residents of New Amsterdam.\\n\\n\\n=== English rule ===\\n\\nIn 1664, unable to summon any significant resistance, Stuyvesant surrendered New Amsterdam to English troops, led by Colonel Richard Nicolls, without bloodshed. The terms of the surrender permitted Dutch residents to remain in the colony and allowed for religious freedom. In 1667, during negotiations leading to the Treaty of Breda after the Second Anglo-Dutch War, the Dutch decided to keep the nascent plantation colony of what is now Suriname (on the northern South American coast) they had gained from the English; and in return, the English kept New Amsterdam. The fledgling settlement was promptly renamed \"New York\" after the Duke of York (the future King James II and VII), who would eventually be deposed in the Glorious Revolution. After the founding, the duke gave part of the colony to proprietors George Carteret and John Berkeley. Fort Orange, 150 miles (240 km) north on the Hudson River, was renamed Albany after James\\'s Scottish title. The transfer was confirmed in 1667 by the Treaty of Breda, which concluded the Second Anglo-Dutch War.On August 24, 1673, during the Third Anglo-Dutch War, Dutch captain Anthony Colve seized the colony of New York from the English at the behest of Cornelis Evertsen the Youngest and rechristened it \"New Orange\" after William III, the Prince of Orange. The Dutch would soon return the island to England under the Treaty of Westminster of November 1674.Several intertribal wars among the Native Americans and some epidemics brought on by contact with the Europeans caused sizeable population losses for the Lenape between the years 1660 and 1670. By 1700, the Lenape population had diminished to 200. New York experienced several yellow fever epidemics in the 18th century, losing ten percent of its population to the disease in 1702 alone.\\n\\n\\n=== Province of New York and slavery ===\\n\\nIn the early 18th century, New York grew in importance as a trading port while as a part of the colony of New York. It also became a center of slavery, with 42% of households enslaving Africans by 1730, the highest percentage outside Charleston, South Carolina. Most cases were that of domestic slavery, as a New York household then commonly enslaved few or several people. Others were hired out to work at labor. Slavery became integrally tied to New York\\'s economy through the labor of slaves throughout the port, and the banking and shipping industries trading with the American South. During construction in Foley Square in the 1990s, the African Burying Ground was discovered; the cemetery included 10,000 to 20,000 of graves of colonial-era Africans, some enslaved and some free.The 1735 trial and acquittal in Manhattan of John Peter Zenger, who had been accused of seditious libel after criticizing colonial governor William Cosby, helped to establish the freedom of the press in North America. In 1754, Columbia University was founded under charter by King George II as King\\'s College in Lower Manhattan.\\n\\n\\n=== American Revolution ===\\n\\nThe Stamp Act Congress met in New York in October 1765, as the Sons of Liberty organization emerged in the city and skirmished over the next ten years with British troops stationed there. The Battle of Long Island, the largest battle of the American Revolutionary War, was fought in August 1776 within the modern-day borough of Brooklyn. After the battle, in which the Americans were defeated, the British made the city their military and political base of operations in North America. The city was a haven for Loyalist refugees and escaped slaves who joined the British lines for freedom newly promised by the Crown for all fighters. As many as 10,000 escaped slaves crowded into the city during the British occupation. When the British forces evacuated at the close of the war in 1783, they transported 3,000 freedmen for resettlement in Nova Scotia. They resettled other freedmen in England and the Caribbean.\\nThe only attempt at a peaceful solution to the war took place at the Conference House on Staten Island between American delegates, including Benjamin Franklin, and British general Lord Howe on September 11, 1776. Shortly after the British occupation began, the Great Fire of New York occurred, a large conflagration on the West Side of Lower Manhattan, which destroyed about a quarter of the buildings in the city, including Trinity Church.In 1785, the assembly of the Congress of the Confederation made New York City the national capital shortly after the war. New York was the last capital of the U.S. under the Articles of Confederation and the first capital under the Constitution of the United States. New York City as the U.S. capital hosted several events of national scope in 1789—the first President of the United States, George Washington, was inaugurated; the first United States Congress and the Supreme Court of the United States each assembled for the first time; and the United States Bill of Rights was drafted, all at Federal Hall on Wall Street. In 1790, New York surpassed Philadelphia as the nation\\'s largest city. At the end of that year, pursuant to the Residence Act, the national capital was moved to Philadelphia.\\n\\n\\n=== 19th century ===\\n\\nOver the course of the nineteenth century, New York City\\'s population grew from 60,000 to 3.43 million. Under New York State\\'s abolition act of 1799, children of slave mothers were to be eventually liberated but to be held in indentured servitude until their mid-to-late twenties. Together with slaves freed by their masters after the Revolutionary War and escaped slaves, a significant free-Black population gradually developed in Manhattan. Under such influential United States founders as Alexander Hamilton and John Jay, the New York Manumission Society worked for abolition and established the African Free School to educate Black children. It was not until 1827 that slavery was completely abolished in the state, and free Blacks struggled afterward with discrimination. New York interracial abolitionist activism continued; among its leaders were graduates of the African Free School. New York city\\'s population jumped from 123,706 in 1820 to 312,710 by 1840, 16,000 of whom were Black.In the 19th century, the city was transformed by both commercial and residential development relating to its status as a national and international trading center, as well as by European immigration, respectively. The city adopted the Commissioners\\' Plan of 1811, which expanded the city street grid to encompass almost all of Manhattan. The 1825 completion of the Erie Canal through central New York connected the Atlantic port to the agricultural markets and commodities of the North American interior via the Hudson River and the Great Lakes. Local politics became dominated by Tammany Hall, a political machine supported by Irish and German immigrants.Several prominent American literary figures lived in New York during the 1830s and 1840s, including William Cullen Bryant, Washington Irving, Herman Melville, Rufus Wilmot Griswold, John Keese, Nathaniel Parker Willis, and Edgar Allan Poe. Public-minded members of the contemporaneous business elite lobbied for the establishment of Central Park, which in 1857 became the first landscaped park in an American city.\\nThe Great Irish Famine brought a large influx of Irish immigrants; more than 200,000 were living in New York by 1860, upwards of a quarter of the city\\'s population. There was also extensive immigration from the German provinces, where revolutions had disrupted societies, and Germans comprised another 25% of New York\\'s population by 1860.Democratic Party candidates were consistently elected to local office, increasing the city\\'s ties to the South and its dominant party. In 1861, Mayor Fernando Wood called upon the aldermen to declare independence from Albany and the United States after the South seceded, but his proposal was not acted on. Anger at new military conscription laws during the American Civil War (1861–1865), which spared wealthier men who could afford to pay a $300 (equivalent to $6,602 in 2021) commutation fee to hire a substitute, led to the Draft Riots of 1863, whose most visible participants were ethnic Irish working class.The draft riots deteriorated into attacks on New York\\'s elite, followed by attacks on Black New Yorkers and their property after fierce competition for a decade between Irish immigrants and Black people for work. Rioters burned the Colored Orphan Asylum to the ground, with more than 200 children escaping harm due to efforts of the New York Police Department, which was mainly made up of Irish immigrants. At least 120 people were killed. Eleven Black men were lynched over five days, and the riots forced hundreds of Blacks to flee the city for Williamsburg, Brooklyn, and New Jersey. The Black population in Manhattan fell below 10,000 by 1865, which it had last been in 1820. The White working class had established dominance. Violence by longshoremen against Black men was especially fierce in the docks area. It was one of the worst incidents of civil unrest in American history.In 1898, the City of New York was formed with the consolidation of Brooklyn (until then a separate city), the County of New York (which then included parts of the Bronx), the County of Richmond, and the western portion of the County of Queens. The opening of the subway in 1904, first built as separate private systems, helped bind the new city together. Throughout the first half of the 20th century, the city became a world center for industry, commerce, and communication.\\n\\n\\n=== 20th century ===\\n\\nIn 1904, the steamship General Slocum caught fire in the East River, killing 1,021 people on board. In 1911, the Triangle Shirtwaist Factory fire, the city\\'s worst industrial disaster, took the lives of 146 garment workers and spurred the growth of the International Ladies\\' Garment Workers\\' Union and major improvements in factory safety standards.New York\\'s non-White population was 36,620 in 1890. New York City was a prime destination in the early twentieth century for African Americans during the Great Migration from the American South, and by 1916, New York City had become home to the largest urban African diaspora in North America. The Harlem Renaissance of literary and cultural life flourished during the era of Prohibition. The larger economic boom generated construction of skyscrapers competing in height and creating an identifiable skyline.\\nNew York became the most populous urbanized area in the world in the early 1920s, overtaking London. The metropolitan area surpassed the 10 million mark in the early 1930s, becoming the first megacity in human history. The Great Depression saw the election of reformer Fiorello La Guardia as mayor and the fall of Tammany Hall after eighty years of political dominance.Returning World War II veterans created a post-war economic boom and the development of large housing tracts in eastern Queens and Nassau County as well as similar suburban areas in New Jersey. New York emerged from the war unscathed as the leading city of the world, with Wall Street leading America\\'s place as the world\\'s dominant economic power. The United Nations headquarters was completed in 1952, solidifying New York\\'s global geopolitical influence, and the rise of abstract expressionism in the city precipitated New York\\'s displacement of Paris as the center of the art world.The Stonewall riots were a series of spontaneous, violent protests by members of the gay community against a police raid that took place in the early morning hours of June 28, 1969, at the Stonewall Inn in the Greenwich Village neighborhood of Lower Manhattan. They are widely considered to constitute the single most important event leading to the gay liberation movement and the modern fight for LGBT rights. Wayne R. Dynes, author of the Encyclopedia of Homosexuality, wrote that drag queens were the only \"transgender folks around\" during the June 1969 Stonewall riots. The transgender community in New York City played a significant role in fighting for LGBT equality during the period of the Stonewall riots and thereafter.In the 1970s, job losses due to industrial restructuring caused New York City to suffer from economic problems and rising crime rates. While a resurgence in the financial industry greatly improved the city\\'s economic health in the 1980s, New York\\'s crime rate continued to increase through that decade and into the beginning of the 1990s. By the mid 1990s, crime rates started to drop dramatically due to revised police strategies, improving economic opportunities, gentrification, and new residents, both American transplants and new immigrants from Asia and Latin America. Important new sectors, such as Silicon Alley, emerged in the city\\'s economy.\\n\\n\\n=== 21st century ===\\n\\nNew York\\'s population reached all-time highs in the 2000 census and then again in the 2010 census.\\nNew York City suffered the bulk of the economic damage and largest loss of human life in the aftermath of the September 11, 2001, attacks. Two of the four airliners hijacked that day were flown into the twin towers of the World Trade Center, destroying the towers and killing 2,192 civilians, 343 firefighters, and 71 law enforcement officers. The North Tower became the tallest building ever to be destroyed anywhere then or subsequently.The area was rebuilt with a new One World Trade Center, a 9/11 memorial and museum, and other new buildings and infrastructure. The World Trade Center PATH station, which had opened on July 19, 1909, as the Hudson Terminal, was also destroyed in the attacks. A temporary station was built and opened on November 23, 2003. An 800,000-square-foot (74,000 m2) permanent rail station designed by Santiago Calatrava, the World Trade Center Transportation Hub, the city\\'s third-largest hub, was completed in 2016. The new One World Trade Center is the tallest skyscraper in the Western Hemisphere and the seventh-tallest building in the world by pinnacle height, with its spire reaching a symbolic 1,776 feet (541.3 m) in reference to the year of U.S. independence.The Occupy Wall Street protests in Zuccotti Park in the Financial District of Lower Manhattan began on September 17, 2011, receiving global attention and popularizing the Occupy movement against social and economic inequality worldwide.New York City was heavily affected by Hurricane Sandy in late October 2012. Sandy\\'s impacts included the flooding of the New York City Subway system, of many suburban communities, and of all road tunnels entering Manhattan except the Lincoln Tunnel. The New York Stock Exchange closed for two consecutive days. Numerous homes and businesses were destroyed by fire, including over 100 homes in Breezy Point, Queens. Large parts of the city and surrounding areas lost electricity for several days. Several thousand people in Midtown Manhattan were evacuated for six days due to a crane collapse at Extell\\'s One57. Bellevue Hospital Center and a few other large hospitals were closed and evacuated. Flooding at 140 West Street and another exchange disrupted voice and data communication in Lower Manhattan. At least 43 people lost their lives in New York City as a result of Sandy, and the economic losses in New York City were estimated to be roughly $19 billion. The disaster spawned long-term efforts towards infrastructural projects to counter climate change and rising seas.In March 2020, the first case of COVID-19 in the city was confirmed in Manhattan. The city rapidly replaced Wuhan, China to become the global epicenter of the pandemic during the early phase, before the infection became widespread across the world and the rest of the nation. As of March 2021, New York City had recorded over 30,000 deaths from COVID-19-related complications. In 2022, the LGBT community in New York City became the epicenter of the monkeypox outbreak in the Western Hemisphere, prompting New York Governor Kathy Hochul and New York City Mayor Eric Adams declared corresponding public health emergencies in the state and city, respectively, in July 2022.\\n\\n\\n== '], response=None, multi_responses=None, reference=\"Early European explorers and Native Americans shaped New York City through exploration, trade, and settlement. Native Americans, including the Lenape, originally inhabited the area. European explorers like Giovanni da Verrazzano and Henry Hudson claimed and charted the region for European powers. The Dutch established a permanent settlement, New Amsterdam, which later became New York under English rule. These interactions and exchanges laid the foundation for the city's development and cultural diversity.\", rubric=None), SingleTurnSample(user_input='New York City population economy challenges', retrieved_contexts=None, reference_contexts=[\"New York City, the most populous city in the United States, is a global cultural, financial, and media hub with significant influence in various sectors including commerce, healthcare, and technology. It is composed of five boroughs: Brooklyn, Queens, Manhattan, The Bronx, and Staten Island, each with its own unique character. The city is known for its diverse population, with over 800 languages spoken, making it the most linguistically diverse city in the world. NYC is a major center for international diplomacy, hosting the United Nations headquarters. The city has a rich history, from its early days as a Dutch trading post to its role as a gateway for immigrants. It is home to iconic landmarks such as the Statue of Liberty, Times Square, and Central Park. New York City is also a leader in arts and culture, with numerous museums, theaters, and music venues. The city's economy is driven by finance, technology, real estate, and tourism, with Wall Street being a major financial center. NYC's public transportation system is extensive, including the largest subway system in the world. The city faces challenges such as income disparity and environmental issues but continues to be a vibrant and dynamic metropolis.\"], response=None, multi_responses=None, reference='New York City, as the most populous city in the United States, faces challenges such as income disparity and environmental issues while maintaining a vibrant and dynamic economy driven by finance, technology, real estate, and tourism.', rubric=None), SingleTurnSample(user_input='How do the economic aspects of New York City, as a global cultural center and financial industry hub with a diverse population, compare in terms of influence and challenges across different reports?', retrieved_contexts=None, reference_contexts=[\"New York City, the most populous city in the United States, is a global cultural, financial, and media hub with significant influence in various sectors including commerce, healthcare, and technology. It is composed of five boroughs: Brooklyn, Queens, Manhattan, The Bronx, and Staten Island, each with its own unique character. The city is known for its diverse population, with over 800 languages spoken, making it the most linguistically diverse city in the world. NYC is a major center for international diplomacy, hosting the United Nations headquarters. The city has a rich history, from its early days as a Dutch trading post to its role as a gateway for immigrants. It is home to iconic landmarks such as the Statue of Liberty, Times Square, and Central Park. New York City is also a leader in arts and culture, with numerous museums, theaters, and music venues. The city's economy is driven by finance, technology, real estate, and tourism, with Wall Street being a major financial center. NYC's public transportation system is extensive, including the largest subway system in the world. The city faces challenges such as income disparity and environmental issues but continues to be a vibrant and dynamic metropolis.\"], response=None, multi_responses=None, reference=\"New York City's economic aspects as a global cultural center and financial industry hub are influential due to its diverse population, significant sectors like finance and technology, and its role in international diplomacy. However, it faces challenges such as income disparity and environmental issues.\", rubric=None), SingleTurnSample(user_input='What are some of the cultural and architectural features that make Brooklyn distinct within New York City?', retrieved_contexts=None, reference_contexts=['Geography ==\\n\\nDuring the Wisconsin glaciation, 75,000 to 11,000 years ago, the New York City area was situated at the edge of a large ice sheet over 2,000 feet (610 m) in depth. The erosive forward movement of the ice (and its subsequent retreat) contributed to the separation of what is now Long Island and Staten Island. That action also left bedrock at a relatively shallow depth, providing a solid foundation for most of Manhattan\\'s skyscrapers.New York City is situated in the northeastern United States, in southeastern New York State, approximately halfway between Washington, D.C. and Boston. The location at the mouth of the Hudson River, which feeds into a naturally sheltered harbor and then into the Atlantic Ocean, has helped the city grow in significance as a trading port. Most of New York City is built on the three islands of Long Island, Manhattan, and Staten Island.\\nThe Hudson River flows through the Hudson Valley into New York Bay. Between New York City and Troy, New York, the river is an estuary. The Hudson River separates the city from the U.S. state of New Jersey. The East River—a tidal strait—flows from Long Island Sound and separates the Bronx and Manhattan from Long Island. The Harlem River, another tidal strait between the East and Hudson rivers, separates most of Manhattan from the Bronx. The Bronx River, which flows through the Bronx and Westchester County, is the only entirely freshwater river in the city.The city\\'s land has been altered substantially by human intervention, with considerable land reclamation along the waterfronts since Dutch colonial times; reclamation is most prominent in Lower Manhattan, with developments such as Battery Park City in the 1970s and 1980s. Some of the natural relief in topography has been evened out, especially in Manhattan.The city\\'s total area is 468.484 square miles (1,213.37 km2); 302.643 sq mi (783.84 km2) of the city is land and 165.841 sq mi (429.53 km2) of this is water. The highest point in the city is Todt Hill on Staten Island, which, at 409.8 feet (124.9 m) above sea level, is the highest point on the eastern seaboard south of Maine. The summit of the ridge is mostly covered in woodlands as part of the Staten Island Greenbelt.\\n\\n\\n=== Boroughs ===\\n\\nNew York City is sometimes referred to collectively as the Five Boroughs. Each borough is coextensive with a respective county of New York State, making New York City one of the U.S. municipalities in multiple counties. There are hundreds of distinct neighborhoods throughout the boroughs, many with a definable history and character.\\nIf the boroughs were each independent cities, four of the boroughs (Brooklyn, Queens, Manhattan, and the Bronx) would be among the ten most populous cities in the United States (Staten Island would be ranked 37th as of 2020); these same boroughs are coterminous with the four most densely populated counties in the United States: New York (Manhattan), Kings (Brooklyn), Bronx, and Queens.\\n\\n\\n==== Manhattan ====\\n\\nManhattan (New York County) is the geographically smallest and most densely populated borough. It is home to Central Park and most of the city\\'s skyscrapers, and is sometimes locally known as The City. Manhattan\\'s population density of 72,033 people per square mile (27,812/km2) in 2015 makes it the highest of any county in the United States and higher than the density of any individual American city.Manhattan is the cultural, administrative, and financial center of New York City and contains the headquarters of many major multinational corporations, the United Nations headquarters, Wall Street, and a number of important universities. The borough of Manhattan is often described as the financial and cultural center of the world.Most of the borough is situated on Manhattan Island, at the mouth of the Hudson River and the East River, and its southern tip, at the confluence of the two rivers, represents the birthplace of New York City itself. Several small islands also compose part of the borough of Manhattan, including Randalls and Wards Islands, and Roosevelt Island in the East River, and Governors Island and Liberty Island to the south in New York Harbor.\\nManhattan Island is loosely divided into the Lower, Midtown, and Uptown regions. Uptown Manhattan is divided by Central Park into the Upper East Side and the Upper West Side, and above the park is Harlem, bordering the Bronx (Bronx County).\\nHarlem was predominantly occupied by Jewish and Italian Americans in the 19th century until the Great Migration. It was the center of the Harlem Renaissance.\\n\\nThe borough of Manhattan also includes a small neighborhood on the mainland, called Marble Hill, which is contiguous with the Bronx. New York City\\'s remaining four boroughs are collectively referred to as the Outer Boroughs.\\n\\n\\n==== Brooklyn ====\\nBrooklyn (Kings County), on the western tip of Long Island, is the city\\'s most populous borough. Brooklyn is known for its cultural, social, and ethnic diversity, an independent art scene, distinct neighborhoods, and a distinctive architectural heritage. Downtown Brooklyn is the largest central core neighborhood in the Outer Boroughs. The borough has a long beachfront shoreline including Coney Island, established in the 1870s as one of the earliest amusement grounds in the U.S. Marine Park and Prospect Park are the two largest parks in Brooklyn. Since 2010, Brooklyn has evolved into a thriving hub of entrepreneurship and high technology startup firms, and of postmodern art and design.\\n\\n\\n==== Queens ====\\nQueens (Queens County), on Long Island north and east of Brooklyn, is geographically the largest borough, the most ethnically diverse county in the United States, and the most ethnically diverse urban area in the world. Historically a collection of small towns and villages founded by the Dutch, the borough has since developed both commercial and residential prominence. Downtown Flushing has become one of the busiest central core neighborhoods in the outer boroughs. Queens is the site of the Citi Field baseball stadium, home of the New York Mets, and hosts the annual U.S. Open tennis tournament at Flushing Meadows–Corona Park. Additionally, two of the three busiest airports serving the New York metropolitan area, John F. Kennedy International Airport and LaGuardia Airport, are in Queens. The third is Newark Liberty International Airport in Newark, New Jersey.\\n\\n\\n==== The Bronx ====\\nThe Bronx (Bronx County) is both New York City\\'s northernmost borough, and the only one that is mostly on the mainland. It is the location of Yankee Stadium, the baseball park of the New York Yankees, and home to the largest cooperatively-owned housing complex in the United States, Co-op City. It is also home to the Bronx Zoo, the world\\'s largest metropolitan zoo, which spans 265 acres (1.07 km2) and houses more than 6,000 animals. The Bronx is also the birthplace of hip hop music and its associated culture. Pelham Bay Park is the largest park in New York City, at 2,772 acres (1,122 ha).\\n\\n\\n==== Staten Island ====\\nStaten Island (Richmond County) is the most suburban in character of the five boroughs. Staten Island is connected to Brooklyn by the Verrazzano-Narrows Bridge, and to Manhattan by way of the free Staten Island Ferry, a daily commuter ferry that provides unobstructed views of the Statue of Liberty, Ellis Island, and Lower Manhattan. In central Staten Island, the Staten Island Greenbelt spans approximately 2,500 acres (10 km2), including 28 miles (45 km) of walking trails and one of the last undisturbed forests in the city. Designated in 1984 to protect the island\\'s natural lands, the Greenbelt comprises seven city parks.\\n\\n\\t\\t\\n\\t\\t\\n\\n\\n=== Architecture ===\\n\\nNew York has architecturally noteworthy buildings in a wide range of styles and from distinct time periods, from the Dutch Colonial Pieter Claesen Wyckoff House in Brooklyn, the oldest section of which dates to 1656, to the modern One World Trade Center, the skyscraper at Ground Zero in Lower Manhattan and the most expensive office tower in the world by construction cost.Manhattan\\'s skyline, with its many skyscrapers, is universally recognized, and the city has been home to several of the tallest buildings in the world. As of 2019, New York City had 6,455 high-rise buildings, the third most in the world after Hong Kong and Seoul. Of these, as of 2011, 550 completed structures were at least 330 feet (100 m) high, with more than fifty completed skyscrapers taller than 656 feet (200 m). These include the Woolworth Building, an early example of Gothic Revival architecture in skyscraper design, built with massively scaled Gothic detailing; completed in 1913, for 17 years it was the world\\'s tallest building.The 1916 Zoning Resolution required setbacks in new buildings and restricted towers to a percentage of the lot size, to allow sunlight to reach the streets below. The Art Deco style of the Chrysler Building (1930) and Empire State Building (1931), with their tapered tops and steel spires, reflected the zoning requirements. The buildings have distinctive ornamentation, such as the eagles at the corners of the 61st floor on the Chrysler Building, and are considered some of the finest examples of the Art Deco style. A highly influential example of the International Style in the United States is the Seagram Building (1957), distinctive for its façade using visible bronze-toned I-beams to evoke the building\\'s structure. The Condé Nast Building (2000) is a prominent example of green design in American skyscrapers and has received an award from the American Institute of Architects and AIA New York State for its design.\\nThe character of New York\\'s large residential districts is often defined by the elegant brownstone rowhouses and townhouses and shabby tenements that were built during a period of rapid expansion from 1870 to 1930. In contrast, New York City also has neighborhoods that are less densely populated and feature free-standing dwellings. In neighborhoods such as Riverdale (in the Bronx), Ditmas Park (in Brooklyn), and Douglaston (in Queens), large single-family homes are common in various architectural styles such as Tudor Revival and Victorian.Stone and brick became the city\\'s building materials of choice after the construction of wood-frame houses was limited in the aftermath of the Great Fire of 1835. A distinctive feature of many of the city\\'s buildings is the roof-mounted wooden water tower. In the 1800s, the city required their installation on buildings higher than six stories to prevent the need for excessively high water pressures at lower elevations, which could break municipal water pipes. Garden apartments became popular during the 1920s in outlying areas, such as Jackson Heights.According to the United States Geological Survey, an updated analysis of seismic hazard in July 2014 revealed a \"slightly lower hazard for tall buildings\" in New York City than previously assessed. Scientists estimated this lessened risk based upon a lower likelihood than previously thought of slow shaking near the city, which would be more likely to cause damage to taller structures from an earthquake in the vicinity of the city. Manhattan contained over 500 million square feet of office space as of 2022; the COVID-19 pandemic and hybrid work model have prompted consideration of commercial-to-residential conversion within Midtown Manhattan.\\n\\n\\n=== Climate ===\\n\\nUnder the Köppen climate classification, using the 0 °C (32 °F) isotherm, New York City features a humid subtropical climate (Cfa), and is thus the northernmost major city on the North American continent with this categorization. The suburbs to the immediate north and west lie in the transitional zone between humid subtropical and humid continental climates (Dfa). By the Trewartha classification, the city is defined as having an oceanic climate (Do). Annually, the city averages 234 days with at least some sunshine. The city lies in the USDA 7b plant hardiness zone.Winters are chilly and damp, and prevailing wind patterns that blow sea breezes offshore temper the moderating effects of the Atlantic Ocean; yet the Atlantic and the partial shielding from colder air by the Appalachian Mountains keep the city warmer in the winter than inland North American cities at similar or lesser latitudes such as Pittsburgh, Cincinnati, and Indianapolis. The daily mean temperature in January, the area\\'s coldest month, is 33.3 °F (0.7 °C). Temperatures usually drop to 10 °F (−12 °C) several times per winter, yet can also reach 60 °F (16 °C) for several days even in the coldest winter month. Spring and autumn are unpredictable and can range from cool to warm, although they are usually mild with low humidity. Summers are typically hot and humid, with a daily mean temperature of 77.5 °F (25.3 °C) in July.Nighttime temperatures are often enhanced due to the urban heat island effect. Daytime temperatures exceed 90 °F (32 °C) on average of 17 days each summer and in some years exceed 100 °F (38 °C), although this is a rare achievement, last occurring on July 18, 2012. Similarly, readings of 0 °F (−18 °C) are also extremely rare, last occurring on February 14, 2016. Extreme temperatures have ranged from −15 °F (−26 °C), recorded on February 9, 1934, up to 106 °F (41 °C) on July 9, 1936; the coldest recorded wind chill was −37 °F (−38 °C) on the same day as the all-time record low. The record cold daily maximum was 2 °F (−17 °C) on December 30, 1917, while, conversely, the record warm daily minimum was 87 °F (31 °C), on July 2, 1903. The average water temperature of the nearby Atlantic Ocean ranges from 39.7 °F (4.3 °C) in February to 74.1 °F (23.4 °C) in August.The city receives 49.5 inches (1,260 mm) of precipitation annually, which is relatively evenly spread throughout the year. Average winter snowfall between 1991 and 2020 has been 29.8 inches (76 cm); this varies considerably between years. Hurricanes and tropical storms are rare in the New York area. Hurricane Sandy brought a destructive storm surge to New York City on the evening of October 29, 2012, flooding numerous streets, tunnels, and subway lines in Lower Manhattan and other areas of the city and cutting off electricity in many parts of the city and its suburbs. The storm and its profound impacts have prompted the discussion of constructing seawalls and other coastal barriers around the shorelines of the city and the metropolitan area to minimize the risk of destructive consequences from another such event in the future.The coldest month on record is January 1857, with a mean temperature of 19.6 °F (−6.9 °C) whereas the warmest months on record are July 1825 and July 1999, both with a mean temperature of 81.4 °F (27.4 °C). The warmest years on record are 2012 and 2020, both with mean temperatures of 57.1 °F (13.9 °C). The coldest year is 1836, with a mean temperature of 47.3 °F (8.5 °C). The driest month on record is June 1949, with 0.02 inches (0.51 mm) of rainfall. The wettest month was August 2011, with 18.95 inches (481 mm) of rainfall. The driest year on record is 1965, with 26.09 inches (663 mm) of rainfall. The wettest year was 1983, with 80.56 inches (2,046 mm) of rainfall. The snowiest month on record is February 2010, with 36.9 inches (94 cm) of snowfall. The snowiest season (Jul–Jun) on record is 1995–1996, with 75.6 inches (192 cm) of snowfall. The least snowy season was 1972–1973, with 2.3 inches (5.8 cm) of snowfall. The earliest seasonal trace of snowfall occurred on October 10, in both 1979 and 1925. The latest seasonal trace of snowfall occurred on May 9, in both 2020 and 1977.\\n\\nSee or edit raw graph data.\\n\\n\\n=== Parks ===\\n\\nThe city of New York has a complex park system, with various lands operated by the National Park Service, the New York State Office of Parks, Recreation and Historic Preservation, and the New York City Department of Parks and Recreation. In its 2018 ParkScore ranking, the Trust for Public Land reported that the park system in New York City was the ninth-best park system among the fifty most populous U.S. cities. ParkScore ranks urban park systems by a formula that analyzes median park size, park acres as percent of city area, the percent of city residents within a half-mile of a park, spending of park services per resident, and the number of playgrounds per 10,000 residents. In 2021, the New York City Council banned the use of synthetic pesticides by city agencies and instead required organic lawn management. The effort was started by teacher Paula Rogovin\\'s kindergarten class at P.S. 290.\\n\\n\\n==== National parks ====\\n\\nGateway National Recreation Area contains over 26,000 acres (110 km2), most of it in New York City. In Brooklyn and Queens, the park contains over 9,000 acres (36 km2) of salt marsh, wetlands, islands, and water, including most of Jamaica Bay and the Jamaica Bay Wildlife Refuge. Also in Queens, the park includes a significant portion of the western Rockaway Peninsula, most notably Jacob Riis Park and Fort Tilden. In Staten Island, it includes Fort Wadsworth, with historic pre-Civil War era Battery Weed and Fort Tompkins, and Great Kills Park, with beaches, trails, and a marina.\\nThe Statue of Liberty National Monument and Ellis Island Immigration Museum are managed by the National Park Service and are in both New York and New Jersey. They are joined in the harbor by Governors Island National Monument. Historic sites under federal management on Manhattan Island include Stonewall National Monument; Castle Clinton National Monument; Federal Hall National Memorial; Theodore Roosevelt Birthplace National Historic Site; General Grant National Memorial (Grant\\'s Tomb); African Burial Ground National Monument; and Hamilton Grange National Memorial. Hundreds of properties are listed on the National Register of Historic Places or as a National Historic Landmark.\\n\\n\\n==== State parks ====\\n\\nThere are seven state parks within the confines of New York City. Some of them include:\\n\\nThe Clay Pit Ponds State Park Preserve is a natural area that includes extensive riding trails.\\nRiverbank State Park is a 28-acre (11 ha) facility that rises 69 feet (21 m) over the Hudson River.\\nMarsha P. Johnson State Park is a state park in Brooklyn and Manhattan that borders the East River that was renamed in honor of Marsha P. Johnson.\\n\\n\\n==== City parks ====\\n\\nNew York City has over 28,000 acres (110 km2) of municipal parkland and 14 miles (23 km) of public beaches. The largest municipal park in the city is Pelham Bay Park in the Bronx, with 2,772 acres (1,122 ha).\\nCentral Park, an 843-acre (3.41 km2) park in middle-upper Manhattan, is the most visited urban park in the United States and one of the most filmed locations in the world, with 40 million visitors in 2013. The park has a wide range of attractions; there are several lakes and ponds, two ice-skating rinks, the Central Park Zoo, the Central Park Conservatory Garden, and the 106-acre (0.43 km2) Jackie Onassis Reservoir. Indoor attractions include Belvedere Castle with its nature center, the Swedish Cottage Marionette Theater, and the historic Carousel. On October 23, 2012, hedge fund manager John A. Paulson announced a $100 million gift to the Central Park Conservancy, the largest ever monetary donation to New York City\\'s park system.\\nWashington Square Park is a prominent landmark in the Greenwich Village neighborhood of Lower Manhattan. The Washington Square Arch at the northern gateway to the park is an iconic symbol of both New York University and Greenwich Village.\\nProspect Park in Brooklyn has a 90-acre (36 ha) meadow, a lake, and extensive woodlands. Within the park is the historic Battle Pass, prominent in the Battle of Long Island.\\nFlushing Meadows–Corona Park in Queens, with its 897 acres (363 ha) making it the city\\'s fourth largest park, was the setting for the 1939 World\\'s Fair and the 1964 World\\'s Fair and is host to the USTA Billie Jean King National Tennis Center and the annual U.S. Open Tennis Championships tournament.\\nOver a fifth of the Bronx\\'s area, 7,000 acres (28 km2), is dedicated to open space and parks, including Pelham Bay Park, Van Cortlandt Park, the Bronx Zoo, and the New York Botanical Gardens.\\nIn Staten Island, the Conference House Park contains the historic Conference House, site of the only attempt of a peaceful resolution to the American Revolution which was conducted in September 1775, attended by Benjamin Franklin representing the Americans and Lord Howe representing the British Crown. The historic Burial Ridge, the largest Native American burial ground within New York City, is within the park.\\n\\n\\n=== Military installations ===\\nBrooklyn is home to Fort Hamilton, the U.S. military\\'s only active duty installation within New York City, aside from Coast Guard operations. The facility was established in 1825 on the site of a small battery used during the American Revolution, and it is one of America\\'s longest serving military forts. Today, Fort Hamilton serves as the headquarters of the North Atlantic Division of the United States Army Corps of Engineers and for the New York City Recruiting Battalion. It also houses the 1179th Transportation Brigade, the 722nd Aeromedical Staging Squadron, and a military entrance processing station. Other formerly active military reservations still used for National Guard and military training or reserve operations in the city include Fort Wadsworth in Staten Island and Fort Totten in Queens.\\n\\n\\n== '], response=None, multi_responses=None, reference='Brooklyn is distinct within New York City due to its cultural, social, and ethnic diversity, independent art scene, distinct neighborhoods, and distinctive architectural heritage. It is also known for its long beachfront shoreline, including Coney Island, and its large parks like Marine Park and Prospect Park. Additionally, Brooklyn has evolved into a hub of entrepreneurship and high technology startup firms, as well as postmodern art and design.', rubric=None), SingleTurnSample(user_input='What measures has New York City implemented to reduce its carbon footprint and environmental impact?', retrieved_contexts=None, reference_contexts=['Environment ==\\n\\n \\nEnvironmental issues in New York City are affected by the city\\'s size, density, abundant public transportation infrastructure, and its location at the mouth of the Hudson River. For example, it is one of the country\\'s biggest sources of pollution and has the lowest per-capita greenhouse gas emissions rate and electricity usage. Governors Island is planned to host a US$1 billion research and education center to make New York City the global leader in addressing the climate crisis.\\n\\n\\n=== Environmental impact reduction ===\\nNew York City has focused on reducing its environmental impact and carbon footprint. Mass transit use in New York City is the highest in the United States. Also, by 2010, the city had 3,715 hybrid taxis and other clean diesel vehicles, representing around 28% of New York\\'s taxi fleet in service, the most of any city in North America. New York City is the host of Climate Week NYC, the largest Climate Week to take place globally and regarded as major annual climate summit.\\nNew York\\'s high rate of public transit use, more than 200,000 daily cyclists as of 2014, and many pedestrian commuters make it the most energy-efficient major city in the United States. Walk and bicycle modes of travel account for 21% of all modes for trips in the city; nationally the rate for metro regions is about 8%. In both its 2011 and 2015 rankings, Walk Score named New York City the most walkable large city in the United States, and in 2018, Stacker ranked New York the most walkable U.S. city. Citibank sponsored the introduction of 10,000 public bicycles for the city\\'s bike-share project in the summer of 2013. New York City\\'s numerical \"in-season cycling indicator\" of bicycling in the city had hit an all-time high of 437 when measured in 2014.The city government was a petitioner in the landmark Massachusetts v. Environmental Protection Agency Supreme Court case forcing the EPA to regulate greenhouse gases as pollutants. The city is a leader in the construction of energy-efficient green office buildings, including the Hearst Tower among others. Mayor Bill de Blasio has committed to an 80% reduction in greenhouse gas emissions between 2014 and 2050 to reduce the city\\'s contributions to climate change, beginning with a comprehensive \"Green Buildings\" plan.\\n\\n\\n=== Water purity and availability ===\\n\\nThe New York City drinking water supply is extracted from the protected Catskill Mountains watershed. As a result of the watershed\\'s integrity and undisturbed natural water filtration system, New York is one of only four major cities in the United States the majority of whose drinking water is pure enough not to require purification through water treatment plants. The city\\'s municipal water system is the largest in the United States, moving over one billion gallons of water per day; a leak in the Delaware aqueduct results in some 20 million gallons a day being lost under the Hudson River. The Croton Watershed north of the city is undergoing construction of a $3.2 billion water purification plant to augment New York City\\'s water supply by an estimated 290 million gallons daily, representing a greater than 20% addition to the city\\'s current availability of water. The ongoing expansion of New York City Water Tunnel No. 3, an integral part of the New York City water supply system, is the largest capital construction project in the city\\'s history, with segments serving Manhattan and the Bronx completed, and with segments serving Brooklyn and Queens planned for construction in 2020. In 2018, New York City announced a $1 billion investment to protect the integrity of its water system and to maintain the purity of its unfiltered water supply.\\n\\n\\n=== Air quality ===\\nAccording to the 2016 World Health Organization Global Urban Ambient Air Pollution Database, the annual average concentration in New York City\\'s air of particulate matter measuring 2.5 micrometers or less (PM2.5) was 7.0 micrograms per cubic meter, or 3.0 micrograms within the recommended limit of the WHO Air Quality Guidelines for the annual mean PM2.5. The New York City Department of Health and Mental Hygiene, in partnership with Queens College, conducts the New York Community Air Survey to measure pollutants at about 150 locations.\\n\\n\\n=== Environmental revitalization ===\\nNewtown Creek, a 3.5-mile (6-kilometer) a long estuary that forms part of the border between the boroughs of Brooklyn and Queens, has been designated a Superfund site for environmental clean-up and remediation of the waterway\\'s recreational and economic resources for many communities. One of the most heavily used bodies of water in the Port of New York and New Jersey, it had been one of the most contaminated industrial sites in the country, containing years of discarded toxins, an estimated 30 million US gallons (110,000 m3) of spilled oil, including the Greenpoint oil spill, raw sewage from New York City\\'s sewer system, and other accumulation.\\n\\n\\n== '], response=None, multi_responses=None, reference='New York City has implemented several measures to reduce its carbon footprint and environmental impact, including the highest mass transit use in the United States, a significant number of hybrid taxis and clean diesel vehicles, hosting Climate Week NYC, promoting cycling and pedestrian commuting, constructing energy-efficient green office buildings, and committing to an 80% reduction in greenhouse gas emissions by 2050. Additionally, the city has introduced a bike-share project and has been involved in legal actions to regulate greenhouse gases as pollutants.', rubric=None), SingleTurnSample(user_input='What role did New York City play during the American Revolution?', retrieved_contexts=None, reference_contexts=['History ==\\n\\n\\n=== Early history ===\\nIn the pre-Columbian era, the area of present-day New York City was inhabited by Algonquian Native Americans, including the Lenape. Their homeland, known as Lenapehoking, included the present-day areas of Staten Island, Manhattan, the Bronx, the western portion of Long Island (including the areas that would later become the boroughs of Brooklyn and Queens), and the Lower Hudson Valley.The first documented visit into New York Harbor by a European was in 1524 by Italian Giovanni da Verrazzano, an explorer from Florence in the service of the French crown. He claimed the area for France and named it Nouvelle Angoulême (New Angoulême). A Spanish expedition, led by the Portuguese captain Estêvão Gomes sailing for Emperor Charles V, arrived in New York Harbor in January 1525 and charted the mouth of the Hudson River, which he named Río de San Antonio (\\'Saint Anthony\\'s River\\'). The Padrón Real of 1527, the first scientific map to show the East Coast of North America continuously, was informed by Gomes\\' expedition and labeled the northeastern United States as Tierra de Esteban Gómez in his honor.In 1609, the English explorer Henry Hudson rediscovered New York Harbor while searching for the Northwest Passage to the Orient for the Dutch East India Company. He proceeded to sail up what the Dutch would name the North River (now the Hudson River), named first by Hudson as the Mauritius after Maurice, Prince of Orange. Hudson\\'s first mate described the harbor as \"a very good Harbour for all windes\" and the river as \"a mile broad\" and \"full of fish\". Hudson sailed roughly 150 miles (240 km) north, past the site of the present-day New York State capital city of Albany, in the belief that it might be an oceanic tributary before the river became too shallow to continue. He made a ten-day exploration of the area and claimed the region for the Dutch East India Company. In 1614, the area between Cape Cod and Delaware Bay was claimed by the Netherlands and called Nieuw-Nederland (\\'New Netherland\\').\\nThe first non–Native American inhabitant of what would eventually become New York City was Juan Rodriguez (transliterated to the Dutch language as Jan Rodrigues), a merchant from Santo Domingo. Born in Santo Domingo of Portuguese and African descent, he arrived in Manhattan during the winter of 1613–14, trapping for pelts and trading with the local population as a representative of the Dutch. Broadway, from 159th Street to 218th Street in Upper Manhattan, is named Juan Rodriguez Way in his honor.\\n\\n\\n=== Dutch rule ===\\n\\nA permanent European presence near New York Harbor was established in 1624, making New York the 12th-oldest continuously occupied European-established settlement in the continental United States—with the founding of a Dutch fur trading settlement on Governors Island. In 1625, construction was started on a citadel and Fort Amsterdam, later called Nieuw Amsterdam (New Amsterdam), on present-day Manhattan Island. The colony of New Amsterdam was centered on what would ultimately be known as Lower Manhattan. Its area extended from the southern tip of Manhattan to modern day Wall Street, where a 12-foot wooden stockade was built in 1653 to protect against Native American and British raids. In 1626, the Dutch colonial Director-General Peter Minuit, acting as charged by the Dutch West India Company, purchased the island of Manhattan from the Canarsie, a small Lenape band, for \"the value of 60 guilders\" (about $900 in 2018). A disproved legend claims that Manhattan was purchased for $24 worth of glass beads.Following the purchase, New Amsterdam grew slowly. To attract settlers, the Dutch instituted the patroon system in 1628, whereby wealthy Dutchmen (patroons, or patrons) who brought 50 colonists to New Netherland would be awarded swaths of land, along with local political autonomy and rights to participate in the lucrative fur trade. This program had little success.Since 1621, the Dutch West India Company had operated as a monopoly in New Netherland, on authority granted by the Dutch States General. In 1639–1640, in an effort to bolster economic growth, the Dutch West India Company relinquished its monopoly over the fur trade, leading to growth in the production and trade of food, timber, tobacco, and slaves (particularly with the Dutch West Indies).In 1647, Peter Stuyvesant began his tenure as the last Director-General of New Netherland. During his tenure, the population of New Netherland grew from 2,000 to 8,000. Stuyvesant has been credited with improving law and order in the colony; however, he also earned a reputation as a despotic leader. He instituted regulations on liquor sales, attempted to assert control over the Dutch Reformed Church, and blocked other religious groups (including Quakers, Jews, and Lutherans) from establishing houses of worship. The Dutch West India Company would eventually attempt to ease tensions between Stuyvesant and residents of New Amsterdam.\\n\\n\\n=== English rule ===\\n\\nIn 1664, unable to summon any significant resistance, Stuyvesant surrendered New Amsterdam to English troops, led by Colonel Richard Nicolls, without bloodshed. The terms of the surrender permitted Dutch residents to remain in the colony and allowed for religious freedom. In 1667, during negotiations leading to the Treaty of Breda after the Second Anglo-Dutch War, the Dutch decided to keep the nascent plantation colony of what is now Suriname (on the northern South American coast) they had gained from the English; and in return, the English kept New Amsterdam. The fledgling settlement was promptly renamed \"New York\" after the Duke of York (the future King James II and VII), who would eventually be deposed in the Glorious Revolution. After the founding, the duke gave part of the colony to proprietors George Carteret and John Berkeley. Fort Orange, 150 miles (240 km) north on the Hudson River, was renamed Albany after James\\'s Scottish title. The transfer was confirmed in 1667 by the Treaty of Breda, which concluded the Second Anglo-Dutch War.On August 24, 1673, during the Third Anglo-Dutch War, Dutch captain Anthony Colve seized the colony of New York from the English at the behest of Cornelis Evertsen the Youngest and rechristened it \"New Orange\" after William III, the Prince of Orange. The Dutch would soon return the island to England under the Treaty of Westminster of November 1674.Several intertribal wars among the Native Americans and some epidemics brought on by contact with the Europeans caused sizeable population losses for the Lenape between the years 1660 and 1670. By 1700, the Lenape population had diminished to 200. New York experienced several yellow fever epidemics in the 18th century, losing ten percent of its population to the disease in 1702 alone.\\n\\n\\n=== Province of New York and slavery ===\\n\\nIn the early 18th century, New York grew in importance as a trading port while as a part of the colony of New York. It also became a center of slavery, with 42% of households enslaving Africans by 1730, the highest percentage outside Charleston, South Carolina. Most cases were that of domestic slavery, as a New York household then commonly enslaved few or several people. Others were hired out to work at labor. Slavery became integrally tied to New York\\'s economy through the labor of slaves throughout the port, and the banking and shipping industries trading with the American South. During construction in Foley Square in the 1990s, the African Burying Ground was discovered; the cemetery included 10,000 to 20,000 of graves of colonial-era Africans, some enslaved and some free.The 1735 trial and acquittal in Manhattan of John Peter Zenger, who had been accused of seditious libel after criticizing colonial governor William Cosby, helped to establish the freedom of the press in North America. In 1754, Columbia University was founded under charter by King George II as King\\'s College in Lower Manhattan.\\n\\n\\n=== American Revolution ===\\n\\nThe Stamp Act Congress met in New York in October 1765, as the Sons of Liberty organization emerged in the city and skirmished over the next ten years with British troops stationed there. The Battle of Long Island, the largest battle of the American Revolutionary War, was fought in August 1776 within the modern-day borough of Brooklyn. After the battle, in which the Americans were defeated, the British made the city their military and political base of operations in North America. The city was a haven for Loyalist refugees and escaped slaves who joined the British lines for freedom newly promised by the Crown for all fighters. As many as 10,000 escaped slaves crowded into the city during the British occupation. When the British forces evacuated at the close of the war in 1783, they transported 3,000 freedmen for resettlement in Nova Scotia. They resettled other freedmen in England and the Caribbean.\\nThe only attempt at a peaceful solution to the war took place at the Conference House on Staten Island between American delegates, including Benjamin Franklin, and British general Lord Howe on September 11, 1776. Shortly after the British occupation began, the Great Fire of New York occurred, a large conflagration on the West Side of Lower Manhattan, which destroyed about a quarter of the buildings in the city, including Trinity Church.In 1785, the assembly of the Congress of the Confederation made New York City the national capital shortly after the war. New York was the last capital of the U.S. under the Articles of Confederation and the first capital under the Constitution of the United States. New York City as the U.S. capital hosted several events of national scope in 1789—the first President of the United States, George Washington, was inaugurated; the first United States Congress and the Supreme Court of the United States each assembled for the first time; and the United States Bill of Rights was drafted, all at Federal Hall on Wall Street. In 1790, New York surpassed Philadelphia as the nation\\'s largest city. At the end of that year, pursuant to the Residence Act, the national capital was moved to Philadelphia.\\n\\n\\n=== 19th century ===\\n\\nOver the course of the nineteenth century, New York City\\'s population grew from 60,000 to 3.43 million. Under New York State\\'s abolition act of 1799, children of slave mothers were to be eventually liberated but to be held in indentured servitude until their mid-to-late twenties. Together with slaves freed by their masters after the Revolutionary War and escaped slaves, a significant free-Black population gradually developed in Manhattan. Under such influential United States founders as Alexander Hamilton and John Jay, the New York Manumission Society worked for abolition and established the African Free School to educate Black children. It was not until 1827 that slavery was completely abolished in the state, and free Blacks struggled afterward with discrimination. New York interracial abolitionist activism continued; among its leaders were graduates of the African Free School. New York city\\'s population jumped from 123,706 in 1820 to 312,710 by 1840, 16,000 of whom were Black.In the 19th century, the city was transformed by both commercial and residential development relating to its status as a national and international trading center, as well as by European immigration, respectively. The city adopted the Commissioners\\' Plan of 1811, which expanded the city street grid to encompass almost all of Manhattan. The 1825 completion of the Erie Canal through central New York connected the Atlantic port to the agricultural markets and commodities of the North American interior via the Hudson River and the Great Lakes. Local politics became dominated by Tammany Hall, a political machine supported by Irish and German immigrants.Several prominent American literary figures lived in New York during the 1830s and 1840s, including William Cullen Bryant, Washington Irving, Herman Melville, Rufus Wilmot Griswold, John Keese, Nathaniel Parker Willis, and Edgar Allan Poe. Public-minded members of the contemporaneous business elite lobbied for the establishment of Central Park, which in 1857 became the first landscaped park in an American city.\\nThe Great Irish Famine brought a large influx of Irish immigrants; more than 200,000 were living in New York by 1860, upwards of a quarter of the city\\'s population. There was also extensive immigration from the German provinces, where revolutions had disrupted societies, and Germans comprised another 25% of New York\\'s population by 1860.Democratic Party candidates were consistently elected to local office, increasing the city\\'s ties to the South and its dominant party. In 1861, Mayor Fernando Wood called upon the aldermen to declare independence from Albany and the United States after the South seceded, but his proposal was not acted on. Anger at new military conscription laws during the American Civil War (1861–1865), which spared wealthier men who could afford to pay a $300 (equivalent to $6,602 in 2021) commutation fee to hire a substitute, led to the Draft Riots of 1863, whose most visible participants were ethnic Irish working class.The draft riots deteriorated into attacks on New York\\'s elite, followed by attacks on Black New Yorkers and their property after fierce competition for a decade between Irish immigrants and Black people for work. Rioters burned the Colored Orphan Asylum to the ground, with more than 200 children escaping harm due to efforts of the New York Police Department, which was mainly made up of Irish immigrants. At least 120 people were killed. Eleven Black men were lynched over five days, and the riots forced hundreds of Blacks to flee the city for Williamsburg, Brooklyn, and New Jersey. The Black population in Manhattan fell below 10,000 by 1865, which it had last been in 1820. The White working class had established dominance. Violence by longshoremen against Black men was especially fierce in the docks area. It was one of the worst incidents of civil unrest in American history.In 1898, the City of New York was formed with the consolidation of Brooklyn (until then a separate city), the County of New York (which then included parts of the Bronx), the County of Richmond, and the western portion of the County of Queens. The opening of the subway in 1904, first built as separate private systems, helped bind the new city together. Throughout the first half of the 20th century, the city became a world center for industry, commerce, and communication.\\n\\n\\n=== 20th century ===\\n\\nIn 1904, the steamship General Slocum caught fire in the East River, killing 1,021 people on board. In 1911, the Triangle Shirtwaist Factory fire, the city\\'s worst industrial disaster, took the lives of 146 garment workers and spurred the growth of the International Ladies\\' Garment Workers\\' Union and major improvements in factory safety standards.New York\\'s non-White population was 36,620 in 1890. New York City was a prime destination in the early twentieth century for African Americans during the Great Migration from the American South, and by 1916, New York City had become home to the largest urban African diaspora in North America. The Harlem Renaissance of literary and cultural life flourished during the era of Prohibition. The larger economic boom generated construction of skyscrapers competing in height and creating an identifiable skyline.\\nNew York became the most populous urbanized area in the world in the early 1920s, overtaking London. The metropolitan area surpassed the 10 million mark in the early 1930s, becoming the first megacity in human history. The Great Depression saw the election of reformer Fiorello La Guardia as mayor and the fall of Tammany Hall after eighty years of political dominance.Returning World War II veterans created a post-war economic boom and the development of large housing tracts in eastern Queens and Nassau County as well as similar suburban areas in New Jersey. New York emerged from the war unscathed as the leading city of the world, with Wall Street leading America\\'s place as the world\\'s dominant economic power. The United Nations headquarters was completed in 1952, solidifying New York\\'s global geopolitical influence, and the rise of abstract expressionism in the city precipitated New York\\'s displacement of Paris as the center of the art world.The Stonewall riots were a series of spontaneous, violent protests by members of the gay community against a police raid that took place in the early morning hours of June 28, 1969, at the Stonewall Inn in the Greenwich Village neighborhood of Lower Manhattan. They are widely considered to constitute the single most important event leading to the gay liberation movement and the modern fight for LGBT rights. Wayne R. Dynes, author of the Encyclopedia of Homosexuality, wrote that drag queens were the only \"transgender folks around\" during the June 1969 Stonewall riots. The transgender community in New York City played a significant role in fighting for LGBT equality during the period of the Stonewall riots and thereafter.In the 1970s, job losses due to industrial restructuring caused New York City to suffer from economic problems and rising crime rates. While a resurgence in the financial industry greatly improved the city\\'s economic health in the 1980s, New York\\'s crime rate continued to increase through that decade and into the beginning of the 1990s. By the mid 1990s, crime rates started to drop dramatically due to revised police strategies, improving economic opportunities, gentrification, and new residents, both American transplants and new immigrants from Asia and Latin America. Important new sectors, such as Silicon Alley, emerged in the city\\'s economy.\\n\\n\\n=== 21st century ===\\n\\nNew York\\'s population reached all-time highs in the 2000 census and then again in the 2010 census.\\nNew York City suffered the bulk of the economic damage and largest loss of human life in the aftermath of the September 11, 2001, attacks. Two of the four airliners hijacked that day were flown into the twin towers of the World Trade Center, destroying the towers and killing 2,192 civilians, 343 firefighters, and 71 law enforcement officers. The North Tower became the tallest building ever to be destroyed anywhere then or subsequently.The area was rebuilt with a new One World Trade Center, a 9/11 memorial and museum, and other new buildings and infrastructure. The World Trade Center PATH station, which had opened on July 19, 1909, as the Hudson Terminal, was also destroyed in the attacks. A temporary station was built and opened on November 23, 2003. An 800,000-square-foot (74,000 m2) permanent rail station designed by Santiago Calatrava, the World Trade Center Transportation Hub, the city\\'s third-largest hub, was completed in 2016. The new One World Trade Center is the tallest skyscraper in the Western Hemisphere and the seventh-tallest building in the world by pinnacle height, with its spire reaching a symbolic 1,776 feet (541.3 m) in reference to the year of U.S. independence.The Occupy Wall Street protests in Zuccotti Park in the Financial District of Lower Manhattan began on September 17, 2011, receiving global attention and popularizing the Occupy movement against social and economic inequality worldwide.New York City was heavily affected by Hurricane Sandy in late October 2012. Sandy\\'s impacts included the flooding of the New York City Subway system, of many suburban communities, and of all road tunnels entering Manhattan except the Lincoln Tunnel. The New York Stock Exchange closed for two consecutive days. Numerous homes and businesses were destroyed by fire, including over 100 homes in Breezy Point, Queens. Large parts of the city and surrounding areas lost electricity for several days. Several thousand people in Midtown Manhattan were evacuated for six days due to a crane collapse at Extell\\'s One57. Bellevue Hospital Center and a few other large hospitals were closed and evacuated. Flooding at 140 West Street and another exchange disrupted voice and data communication in Lower Manhattan. At least 43 people lost their lives in New York City as a result of Sandy, and the economic losses in New York City were estimated to be roughly $19 billion. The disaster spawned long-term efforts towards infrastructural projects to counter climate change and rising seas.In March 2020, the first case of COVID-19 in the city was confirmed in Manhattan. The city rapidly replaced Wuhan, China to become the global epicenter of the pandemic during the early phase, before the infection became widespread across the world and the rest of the nation. As of March 2021, New York City had recorded over 30,000 deaths from COVID-19-related complications. In 2022, the LGBT community in New York City became the epicenter of the monkeypox outbreak in the Western Hemisphere, prompting New York Governor Kathy Hochul and New York City Mayor Eric Adams declared corresponding public health emergencies in the state and city, respectively, in July 2022.\\n\\n\\n== '], response=None, multi_responses=None, reference='During the American Revolution, New York City played a significant role as a military and political base for the British after the Battle of Long Island in 1776. The city served as a haven for Loyalist refugees and escaped slaves who joined the British lines. It was also the site of the only attempt at a peaceful resolution to the war at the Conference House on Staten Island. After the war, New York City became the national capital under the Articles of Confederation and hosted several important events, including the inauguration of George Washington as the first President of the United States.', rubric=None)])" ] }, - "execution_count": 23, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# convert to HF dataset\n", - "ds = testset.to_dataset()\n", - "\n", - "ds_dict = ds.to_dict()\n", - "ds_dict[\"question\"]\n", - "ds_dict[\"ground_truth\"]" + "# convert to Ragas Evaluation Dataset\n", + "ragas_dataset = testset.to_evaluation_dataset()\n", + "ragas_dataset" ] }, { @@ -466,19 +544,19 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 18, "id": "05633cc2", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "261183e30af84047a6d8c18fdbef1d72", + "model_id": "97e71379c54d4657b06d1b19a8e100fa", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Running Query Engine: 0%| | 0/5 [00:00\n", " \n", " \n", - " question\n", - " contexts\n", - " answer\n", - " ground_truth\n", + " user_input\n", + " retrieved_contexts\n", + " reference_contexts\n", + " response\n", + " reference\n", " faithfulness\n", " answer_relevancy\n", " context_precision\n", " context_recall\n", - " harmfulness\n", " \n", " \n", " \n", " \n", " 0\n", - " What cultural movement began in New York City ...\n", - " [=== 19th century ===\\n\\nOver the course of th...\n", - " The Harlem Renaissance of literary and cultura...\n", - " The Harlem Renaissance\n", - " 0.5\n", - " 0.907646\n", - " 0.5\n", + " What events led to New York being named after ...\n", + " [New York City is the headquarters of the glob...\n", + " [Etymology ==\\n\\nIn 1664, New York was named i...\n", + " New York was named in honor of the Duke of Yor...\n", + " New York was named after the Duke of York in 1...\n", + " 1.000000\n", + " 0.950377\n", + " 1.0\n", " 1.0\n", - " 0\n", " \n", " \n", " 1\n", - " What is the significance of New York City's tr...\n", - " [== Transportation ==\\n\\nNew York City's compr...\n", - " New York City's transportation system is signi...\n", - " New York City's transportation system is both ...\n", - " 1.0\n", - " 0.986921\n", - " 1.0\n", + " How early European explorers and Native Americ...\n", + " [=== Dutch rule ===\\n\\nA permanent European pr...\n", + " [History ==\\n\\n\\n=== Early history ===\\nIn the...\n", + " Early European explorers established a permane...\n", + " Early European explorers and Native Americans ...\n", + " 1.000000\n", + " 0.896300\n", " 1.0\n", - " 0\n", + " 0.8\n", " \n", " \n", " 2\n", - " What factors led to the creation of Central Pa...\n", - " [=== 19th century ===\\n\\nOver the course of th...\n", - " Prominent American literary figures lived in N...\n", - " Public-minded members of the contemporaneous b...\n", - " 1.0\n", - " 0.805014\n", - " 1.0\n", + " New York City population economy challenges\n", + " [=== Wealth and income disparity ===\\nNew York...\n", + " [New York City, the most populous city in the ...\n", + " New York City has faced challenges related to ...\n", + " New York City, as the most populous city in th...\n", + " 1.000000\n", + " 0.915717\n", " 1.0\n", - " 0\n", + " 0.0\n", " \n", " \n", " 3\n", - " What was the impact of the Treaty of Breda on ...\n", - " [=== Dutch rule ===\\n\\nA permanent European pr...\n", - " The Treaty of Breda resulted in the transfer o...\n", - " The Treaty of Breda confirmed the transfer of ...\n", + " How do the economic aspects of New York City, ...\n", + " [=== Wealth and income disparity ===\\nNew York...\n", + " [New York City, the most populous city in the ...\n", + " The economic aspects of New York City, as a gl...\n", + " New York City's economic aspects as a global c...\n", + " 0.913043\n", + " 0.929317\n", " 1.0\n", - " 0.860931\n", + " 0.0\n", + " \n", + " \n", + " 4\n", + " What are some of the cultural and architectura...\n", + " [==== Staten Island ====\\nStaten Island (Richm...\n", + " [Geography ==\\n\\nDuring the Wisconsin glaciati...\n", + " Brooklyn is known for its cultural diversity, ...\n", + " Brooklyn is distinct within New York City due ...\n", + " 1.000000\n", + " 0.902664\n", + " 0.5\n", + " 1.0\n", + " \n", + " \n", + " 5\n", + " What measures has New York City implemented to...\n", + " [==== International events ====\\nIn terms of h...\n", + " [Environment ==\\n\\n \\nEnvironmental issues in ...\n", + " New York City has implemented various measures...\n", + " New York City has implemented several measures...\n", + " 0.909091\n", + " 1.000000\n", " 1.0\n", " 1.0\n", - " 0\n", " \n", " \n", - " 4\n", - " What role did New York play in the American Re...\n", + " 6\n", + " What role did New York City play during the Am...\n", " [=== Province of New York and slavery ===\\n\\nI...\n", - " New York served as a significant location duri...\n", - " New York played a significant role in the Amer...\n", + " [History ==\\n\\n\\n=== Early history ===\\nIn the...\n", + " New York City served as a significant military...\n", + " During the American Revolution, New York City ...\n", + " 1.000000\n", + " 1.000000\n", " 1.0\n", - " 0.935846\n", " 1.0\n", - " 1.0\n", - " 0\n", " \n", " \n", "\n", "" ], "text/plain": [ - " question \\\n", - "0 What cultural movement began in New York City ... \n", - "1 What is the significance of New York City's tr... \n", - "2 What factors led to the creation of Central Pa... \n", - "3 What was the impact of the Treaty of Breda on ... \n", - "4 What role did New York play in the American Re... \n", + " user_input \\\n", + "0 What events led to New York being named after ... \n", + "1 How early European explorers and Native Americ... \n", + "2 New York City population economy challenges \n", + "3 How do the economic aspects of New York City, ... \n", + "4 What are some of the cultural and architectura... \n", + "5 What measures has New York City implemented to... \n", + "6 What role did New York City play during the Am... \n", + "\n", + " retrieved_contexts \\\n", + "0 [New York City is the headquarters of the glob... \n", + "1 [=== Dutch rule ===\\n\\nA permanent European pr... \n", + "2 [=== Wealth and income disparity ===\\nNew York... \n", + "3 [=== Wealth and income disparity ===\\nNew York... \n", + "4 [==== Staten Island ====\\nStaten Island (Richm... \n", + "5 [==== International events ====\\nIn terms of h... \n", + "6 [=== Province of New York and slavery ===\\n\\nI... \n", "\n", - " contexts \\\n", - "0 [=== 19th century ===\\n\\nOver the course of th... \n", - "1 [== Transportation ==\\n\\nNew York City's compr... \n", - "2 [=== 19th century ===\\n\\nOver the course of th... \n", - "3 [=== Dutch rule ===\\n\\nA permanent European pr... \n", - "4 [=== Province of New York and slavery ===\\n\\nI... \n", + " reference_contexts \\\n", + "0 [Etymology ==\\n\\nIn 1664, New York was named i... \n", + "1 [History ==\\n\\n\\n=== Early history ===\\nIn the... \n", + "2 [New York City, the most populous city in the ... \n", + "3 [New York City, the most populous city in the ... \n", + "4 [Geography ==\\n\\nDuring the Wisconsin glaciati... \n", + "5 [Environment ==\\n\\n \\nEnvironmental issues in ... \n", + "6 [History ==\\n\\n\\n=== Early history ===\\nIn the... \n", "\n", - " answer \\\n", - "0 The Harlem Renaissance of literary and cultura... \n", - "1 New York City's transportation system is signi... \n", - "2 Prominent American literary figures lived in N... \n", - "3 The Treaty of Breda resulted in the transfer o... \n", - "4 New York served as a significant location duri... \n", + " response \\\n", + "0 New York was named in honor of the Duke of Yor... \n", + "1 Early European explorers established a permane... \n", + "2 New York City has faced challenges related to ... \n", + "3 The economic aspects of New York City, as a gl... \n", + "4 Brooklyn is known for its cultural diversity, ... \n", + "5 New York City has implemented various measures... \n", + "6 New York City served as a significant military... \n", "\n", - " ground_truth faithfulness \\\n", - "0 The Harlem Renaissance 0.5 \n", - "1 New York City's transportation system is both ... 1.0 \n", - "2 Public-minded members of the contemporaneous b... 1.0 \n", - "3 The Treaty of Breda confirmed the transfer of ... 1.0 \n", - "4 New York played a significant role in the Amer... 1.0 \n", + " reference faithfulness \\\n", + "0 New York was named after the Duke of York in 1... 1.000000 \n", + "1 Early European explorers and Native Americans ... 1.000000 \n", + "2 New York City, as the most populous city in th... 1.000000 \n", + "3 New York City's economic aspects as a global c... 0.913043 \n", + "4 Brooklyn is distinct within New York City due ... 1.000000 \n", + "5 New York City has implemented several measures... 0.909091 \n", + "6 During the American Revolution, New York City ... 1.000000 \n", "\n", - " answer_relevancy context_precision context_recall harmfulness \n", - "0 0.907646 0.5 1.0 0 \n", - "1 0.986921 1.0 1.0 0 \n", - "2 0.805014 1.0 1.0 0 \n", - "3 0.860931 1.0 1.0 0 \n", - "4 0.935846 1.0 1.0 0 " + " answer_relevancy context_precision context_recall \n", + "0 0.950377 1.0 1.0 \n", + "1 0.896300 1.0 0.8 \n", + "2 0.915717 1.0 0.0 \n", + "3 0.929317 1.0 0.0 \n", + "4 0.902664 0.5 1.0 \n", + "5 1.000000 1.0 1.0 \n", + "6 1.000000 1.0 1.0 " ] }, - "execution_count": 26, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } From a63649683aed871923340c65b065f61bc893b919 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Fri, 1 Nov 2024 11:12:54 +0530 Subject: [PATCH 7/9] feat: small tweaks and improvements --- src/ragas/dataset_schema.py | 3 +++ src/ragas/testset/synthesizers/generate.py | 23 +++++++++++++++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ragas/dataset_schema.py b/src/ragas/dataset_schema.py index 2403ec014..b79f800a2 100644 --- a/src/ragas/dataset_schema.py +++ b/src/ragas/dataset_schema.py @@ -344,6 +344,9 @@ def from_list(cls, data: t.List[t.Dict]) -> EvaluationDataset: samples.extend(SingleTurnSample(**sample) for sample in data) return cls(samples=samples) + def __repr__(self) -> str: + return f"EvaluationDataset(features={self.features()}, len={len(self.samples)})" + @dataclass class EvaluationResult: diff --git a/src/ragas/testset/synthesizers/generate.py b/src/ragas/testset/synthesizers/generate.py index 7cb50f889..f386a6128 100644 --- a/src/ragas/testset/synthesizers/generate.py +++ b/src/ragas/testset/synthesizers/generate.py @@ -86,7 +86,10 @@ def from_llama_index( llm: LlamaIndexLLM, embedding_model: LlamaIndexEmbedding, knowledge_graph: t.Optional[KnowledgeGraph] = None, - ) -> "TestsetGenerator": + ) -> TestsetGenerator: + """ + Creates a `TestsetGenerator` from a LlamaIndex LLM and embedding model. + """ knowledge_graph = knowledge_graph or KnowledgeGraph() return cls( LlamaIndexLLMWrapper(llm), @@ -163,8 +166,8 @@ def generate_with_llamaindex_docs( documents: t.Sequence[LlamaIndexDocument], testset_size: int, transforms: t.Optional[Transforms] = None, - transforms_llm: t.Optional[BaseRagasLLM] = None, - transforms_embedding_model: t.Optional[BaseRagasEmbeddings] = None, + transforms_llm: t.Optional[LlamaIndexLLM] = None, + transforms_embedding_model: t.Optional[LlamaIndexEmbedding] = None, query_distribution: t.Optional[QueryDistribution] = None, run_config: t.Optional[RunConfig] = None, callbacks: t.Optional[Callbacks] = None, @@ -188,9 +191,19 @@ def generate_with_llamaindex_docs( ) if not transforms: + if transforms_llm is None: + llm_for_transforms = self.llm + else: + llm_for_transforms = LlamaIndexLLMWrapper(transforms_llm) + if transforms_embedding_model is None: + embedding_model_for_transforms = self.embedding_model + else: + embedding_model_for_transforms = LlamaIndexEmbeddingsWrapper( + transforms_embedding_model + ) transforms = default_transforms( - llm=transforms_llm or self.llm, - embedding_model=transforms_embedding_model or self.embedding_model, + llm=llm_for_transforms, + embedding_model=embedding_model_for_transforms, ) # convert the documents to Ragas nodes From fedb6dfa0b3ff5e280c245fbf3a22a9f72f283b2 Mon Sep 17 00:00:00 2001 From: jjmachan Date: Fri, 1 Nov 2024 11:14:32 +0530 Subject: [PATCH 8/9] docs: improved documentation --- docs/howtos/integrations/llamaindex.ipynb | 88 +++++++++++----------- src/ragas/testset/synthesizers/generate.py | 35 ++++++++- 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/docs/howtos/integrations/llamaindex.ipynb b/docs/howtos/integrations/llamaindex.ipynb index 7ccef2ed8..471940f3f 100644 --- a/docs/howtos/integrations/llamaindex.ipynb +++ b/docs/howtos/integrations/llamaindex.ipynb @@ -47,7 +47,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "id": "e2107b62", "metadata": {}, "outputs": [], @@ -77,14 +77,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 3, "id": "fe03839d", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c79d70c876624617a0d8245bb57dfa34", + "model_id": "32696e6eb3f04785a9251312a5c3e82d", "version_major": 2, "version_minor": 0 }, @@ -98,7 +98,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ba81961f6f3d4bbf9a4bcb96d141afbc", + "model_id": "c8bb587423044017a0e983a6afcdfb52", "version_major": 2, "version_minor": 0 }, @@ -112,7 +112,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4cc614f70a11419db433f2968c73f3e8", + "model_id": "ed0f3309d44647809b2dad4972caea3a", "version_major": 2, "version_minor": 0 }, @@ -126,7 +126,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a251036797464dc9b7c2bf81fe04f93b", + "model_id": "9fce3c62541544a38e9d6654a18ba0d8", "version_major": 2, "version_minor": 0 }, @@ -141,15 +141,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "unable to apply transformation: Error code: 400 - {'error': {'message': \"This model's maximum context length is 8192 tokens, however you requested 27528 tokens (27528 in your prompt; 0 for the completion). Please reduce your prompt; or completion length.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n", "unable to apply transformation: Error code: 400 - {'error': {'message': \"This model's maximum context length is 8192 tokens, however you requested 14879 tokens (14879 in your prompt; 0 for the completion). Please reduce your prompt; or completion length.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n", + "unable to apply transformation: Error code: 400 - {'error': {'message': \"This model's maximum context length is 8192 tokens, however you requested 27528 tokens (27528 in your prompt; 0 for the completion). Please reduce your prompt; or completion length.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n", "unable to apply transformation: Error code: 400 - {'error': {'message': \"'$.input' is invalid. Please check the API reference: https://platform.openai.com/docs/api-reference.\", 'type': 'invalid_request_error', 'param': None, 'code': None}}\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e657c3d30a8e491db51713a7d130bd28", + "model_id": "859ff42bf5e54444bf1bd61bebb8f9a7", "version_major": 2, "version_minor": 0 }, @@ -164,13 +164,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "unable to apply transformation: Node d3a44301-4216-4805-9cad-7321a50eddcf has no embedding\n" + "unable to apply transformation: Node 9e1b2de8-684b-40e8-a477-438611fe1bcd has no embedding\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "894b12e58d0e4009bc8d89206b07a695", + "model_id": "b6ccb0c7cc9c4b0680d05569ecd0514c", "version_major": 2, "version_minor": 0 }, @@ -184,7 +184,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "f992f5c7b40f400588a353365dae633e", + "model_id": "624a55e08a0642648c9f0c2bdce079d4", "version_major": 2, "version_minor": 0 }, @@ -198,7 +198,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6fc96052895c43d1ad15e14c8b119f3d", + "model_id": "68736603ab5a40a292361f54261813e8", "version_major": 2, "version_minor": 0 }, @@ -212,7 +212,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9bdf6942baa843209f7ad9d4d3da3367", + "model_id": "82aa23a94e38467c88a5bfadbfcfcad6", "version_major": 2, "version_minor": 0 }, @@ -226,7 +226,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5404d40f329c4aa8b80fe3fcc9062ac4", + "model_id": "626375e02b964ffabf03d6f171b6e98b", "version_major": 2, "version_minor": 0 }, @@ -248,7 +248,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 4, "id": "0b75a723", "metadata": {}, "outputs": [ @@ -282,37 +282,37 @@ " \n", " \n", " 0\n", - " What events led to New York being named after ...\n", + " Why was New York named after the Duke of York?\n", " [Etymology ==\\n\\nIn 1664, New York was named i...\n", " New York was named after the Duke of York in 1...\n", " AbstractQuerySynthesizer\n", " \n", " \n", " 1\n", - " How early European explorers and Native Americ...\n", + " How did the early Europan exploraton and setle...\n", " [History ==\\n\\n\\n=== Early history ===\\nIn the...\n", - " Early European explorers and Native Americans ...\n", + " The early European exploration and settlement ...\n", " AbstractQuerySynthesizer\n", " \n", " \n", " 2\n", - " New York City population economy challenges\n", + " New York City population culture finance diver...\n", " [New York City, the most populous city in the ...\n", - " New York City, as the most populous city in th...\n", + " New York City is a global cultural, financial,...\n", " ComparativeAbstractQuerySynthesizer\n", " \n", " \n", " 3\n", " How do the economic aspects of New York City, ...\n", " [New York City, the most populous city in the ...\n", - " New York City's economic aspects as a global c...\n", + " New York City's economic aspects, such as its ...\n", " ComparativeAbstractQuerySynthesizer\n", " \n", " \n", " 4\n", - " What are some of the cultural and architectura...\n", - " [Geography ==\\n\\nDuring the Wisconsin glaciati...\n", - " Brooklyn is distinct within New York City due ...\n", + " What role do biomedical research institutions ...\n", + " [Education ==\\n\\n \\n\\nNew York City has the la...\n", + " Biomedical research institutions in New York C...\n", " SpecificQuerySynthesizer\n", " \n", " \n", @@ -321,25 +321,25 @@ ], "text/plain": [ " user_input \\\n", - "0 What events led to New York being named after ... \n", - "1 How early European explorers and Native Americ... \n", - "2 New York City population economy challenges \n", + "0 Why was New York named after the Duke of York? \n", + "1 How did the early Europan exploraton and setle... \n", + "2 New York City population culture finance diver... \n", "3 How do the economic aspects of New York City, ... \n", - "4 What are some of the cultural and architectura... \n", + "4 What role do biomedical research institutions ... \n", "\n", " reference_contexts \\\n", "0 [Etymology ==\\n\\nIn 1664, New York was named i... \n", "1 [History ==\\n\\n\\n=== Early history ===\\nIn the... \n", "2 [New York City, the most populous city in the ... \n", "3 [New York City, the most populous city in the ... \n", - "4 [Geography ==\\n\\nDuring the Wisconsin glaciati... \n", + "4 [Education ==\\n\\n \\n\\nNew York City has the la... \n", "\n", " reference \\\n", "0 New York was named after the Duke of York in 1... \n", - "1 Early European explorers and Native Americans ... \n", - "2 New York City, as the most populous city in th... \n", - "3 New York City's economic aspects as a global c... \n", - "4 Brooklyn is distinct within New York City due ... \n", + "1 The early European exploration and settlement ... \n", + "2 New York City is a global cultural, financial,... \n", + "3 New York City's economic aspects, such as its ... \n", + "4 Biomedical research institutions in New York C... \n", "\n", " synthesizer_name \n", "0 AbstractQuerySynthesizer \n", @@ -349,7 +349,7 @@ "4 SpecificQuerySynthesizer " ] }, - "execution_count": 8, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -381,7 +381,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "id": "37c4a1cb", "metadata": {}, "outputs": [], @@ -404,17 +404,17 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 6, "id": "895d95b2", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'What events led to New York being named after the Duke of York?'" + "'Why was New York named after the Duke of York?'" ] }, - "execution_count": 11, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -427,7 +427,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 7, "id": "a25026c2", "metadata": {}, "outputs": [ @@ -435,7 +435,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "New York was named in honor of the Duke of York because King Charles II appointed the Duke as proprietor of the former territory of New Netherland, which included the city of New Amsterdam, when England seized it from Dutch control.\n" + "New York was named after the Duke of York because in 1664, the city was named in honor of the Duke of York, who later became King James II of England.\n" ] } ], @@ -479,7 +479,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 8, "id": "9875132a", "metadata": {}, "outputs": [], @@ -513,17 +513,17 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 9, "id": "4b2a81ed", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "EvaluationDataset(samples=[SingleTurnSample(user_input='What events led to New York being named after the Duke of York?', retrieved_contexts=None, reference_contexts=[\"Etymology ==\\n\\nIn 1664, New York was named in honor of the Duke of York, who would become King James II of England. James's elder brother, King Charles II, appointed the Duke as proprietor of the former territory of New Netherland, including the city of New Amsterdam, when England seized it from Dutch control.\\n\\n\\n== \"], response=None, multi_responses=None, reference='New York was named after the Duke of York in 1664 when King Charles II appointed him as proprietor of the former territory of New Netherland, including the city of New Amsterdam, after England seized it from Dutch control.', rubric=None), SingleTurnSample(user_input='How early European explorers and Native Americans shape New York City?', retrieved_contexts=None, reference_contexts=['History ==\\n\\n\\n=== Early history ===\\nIn the pre-Columbian era, the area of present-day New York City was inhabited by Algonquian Native Americans, including the Lenape. Their homeland, known as Lenapehoking, included the present-day areas of Staten Island, Manhattan, the Bronx, the western portion of Long Island (including the areas that would later become the boroughs of Brooklyn and Queens), and the Lower Hudson Valley.The first documented visit into New York Harbor by a European was in 1524 by Italian Giovanni da Verrazzano, an explorer from Florence in the service of the French crown. He claimed the area for France and named it Nouvelle Angoulême (New Angoulême). A Spanish expedition, led by the Portuguese captain Estêvão Gomes sailing for Emperor Charles V, arrived in New York Harbor in January 1525 and charted the mouth of the Hudson River, which he named Río de San Antonio (\\'Saint Anthony\\'s River\\'). The Padrón Real of 1527, the first scientific map to show the East Coast of North America continuously, was informed by Gomes\\' expedition and labeled the northeastern United States as Tierra de Esteban Gómez in his honor.In 1609, the English explorer Henry Hudson rediscovered New York Harbor while searching for the Northwest Passage to the Orient for the Dutch East India Company. He proceeded to sail up what the Dutch would name the North River (now the Hudson River), named first by Hudson as the Mauritius after Maurice, Prince of Orange. Hudson\\'s first mate described the harbor as \"a very good Harbour for all windes\" and the river as \"a mile broad\" and \"full of fish\". Hudson sailed roughly 150 miles (240 km) north, past the site of the present-day New York State capital city of Albany, in the belief that it might be an oceanic tributary before the river became too shallow to continue. He made a ten-day exploration of the area and claimed the region for the Dutch East India Company. In 1614, the area between Cape Cod and Delaware Bay was claimed by the Netherlands and called Nieuw-Nederland (\\'New Netherland\\').\\nThe first non–Native American inhabitant of what would eventually become New York City was Juan Rodriguez (transliterated to the Dutch language as Jan Rodrigues), a merchant from Santo Domingo. Born in Santo Domingo of Portuguese and African descent, he arrived in Manhattan during the winter of 1613–14, trapping for pelts and trading with the local population as a representative of the Dutch. Broadway, from 159th Street to 218th Street in Upper Manhattan, is named Juan Rodriguez Way in his honor.\\n\\n\\n=== Dutch rule ===\\n\\nA permanent European presence near New York Harbor was established in 1624, making New York the 12th-oldest continuously occupied European-established settlement in the continental United States—with the founding of a Dutch fur trading settlement on Governors Island. In 1625, construction was started on a citadel and Fort Amsterdam, later called Nieuw Amsterdam (New Amsterdam), on present-day Manhattan Island. The colony of New Amsterdam was centered on what would ultimately be known as Lower Manhattan. Its area extended from the southern tip of Manhattan to modern day Wall Street, where a 12-foot wooden stockade was built in 1653 to protect against Native American and British raids. In 1626, the Dutch colonial Director-General Peter Minuit, acting as charged by the Dutch West India Company, purchased the island of Manhattan from the Canarsie, a small Lenape band, for \"the value of 60 guilders\" (about $900 in 2018). A disproved legend claims that Manhattan was purchased for $24 worth of glass beads.Following the purchase, New Amsterdam grew slowly. To attract settlers, the Dutch instituted the patroon system in 1628, whereby wealthy Dutchmen (patroons, or patrons) who brought 50 colonists to New Netherland would be awarded swaths of land, along with local political autonomy and rights to participate in the lucrative fur trade. This program had little success.Since 1621, the Dutch West India Company had operated as a monopoly in New Netherland, on authority granted by the Dutch States General. In 1639–1640, in an effort to bolster economic growth, the Dutch West India Company relinquished its monopoly over the fur trade, leading to growth in the production and trade of food, timber, tobacco, and slaves (particularly with the Dutch West Indies).In 1647, Peter Stuyvesant began his tenure as the last Director-General of New Netherland. During his tenure, the population of New Netherland grew from 2,000 to 8,000. Stuyvesant has been credited with improving law and order in the colony; however, he also earned a reputation as a despotic leader. He instituted regulations on liquor sales, attempted to assert control over the Dutch Reformed Church, and blocked other religious groups (including Quakers, Jews, and Lutherans) from establishing houses of worship. The Dutch West India Company would eventually attempt to ease tensions between Stuyvesant and residents of New Amsterdam.\\n\\n\\n=== English rule ===\\n\\nIn 1664, unable to summon any significant resistance, Stuyvesant surrendered New Amsterdam to English troops, led by Colonel Richard Nicolls, without bloodshed. The terms of the surrender permitted Dutch residents to remain in the colony and allowed for religious freedom. In 1667, during negotiations leading to the Treaty of Breda after the Second Anglo-Dutch War, the Dutch decided to keep the nascent plantation colony of what is now Suriname (on the northern South American coast) they had gained from the English; and in return, the English kept New Amsterdam. The fledgling settlement was promptly renamed \"New York\" after the Duke of York (the future King James II and VII), who would eventually be deposed in the Glorious Revolution. After the founding, the duke gave part of the colony to proprietors George Carteret and John Berkeley. Fort Orange, 150 miles (240 km) north on the Hudson River, was renamed Albany after James\\'s Scottish title. The transfer was confirmed in 1667 by the Treaty of Breda, which concluded the Second Anglo-Dutch War.On August 24, 1673, during the Third Anglo-Dutch War, Dutch captain Anthony Colve seized the colony of New York from the English at the behest of Cornelis Evertsen the Youngest and rechristened it \"New Orange\" after William III, the Prince of Orange. The Dutch would soon return the island to England under the Treaty of Westminster of November 1674.Several intertribal wars among the Native Americans and some epidemics brought on by contact with the Europeans caused sizeable population losses for the Lenape between the years 1660 and 1670. By 1700, the Lenape population had diminished to 200. New York experienced several yellow fever epidemics in the 18th century, losing ten percent of its population to the disease in 1702 alone.\\n\\n\\n=== Province of New York and slavery ===\\n\\nIn the early 18th century, New York grew in importance as a trading port while as a part of the colony of New York. It also became a center of slavery, with 42% of households enslaving Africans by 1730, the highest percentage outside Charleston, South Carolina. Most cases were that of domestic slavery, as a New York household then commonly enslaved few or several people. Others were hired out to work at labor. Slavery became integrally tied to New York\\'s economy through the labor of slaves throughout the port, and the banking and shipping industries trading with the American South. During construction in Foley Square in the 1990s, the African Burying Ground was discovered; the cemetery included 10,000 to 20,000 of graves of colonial-era Africans, some enslaved and some free.The 1735 trial and acquittal in Manhattan of John Peter Zenger, who had been accused of seditious libel after criticizing colonial governor William Cosby, helped to establish the freedom of the press in North America. In 1754, Columbia University was founded under charter by King George II as King\\'s College in Lower Manhattan.\\n\\n\\n=== American Revolution ===\\n\\nThe Stamp Act Congress met in New York in October 1765, as the Sons of Liberty organization emerged in the city and skirmished over the next ten years with British troops stationed there. The Battle of Long Island, the largest battle of the American Revolutionary War, was fought in August 1776 within the modern-day borough of Brooklyn. After the battle, in which the Americans were defeated, the British made the city their military and political base of operations in North America. The city was a haven for Loyalist refugees and escaped slaves who joined the British lines for freedom newly promised by the Crown for all fighters. As many as 10,000 escaped slaves crowded into the city during the British occupation. When the British forces evacuated at the close of the war in 1783, they transported 3,000 freedmen for resettlement in Nova Scotia. They resettled other freedmen in England and the Caribbean.\\nThe only attempt at a peaceful solution to the war took place at the Conference House on Staten Island between American delegates, including Benjamin Franklin, and British general Lord Howe on September 11, 1776. Shortly after the British occupation began, the Great Fire of New York occurred, a large conflagration on the West Side of Lower Manhattan, which destroyed about a quarter of the buildings in the city, including Trinity Church.In 1785, the assembly of the Congress of the Confederation made New York City the national capital shortly after the war. New York was the last capital of the U.S. under the Articles of Confederation and the first capital under the Constitution of the United States. New York City as the U.S. capital hosted several events of national scope in 1789—the first President of the United States, George Washington, was inaugurated; the first United States Congress and the Supreme Court of the United States each assembled for the first time; and the United States Bill of Rights was drafted, all at Federal Hall on Wall Street. In 1790, New York surpassed Philadelphia as the nation\\'s largest city. At the end of that year, pursuant to the Residence Act, the national capital was moved to Philadelphia.\\n\\n\\n=== 19th century ===\\n\\nOver the course of the nineteenth century, New York City\\'s population grew from 60,000 to 3.43 million. Under New York State\\'s abolition act of 1799, children of slave mothers were to be eventually liberated but to be held in indentured servitude until their mid-to-late twenties. Together with slaves freed by their masters after the Revolutionary War and escaped slaves, a significant free-Black population gradually developed in Manhattan. Under such influential United States founders as Alexander Hamilton and John Jay, the New York Manumission Society worked for abolition and established the African Free School to educate Black children. It was not until 1827 that slavery was completely abolished in the state, and free Blacks struggled afterward with discrimination. New York interracial abolitionist activism continued; among its leaders were graduates of the African Free School. New York city\\'s population jumped from 123,706 in 1820 to 312,710 by 1840, 16,000 of whom were Black.In the 19th century, the city was transformed by both commercial and residential development relating to its status as a national and international trading center, as well as by European immigration, respectively. The city adopted the Commissioners\\' Plan of 1811, which expanded the city street grid to encompass almost all of Manhattan. The 1825 completion of the Erie Canal through central New York connected the Atlantic port to the agricultural markets and commodities of the North American interior via the Hudson River and the Great Lakes. Local politics became dominated by Tammany Hall, a political machine supported by Irish and German immigrants.Several prominent American literary figures lived in New York during the 1830s and 1840s, including William Cullen Bryant, Washington Irving, Herman Melville, Rufus Wilmot Griswold, John Keese, Nathaniel Parker Willis, and Edgar Allan Poe. Public-minded members of the contemporaneous business elite lobbied for the establishment of Central Park, which in 1857 became the first landscaped park in an American city.\\nThe Great Irish Famine brought a large influx of Irish immigrants; more than 200,000 were living in New York by 1860, upwards of a quarter of the city\\'s population. There was also extensive immigration from the German provinces, where revolutions had disrupted societies, and Germans comprised another 25% of New York\\'s population by 1860.Democratic Party candidates were consistently elected to local office, increasing the city\\'s ties to the South and its dominant party. In 1861, Mayor Fernando Wood called upon the aldermen to declare independence from Albany and the United States after the South seceded, but his proposal was not acted on. Anger at new military conscription laws during the American Civil War (1861–1865), which spared wealthier men who could afford to pay a $300 (equivalent to $6,602 in 2021) commutation fee to hire a substitute, led to the Draft Riots of 1863, whose most visible participants were ethnic Irish working class.The draft riots deteriorated into attacks on New York\\'s elite, followed by attacks on Black New Yorkers and their property after fierce competition for a decade between Irish immigrants and Black people for work. Rioters burned the Colored Orphan Asylum to the ground, with more than 200 children escaping harm due to efforts of the New York Police Department, which was mainly made up of Irish immigrants. At least 120 people were killed. Eleven Black men were lynched over five days, and the riots forced hundreds of Blacks to flee the city for Williamsburg, Brooklyn, and New Jersey. The Black population in Manhattan fell below 10,000 by 1865, which it had last been in 1820. The White working class had established dominance. Violence by longshoremen against Black men was especially fierce in the docks area. It was one of the worst incidents of civil unrest in American history.In 1898, the City of New York was formed with the consolidation of Brooklyn (until then a separate city), the County of New York (which then included parts of the Bronx), the County of Richmond, and the western portion of the County of Queens. The opening of the subway in 1904, first built as separate private systems, helped bind the new city together. Throughout the first half of the 20th century, the city became a world center for industry, commerce, and communication.\\n\\n\\n=== 20th century ===\\n\\nIn 1904, the steamship General Slocum caught fire in the East River, killing 1,021 people on board. In 1911, the Triangle Shirtwaist Factory fire, the city\\'s worst industrial disaster, took the lives of 146 garment workers and spurred the growth of the International Ladies\\' Garment Workers\\' Union and major improvements in factory safety standards.New York\\'s non-White population was 36,620 in 1890. New York City was a prime destination in the early twentieth century for African Americans during the Great Migration from the American South, and by 1916, New York City had become home to the largest urban African diaspora in North America. The Harlem Renaissance of literary and cultural life flourished during the era of Prohibition. The larger economic boom generated construction of skyscrapers competing in height and creating an identifiable skyline.\\nNew York became the most populous urbanized area in the world in the early 1920s, overtaking London. The metropolitan area surpassed the 10 million mark in the early 1930s, becoming the first megacity in human history. The Great Depression saw the election of reformer Fiorello La Guardia as mayor and the fall of Tammany Hall after eighty years of political dominance.Returning World War II veterans created a post-war economic boom and the development of large housing tracts in eastern Queens and Nassau County as well as similar suburban areas in New Jersey. New York emerged from the war unscathed as the leading city of the world, with Wall Street leading America\\'s place as the world\\'s dominant economic power. The United Nations headquarters was completed in 1952, solidifying New York\\'s global geopolitical influence, and the rise of abstract expressionism in the city precipitated New York\\'s displacement of Paris as the center of the art world.The Stonewall riots were a series of spontaneous, violent protests by members of the gay community against a police raid that took place in the early morning hours of June 28, 1969, at the Stonewall Inn in the Greenwich Village neighborhood of Lower Manhattan. They are widely considered to constitute the single most important event leading to the gay liberation movement and the modern fight for LGBT rights. Wayne R. Dynes, author of the Encyclopedia of Homosexuality, wrote that drag queens were the only \"transgender folks around\" during the June 1969 Stonewall riots. The transgender community in New York City played a significant role in fighting for LGBT equality during the period of the Stonewall riots and thereafter.In the 1970s, job losses due to industrial restructuring caused New York City to suffer from economic problems and rising crime rates. While a resurgence in the financial industry greatly improved the city\\'s economic health in the 1980s, New York\\'s crime rate continued to increase through that decade and into the beginning of the 1990s. By the mid 1990s, crime rates started to drop dramatically due to revised police strategies, improving economic opportunities, gentrification, and new residents, both American transplants and new immigrants from Asia and Latin America. Important new sectors, such as Silicon Alley, emerged in the city\\'s economy.\\n\\n\\n=== 21st century ===\\n\\nNew York\\'s population reached all-time highs in the 2000 census and then again in the 2010 census.\\nNew York City suffered the bulk of the economic damage and largest loss of human life in the aftermath of the September 11, 2001, attacks. Two of the four airliners hijacked that day were flown into the twin towers of the World Trade Center, destroying the towers and killing 2,192 civilians, 343 firefighters, and 71 law enforcement officers. The North Tower became the tallest building ever to be destroyed anywhere then or subsequently.The area was rebuilt with a new One World Trade Center, a 9/11 memorial and museum, and other new buildings and infrastructure. The World Trade Center PATH station, which had opened on July 19, 1909, as the Hudson Terminal, was also destroyed in the attacks. A temporary station was built and opened on November 23, 2003. An 800,000-square-foot (74,000 m2) permanent rail station designed by Santiago Calatrava, the World Trade Center Transportation Hub, the city\\'s third-largest hub, was completed in 2016. The new One World Trade Center is the tallest skyscraper in the Western Hemisphere and the seventh-tallest building in the world by pinnacle height, with its spire reaching a symbolic 1,776 feet (541.3 m) in reference to the year of U.S. independence.The Occupy Wall Street protests in Zuccotti Park in the Financial District of Lower Manhattan began on September 17, 2011, receiving global attention and popularizing the Occupy movement against social and economic inequality worldwide.New York City was heavily affected by Hurricane Sandy in late October 2012. Sandy\\'s impacts included the flooding of the New York City Subway system, of many suburban communities, and of all road tunnels entering Manhattan except the Lincoln Tunnel. The New York Stock Exchange closed for two consecutive days. Numerous homes and businesses were destroyed by fire, including over 100 homes in Breezy Point, Queens. Large parts of the city and surrounding areas lost electricity for several days. Several thousand people in Midtown Manhattan were evacuated for six days due to a crane collapse at Extell\\'s One57. Bellevue Hospital Center and a few other large hospitals were closed and evacuated. Flooding at 140 West Street and another exchange disrupted voice and data communication in Lower Manhattan. At least 43 people lost their lives in New York City as a result of Sandy, and the economic losses in New York City were estimated to be roughly $19 billion. The disaster spawned long-term efforts towards infrastructural projects to counter climate change and rising seas.In March 2020, the first case of COVID-19 in the city was confirmed in Manhattan. The city rapidly replaced Wuhan, China to become the global epicenter of the pandemic during the early phase, before the infection became widespread across the world and the rest of the nation. As of March 2021, New York City had recorded over 30,000 deaths from COVID-19-related complications. In 2022, the LGBT community in New York City became the epicenter of the monkeypox outbreak in the Western Hemisphere, prompting New York Governor Kathy Hochul and New York City Mayor Eric Adams declared corresponding public health emergencies in the state and city, respectively, in July 2022.\\n\\n\\n== '], response=None, multi_responses=None, reference=\"Early European explorers and Native Americans shaped New York City through exploration, trade, and settlement. Native Americans, including the Lenape, originally inhabited the area. European explorers like Giovanni da Verrazzano and Henry Hudson claimed and charted the region for European powers. The Dutch established a permanent settlement, New Amsterdam, which later became New York under English rule. These interactions and exchanges laid the foundation for the city's development and cultural diversity.\", rubric=None), SingleTurnSample(user_input='New York City population economy challenges', retrieved_contexts=None, reference_contexts=[\"New York City, the most populous city in the United States, is a global cultural, financial, and media hub with significant influence in various sectors including commerce, healthcare, and technology. It is composed of five boroughs: Brooklyn, Queens, Manhattan, The Bronx, and Staten Island, each with its own unique character. The city is known for its diverse population, with over 800 languages spoken, making it the most linguistically diverse city in the world. NYC is a major center for international diplomacy, hosting the United Nations headquarters. The city has a rich history, from its early days as a Dutch trading post to its role as a gateway for immigrants. It is home to iconic landmarks such as the Statue of Liberty, Times Square, and Central Park. New York City is also a leader in arts and culture, with numerous museums, theaters, and music venues. The city's economy is driven by finance, technology, real estate, and tourism, with Wall Street being a major financial center. NYC's public transportation system is extensive, including the largest subway system in the world. The city faces challenges such as income disparity and environmental issues but continues to be a vibrant and dynamic metropolis.\"], response=None, multi_responses=None, reference='New York City, as the most populous city in the United States, faces challenges such as income disparity and environmental issues while maintaining a vibrant and dynamic economy driven by finance, technology, real estate, and tourism.', rubric=None), SingleTurnSample(user_input='How do the economic aspects of New York City, as a global cultural center and financial industry hub with a diverse population, compare in terms of influence and challenges across different reports?', retrieved_contexts=None, reference_contexts=[\"New York City, the most populous city in the United States, is a global cultural, financial, and media hub with significant influence in various sectors including commerce, healthcare, and technology. It is composed of five boroughs: Brooklyn, Queens, Manhattan, The Bronx, and Staten Island, each with its own unique character. The city is known for its diverse population, with over 800 languages spoken, making it the most linguistically diverse city in the world. NYC is a major center for international diplomacy, hosting the United Nations headquarters. The city has a rich history, from its early days as a Dutch trading post to its role as a gateway for immigrants. It is home to iconic landmarks such as the Statue of Liberty, Times Square, and Central Park. New York City is also a leader in arts and culture, with numerous museums, theaters, and music venues. The city's economy is driven by finance, technology, real estate, and tourism, with Wall Street being a major financial center. NYC's public transportation system is extensive, including the largest subway system in the world. The city faces challenges such as income disparity and environmental issues but continues to be a vibrant and dynamic metropolis.\"], response=None, multi_responses=None, reference=\"New York City's economic aspects as a global cultural center and financial industry hub are influential due to its diverse population, significant sectors like finance and technology, and its role in international diplomacy. However, it faces challenges such as income disparity and environmental issues.\", rubric=None), SingleTurnSample(user_input='What are some of the cultural and architectural features that make Brooklyn distinct within New York City?', retrieved_contexts=None, reference_contexts=['Geography ==\\n\\nDuring the Wisconsin glaciation, 75,000 to 11,000 years ago, the New York City area was situated at the edge of a large ice sheet over 2,000 feet (610 m) in depth. The erosive forward movement of the ice (and its subsequent retreat) contributed to the separation of what is now Long Island and Staten Island. That action also left bedrock at a relatively shallow depth, providing a solid foundation for most of Manhattan\\'s skyscrapers.New York City is situated in the northeastern United States, in southeastern New York State, approximately halfway between Washington, D.C. and Boston. The location at the mouth of the Hudson River, which feeds into a naturally sheltered harbor and then into the Atlantic Ocean, has helped the city grow in significance as a trading port. Most of New York City is built on the three islands of Long Island, Manhattan, and Staten Island.\\nThe Hudson River flows through the Hudson Valley into New York Bay. Between New York City and Troy, New York, the river is an estuary. The Hudson River separates the city from the U.S. state of New Jersey. The East River—a tidal strait—flows from Long Island Sound and separates the Bronx and Manhattan from Long Island. The Harlem River, another tidal strait between the East and Hudson rivers, separates most of Manhattan from the Bronx. The Bronx River, which flows through the Bronx and Westchester County, is the only entirely freshwater river in the city.The city\\'s land has been altered substantially by human intervention, with considerable land reclamation along the waterfronts since Dutch colonial times; reclamation is most prominent in Lower Manhattan, with developments such as Battery Park City in the 1970s and 1980s. Some of the natural relief in topography has been evened out, especially in Manhattan.The city\\'s total area is 468.484 square miles (1,213.37 km2); 302.643 sq mi (783.84 km2) of the city is land and 165.841 sq mi (429.53 km2) of this is water. The highest point in the city is Todt Hill on Staten Island, which, at 409.8 feet (124.9 m) above sea level, is the highest point on the eastern seaboard south of Maine. The summit of the ridge is mostly covered in woodlands as part of the Staten Island Greenbelt.\\n\\n\\n=== Boroughs ===\\n\\nNew York City is sometimes referred to collectively as the Five Boroughs. Each borough is coextensive with a respective county of New York State, making New York City one of the U.S. municipalities in multiple counties. There are hundreds of distinct neighborhoods throughout the boroughs, many with a definable history and character.\\nIf the boroughs were each independent cities, four of the boroughs (Brooklyn, Queens, Manhattan, and the Bronx) would be among the ten most populous cities in the United States (Staten Island would be ranked 37th as of 2020); these same boroughs are coterminous with the four most densely populated counties in the United States: New York (Manhattan), Kings (Brooklyn), Bronx, and Queens.\\n\\n\\n==== Manhattan ====\\n\\nManhattan (New York County) is the geographically smallest and most densely populated borough. It is home to Central Park and most of the city\\'s skyscrapers, and is sometimes locally known as The City. Manhattan\\'s population density of 72,033 people per square mile (27,812/km2) in 2015 makes it the highest of any county in the United States and higher than the density of any individual American city.Manhattan is the cultural, administrative, and financial center of New York City and contains the headquarters of many major multinational corporations, the United Nations headquarters, Wall Street, and a number of important universities. The borough of Manhattan is often described as the financial and cultural center of the world.Most of the borough is situated on Manhattan Island, at the mouth of the Hudson River and the East River, and its southern tip, at the confluence of the two rivers, represents the birthplace of New York City itself. Several small islands also compose part of the borough of Manhattan, including Randalls and Wards Islands, and Roosevelt Island in the East River, and Governors Island and Liberty Island to the south in New York Harbor.\\nManhattan Island is loosely divided into the Lower, Midtown, and Uptown regions. Uptown Manhattan is divided by Central Park into the Upper East Side and the Upper West Side, and above the park is Harlem, bordering the Bronx (Bronx County).\\nHarlem was predominantly occupied by Jewish and Italian Americans in the 19th century until the Great Migration. It was the center of the Harlem Renaissance.\\n\\nThe borough of Manhattan also includes a small neighborhood on the mainland, called Marble Hill, which is contiguous with the Bronx. New York City\\'s remaining four boroughs are collectively referred to as the Outer Boroughs.\\n\\n\\n==== Brooklyn ====\\nBrooklyn (Kings County), on the western tip of Long Island, is the city\\'s most populous borough. Brooklyn is known for its cultural, social, and ethnic diversity, an independent art scene, distinct neighborhoods, and a distinctive architectural heritage. Downtown Brooklyn is the largest central core neighborhood in the Outer Boroughs. The borough has a long beachfront shoreline including Coney Island, established in the 1870s as one of the earliest amusement grounds in the U.S. Marine Park and Prospect Park are the two largest parks in Brooklyn. Since 2010, Brooklyn has evolved into a thriving hub of entrepreneurship and high technology startup firms, and of postmodern art and design.\\n\\n\\n==== Queens ====\\nQueens (Queens County), on Long Island north and east of Brooklyn, is geographically the largest borough, the most ethnically diverse county in the United States, and the most ethnically diverse urban area in the world. Historically a collection of small towns and villages founded by the Dutch, the borough has since developed both commercial and residential prominence. Downtown Flushing has become one of the busiest central core neighborhoods in the outer boroughs. Queens is the site of the Citi Field baseball stadium, home of the New York Mets, and hosts the annual U.S. Open tennis tournament at Flushing Meadows–Corona Park. Additionally, two of the three busiest airports serving the New York metropolitan area, John F. Kennedy International Airport and LaGuardia Airport, are in Queens. The third is Newark Liberty International Airport in Newark, New Jersey.\\n\\n\\n==== The Bronx ====\\nThe Bronx (Bronx County) is both New York City\\'s northernmost borough, and the only one that is mostly on the mainland. It is the location of Yankee Stadium, the baseball park of the New York Yankees, and home to the largest cooperatively-owned housing complex in the United States, Co-op City. It is also home to the Bronx Zoo, the world\\'s largest metropolitan zoo, which spans 265 acres (1.07 km2) and houses more than 6,000 animals. The Bronx is also the birthplace of hip hop music and its associated culture. Pelham Bay Park is the largest park in New York City, at 2,772 acres (1,122 ha).\\n\\n\\n==== Staten Island ====\\nStaten Island (Richmond County) is the most suburban in character of the five boroughs. Staten Island is connected to Brooklyn by the Verrazzano-Narrows Bridge, and to Manhattan by way of the free Staten Island Ferry, a daily commuter ferry that provides unobstructed views of the Statue of Liberty, Ellis Island, and Lower Manhattan. In central Staten Island, the Staten Island Greenbelt spans approximately 2,500 acres (10 km2), including 28 miles (45 km) of walking trails and one of the last undisturbed forests in the city. Designated in 1984 to protect the island\\'s natural lands, the Greenbelt comprises seven city parks.\\n\\n\\t\\t\\n\\t\\t\\n\\n\\n=== Architecture ===\\n\\nNew York has architecturally noteworthy buildings in a wide range of styles and from distinct time periods, from the Dutch Colonial Pieter Claesen Wyckoff House in Brooklyn, the oldest section of which dates to 1656, to the modern One World Trade Center, the skyscraper at Ground Zero in Lower Manhattan and the most expensive office tower in the world by construction cost.Manhattan\\'s skyline, with its many skyscrapers, is universally recognized, and the city has been home to several of the tallest buildings in the world. As of 2019, New York City had 6,455 high-rise buildings, the third most in the world after Hong Kong and Seoul. Of these, as of 2011, 550 completed structures were at least 330 feet (100 m) high, with more than fifty completed skyscrapers taller than 656 feet (200 m). These include the Woolworth Building, an early example of Gothic Revival architecture in skyscraper design, built with massively scaled Gothic detailing; completed in 1913, for 17 years it was the world\\'s tallest building.The 1916 Zoning Resolution required setbacks in new buildings and restricted towers to a percentage of the lot size, to allow sunlight to reach the streets below. The Art Deco style of the Chrysler Building (1930) and Empire State Building (1931), with their tapered tops and steel spires, reflected the zoning requirements. The buildings have distinctive ornamentation, such as the eagles at the corners of the 61st floor on the Chrysler Building, and are considered some of the finest examples of the Art Deco style. A highly influential example of the International Style in the United States is the Seagram Building (1957), distinctive for its façade using visible bronze-toned I-beams to evoke the building\\'s structure. The Condé Nast Building (2000) is a prominent example of green design in American skyscrapers and has received an award from the American Institute of Architects and AIA New York State for its design.\\nThe character of New York\\'s large residential districts is often defined by the elegant brownstone rowhouses and townhouses and shabby tenements that were built during a period of rapid expansion from 1870 to 1930. In contrast, New York City also has neighborhoods that are less densely populated and feature free-standing dwellings. In neighborhoods such as Riverdale (in the Bronx), Ditmas Park (in Brooklyn), and Douglaston (in Queens), large single-family homes are common in various architectural styles such as Tudor Revival and Victorian.Stone and brick became the city\\'s building materials of choice after the construction of wood-frame houses was limited in the aftermath of the Great Fire of 1835. A distinctive feature of many of the city\\'s buildings is the roof-mounted wooden water tower. In the 1800s, the city required their installation on buildings higher than six stories to prevent the need for excessively high water pressures at lower elevations, which could break municipal water pipes. Garden apartments became popular during the 1920s in outlying areas, such as Jackson Heights.According to the United States Geological Survey, an updated analysis of seismic hazard in July 2014 revealed a \"slightly lower hazard for tall buildings\" in New York City than previously assessed. Scientists estimated this lessened risk based upon a lower likelihood than previously thought of slow shaking near the city, which would be more likely to cause damage to taller structures from an earthquake in the vicinity of the city. Manhattan contained over 500 million square feet of office space as of 2022; the COVID-19 pandemic and hybrid work model have prompted consideration of commercial-to-residential conversion within Midtown Manhattan.\\n\\n\\n=== Climate ===\\n\\nUnder the Köppen climate classification, using the 0 °C (32 °F) isotherm, New York City features a humid subtropical climate (Cfa), and is thus the northernmost major city on the North American continent with this categorization. The suburbs to the immediate north and west lie in the transitional zone between humid subtropical and humid continental climates (Dfa). By the Trewartha classification, the city is defined as having an oceanic climate (Do). Annually, the city averages 234 days with at least some sunshine. The city lies in the USDA 7b plant hardiness zone.Winters are chilly and damp, and prevailing wind patterns that blow sea breezes offshore temper the moderating effects of the Atlantic Ocean; yet the Atlantic and the partial shielding from colder air by the Appalachian Mountains keep the city warmer in the winter than inland North American cities at similar or lesser latitudes such as Pittsburgh, Cincinnati, and Indianapolis. The daily mean temperature in January, the area\\'s coldest month, is 33.3 °F (0.7 °C). Temperatures usually drop to 10 °F (−12 °C) several times per winter, yet can also reach 60 °F (16 °C) for several days even in the coldest winter month. Spring and autumn are unpredictable and can range from cool to warm, although they are usually mild with low humidity. Summers are typically hot and humid, with a daily mean temperature of 77.5 °F (25.3 °C) in July.Nighttime temperatures are often enhanced due to the urban heat island effect. Daytime temperatures exceed 90 °F (32 °C) on average of 17 days each summer and in some years exceed 100 °F (38 °C), although this is a rare achievement, last occurring on July 18, 2012. Similarly, readings of 0 °F (−18 °C) are also extremely rare, last occurring on February 14, 2016. Extreme temperatures have ranged from −15 °F (−26 °C), recorded on February 9, 1934, up to 106 °F (41 °C) on July 9, 1936; the coldest recorded wind chill was −37 °F (−38 °C) on the same day as the all-time record low. The record cold daily maximum was 2 °F (−17 °C) on December 30, 1917, while, conversely, the record warm daily minimum was 87 °F (31 °C), on July 2, 1903. The average water temperature of the nearby Atlantic Ocean ranges from 39.7 °F (4.3 °C) in February to 74.1 °F (23.4 °C) in August.The city receives 49.5 inches (1,260 mm) of precipitation annually, which is relatively evenly spread throughout the year. Average winter snowfall between 1991 and 2020 has been 29.8 inches (76 cm); this varies considerably between years. Hurricanes and tropical storms are rare in the New York area. Hurricane Sandy brought a destructive storm surge to New York City on the evening of October 29, 2012, flooding numerous streets, tunnels, and subway lines in Lower Manhattan and other areas of the city and cutting off electricity in many parts of the city and its suburbs. The storm and its profound impacts have prompted the discussion of constructing seawalls and other coastal barriers around the shorelines of the city and the metropolitan area to minimize the risk of destructive consequences from another such event in the future.The coldest month on record is January 1857, with a mean temperature of 19.6 °F (−6.9 °C) whereas the warmest months on record are July 1825 and July 1999, both with a mean temperature of 81.4 °F (27.4 °C). The warmest years on record are 2012 and 2020, both with mean temperatures of 57.1 °F (13.9 °C). The coldest year is 1836, with a mean temperature of 47.3 °F (8.5 °C). The driest month on record is June 1949, with 0.02 inches (0.51 mm) of rainfall. The wettest month was August 2011, with 18.95 inches (481 mm) of rainfall. The driest year on record is 1965, with 26.09 inches (663 mm) of rainfall. The wettest year was 1983, with 80.56 inches (2,046 mm) of rainfall. The snowiest month on record is February 2010, with 36.9 inches (94 cm) of snowfall. The snowiest season (Jul–Jun) on record is 1995–1996, with 75.6 inches (192 cm) of snowfall. The least snowy season was 1972–1973, with 2.3 inches (5.8 cm) of snowfall. The earliest seasonal trace of snowfall occurred on October 10, in both 1979 and 1925. The latest seasonal trace of snowfall occurred on May 9, in both 2020 and 1977.\\n\\nSee or edit raw graph data.\\n\\n\\n=== Parks ===\\n\\nThe city of New York has a complex park system, with various lands operated by the National Park Service, the New York State Office of Parks, Recreation and Historic Preservation, and the New York City Department of Parks and Recreation. In its 2018 ParkScore ranking, the Trust for Public Land reported that the park system in New York City was the ninth-best park system among the fifty most populous U.S. cities. ParkScore ranks urban park systems by a formula that analyzes median park size, park acres as percent of city area, the percent of city residents within a half-mile of a park, spending of park services per resident, and the number of playgrounds per 10,000 residents. In 2021, the New York City Council banned the use of synthetic pesticides by city agencies and instead required organic lawn management. The effort was started by teacher Paula Rogovin\\'s kindergarten class at P.S. 290.\\n\\n\\n==== National parks ====\\n\\nGateway National Recreation Area contains over 26,000 acres (110 km2), most of it in New York City. In Brooklyn and Queens, the park contains over 9,000 acres (36 km2) of salt marsh, wetlands, islands, and water, including most of Jamaica Bay and the Jamaica Bay Wildlife Refuge. Also in Queens, the park includes a significant portion of the western Rockaway Peninsula, most notably Jacob Riis Park and Fort Tilden. In Staten Island, it includes Fort Wadsworth, with historic pre-Civil War era Battery Weed and Fort Tompkins, and Great Kills Park, with beaches, trails, and a marina.\\nThe Statue of Liberty National Monument and Ellis Island Immigration Museum are managed by the National Park Service and are in both New York and New Jersey. They are joined in the harbor by Governors Island National Monument. Historic sites under federal management on Manhattan Island include Stonewall National Monument; Castle Clinton National Monument; Federal Hall National Memorial; Theodore Roosevelt Birthplace National Historic Site; General Grant National Memorial (Grant\\'s Tomb); African Burial Ground National Monument; and Hamilton Grange National Memorial. Hundreds of properties are listed on the National Register of Historic Places or as a National Historic Landmark.\\n\\n\\n==== State parks ====\\n\\nThere are seven state parks within the confines of New York City. Some of them include:\\n\\nThe Clay Pit Ponds State Park Preserve is a natural area that includes extensive riding trails.\\nRiverbank State Park is a 28-acre (11 ha) facility that rises 69 feet (21 m) over the Hudson River.\\nMarsha P. Johnson State Park is a state park in Brooklyn and Manhattan that borders the East River that was renamed in honor of Marsha P. Johnson.\\n\\n\\n==== City parks ====\\n\\nNew York City has over 28,000 acres (110 km2) of municipal parkland and 14 miles (23 km) of public beaches. The largest municipal park in the city is Pelham Bay Park in the Bronx, with 2,772 acres (1,122 ha).\\nCentral Park, an 843-acre (3.41 km2) park in middle-upper Manhattan, is the most visited urban park in the United States and one of the most filmed locations in the world, with 40 million visitors in 2013. The park has a wide range of attractions; there are several lakes and ponds, two ice-skating rinks, the Central Park Zoo, the Central Park Conservatory Garden, and the 106-acre (0.43 km2) Jackie Onassis Reservoir. Indoor attractions include Belvedere Castle with its nature center, the Swedish Cottage Marionette Theater, and the historic Carousel. On October 23, 2012, hedge fund manager John A. Paulson announced a $100 million gift to the Central Park Conservancy, the largest ever monetary donation to New York City\\'s park system.\\nWashington Square Park is a prominent landmark in the Greenwich Village neighborhood of Lower Manhattan. The Washington Square Arch at the northern gateway to the park is an iconic symbol of both New York University and Greenwich Village.\\nProspect Park in Brooklyn has a 90-acre (36 ha) meadow, a lake, and extensive woodlands. Within the park is the historic Battle Pass, prominent in the Battle of Long Island.\\nFlushing Meadows–Corona Park in Queens, with its 897 acres (363 ha) making it the city\\'s fourth largest park, was the setting for the 1939 World\\'s Fair and the 1964 World\\'s Fair and is host to the USTA Billie Jean King National Tennis Center and the annual U.S. Open Tennis Championships tournament.\\nOver a fifth of the Bronx\\'s area, 7,000 acres (28 km2), is dedicated to open space and parks, including Pelham Bay Park, Van Cortlandt Park, the Bronx Zoo, and the New York Botanical Gardens.\\nIn Staten Island, the Conference House Park contains the historic Conference House, site of the only attempt of a peaceful resolution to the American Revolution which was conducted in September 1775, attended by Benjamin Franklin representing the Americans and Lord Howe representing the British Crown. The historic Burial Ridge, the largest Native American burial ground within New York City, is within the park.\\n\\n\\n=== Military installations ===\\nBrooklyn is home to Fort Hamilton, the U.S. military\\'s only active duty installation within New York City, aside from Coast Guard operations. The facility was established in 1825 on the site of a small battery used during the American Revolution, and it is one of America\\'s longest serving military forts. Today, Fort Hamilton serves as the headquarters of the North Atlantic Division of the United States Army Corps of Engineers and for the New York City Recruiting Battalion. It also houses the 1179th Transportation Brigade, the 722nd Aeromedical Staging Squadron, and a military entrance processing station. Other formerly active military reservations still used for National Guard and military training or reserve operations in the city include Fort Wadsworth in Staten Island and Fort Totten in Queens.\\n\\n\\n== '], response=None, multi_responses=None, reference='Brooklyn is distinct within New York City due to its cultural, social, and ethnic diversity, independent art scene, distinct neighborhoods, and distinctive architectural heritage. It is also known for its long beachfront shoreline, including Coney Island, and its large parks like Marine Park and Prospect Park. Additionally, Brooklyn has evolved into a hub of entrepreneurship and high technology startup firms, as well as postmodern art and design.', rubric=None), SingleTurnSample(user_input='What measures has New York City implemented to reduce its carbon footprint and environmental impact?', retrieved_contexts=None, reference_contexts=['Environment ==\\n\\n \\nEnvironmental issues in New York City are affected by the city\\'s size, density, abundant public transportation infrastructure, and its location at the mouth of the Hudson River. For example, it is one of the country\\'s biggest sources of pollution and has the lowest per-capita greenhouse gas emissions rate and electricity usage. Governors Island is planned to host a US$1 billion research and education center to make New York City the global leader in addressing the climate crisis.\\n\\n\\n=== Environmental impact reduction ===\\nNew York City has focused on reducing its environmental impact and carbon footprint. Mass transit use in New York City is the highest in the United States. Also, by 2010, the city had 3,715 hybrid taxis and other clean diesel vehicles, representing around 28% of New York\\'s taxi fleet in service, the most of any city in North America. New York City is the host of Climate Week NYC, the largest Climate Week to take place globally and regarded as major annual climate summit.\\nNew York\\'s high rate of public transit use, more than 200,000 daily cyclists as of 2014, and many pedestrian commuters make it the most energy-efficient major city in the United States. Walk and bicycle modes of travel account for 21% of all modes for trips in the city; nationally the rate for metro regions is about 8%. In both its 2011 and 2015 rankings, Walk Score named New York City the most walkable large city in the United States, and in 2018, Stacker ranked New York the most walkable U.S. city. Citibank sponsored the introduction of 10,000 public bicycles for the city\\'s bike-share project in the summer of 2013. New York City\\'s numerical \"in-season cycling indicator\" of bicycling in the city had hit an all-time high of 437 when measured in 2014.The city government was a petitioner in the landmark Massachusetts v. Environmental Protection Agency Supreme Court case forcing the EPA to regulate greenhouse gases as pollutants. The city is a leader in the construction of energy-efficient green office buildings, including the Hearst Tower among others. Mayor Bill de Blasio has committed to an 80% reduction in greenhouse gas emissions between 2014 and 2050 to reduce the city\\'s contributions to climate change, beginning with a comprehensive \"Green Buildings\" plan.\\n\\n\\n=== Water purity and availability ===\\n\\nThe New York City drinking water supply is extracted from the protected Catskill Mountains watershed. As a result of the watershed\\'s integrity and undisturbed natural water filtration system, New York is one of only four major cities in the United States the majority of whose drinking water is pure enough not to require purification through water treatment plants. The city\\'s municipal water system is the largest in the United States, moving over one billion gallons of water per day; a leak in the Delaware aqueduct results in some 20 million gallons a day being lost under the Hudson River. The Croton Watershed north of the city is undergoing construction of a $3.2 billion water purification plant to augment New York City\\'s water supply by an estimated 290 million gallons daily, representing a greater than 20% addition to the city\\'s current availability of water. The ongoing expansion of New York City Water Tunnel No. 3, an integral part of the New York City water supply system, is the largest capital construction project in the city\\'s history, with segments serving Manhattan and the Bronx completed, and with segments serving Brooklyn and Queens planned for construction in 2020. In 2018, New York City announced a $1 billion investment to protect the integrity of its water system and to maintain the purity of its unfiltered water supply.\\n\\n\\n=== Air quality ===\\nAccording to the 2016 World Health Organization Global Urban Ambient Air Pollution Database, the annual average concentration in New York City\\'s air of particulate matter measuring 2.5 micrometers or less (PM2.5) was 7.0 micrograms per cubic meter, or 3.0 micrograms within the recommended limit of the WHO Air Quality Guidelines for the annual mean PM2.5. The New York City Department of Health and Mental Hygiene, in partnership with Queens College, conducts the New York Community Air Survey to measure pollutants at about 150 locations.\\n\\n\\n=== Environmental revitalization ===\\nNewtown Creek, a 3.5-mile (6-kilometer) a long estuary that forms part of the border between the boroughs of Brooklyn and Queens, has been designated a Superfund site for environmental clean-up and remediation of the waterway\\'s recreational and economic resources for many communities. One of the most heavily used bodies of water in the Port of New York and New Jersey, it had been one of the most contaminated industrial sites in the country, containing years of discarded toxins, an estimated 30 million US gallons (110,000 m3) of spilled oil, including the Greenpoint oil spill, raw sewage from New York City\\'s sewer system, and other accumulation.\\n\\n\\n== '], response=None, multi_responses=None, reference='New York City has implemented several measures to reduce its carbon footprint and environmental impact, including the highest mass transit use in the United States, a significant number of hybrid taxis and clean diesel vehicles, hosting Climate Week NYC, promoting cycling and pedestrian commuting, constructing energy-efficient green office buildings, and committing to an 80% reduction in greenhouse gas emissions by 2050. Additionally, the city has introduced a bike-share project and has been involved in legal actions to regulate greenhouse gases as pollutants.', rubric=None), SingleTurnSample(user_input='What role did New York City play during the American Revolution?', retrieved_contexts=None, reference_contexts=['History ==\\n\\n\\n=== Early history ===\\nIn the pre-Columbian era, the area of present-day New York City was inhabited by Algonquian Native Americans, including the Lenape. Their homeland, known as Lenapehoking, included the present-day areas of Staten Island, Manhattan, the Bronx, the western portion of Long Island (including the areas that would later become the boroughs of Brooklyn and Queens), and the Lower Hudson Valley.The first documented visit into New York Harbor by a European was in 1524 by Italian Giovanni da Verrazzano, an explorer from Florence in the service of the French crown. He claimed the area for France and named it Nouvelle Angoulême (New Angoulême). A Spanish expedition, led by the Portuguese captain Estêvão Gomes sailing for Emperor Charles V, arrived in New York Harbor in January 1525 and charted the mouth of the Hudson River, which he named Río de San Antonio (\\'Saint Anthony\\'s River\\'). The Padrón Real of 1527, the first scientific map to show the East Coast of North America continuously, was informed by Gomes\\' expedition and labeled the northeastern United States as Tierra de Esteban Gómez in his honor.In 1609, the English explorer Henry Hudson rediscovered New York Harbor while searching for the Northwest Passage to the Orient for the Dutch East India Company. He proceeded to sail up what the Dutch would name the North River (now the Hudson River), named first by Hudson as the Mauritius after Maurice, Prince of Orange. Hudson\\'s first mate described the harbor as \"a very good Harbour for all windes\" and the river as \"a mile broad\" and \"full of fish\". Hudson sailed roughly 150 miles (240 km) north, past the site of the present-day New York State capital city of Albany, in the belief that it might be an oceanic tributary before the river became too shallow to continue. He made a ten-day exploration of the area and claimed the region for the Dutch East India Company. In 1614, the area between Cape Cod and Delaware Bay was claimed by the Netherlands and called Nieuw-Nederland (\\'New Netherland\\').\\nThe first non–Native American inhabitant of what would eventually become New York City was Juan Rodriguez (transliterated to the Dutch language as Jan Rodrigues), a merchant from Santo Domingo. Born in Santo Domingo of Portuguese and African descent, he arrived in Manhattan during the winter of 1613–14, trapping for pelts and trading with the local population as a representative of the Dutch. Broadway, from 159th Street to 218th Street in Upper Manhattan, is named Juan Rodriguez Way in his honor.\\n\\n\\n=== Dutch rule ===\\n\\nA permanent European presence near New York Harbor was established in 1624, making New York the 12th-oldest continuously occupied European-established settlement in the continental United States—with the founding of a Dutch fur trading settlement on Governors Island. In 1625, construction was started on a citadel and Fort Amsterdam, later called Nieuw Amsterdam (New Amsterdam), on present-day Manhattan Island. The colony of New Amsterdam was centered on what would ultimately be known as Lower Manhattan. Its area extended from the southern tip of Manhattan to modern day Wall Street, where a 12-foot wooden stockade was built in 1653 to protect against Native American and British raids. In 1626, the Dutch colonial Director-General Peter Minuit, acting as charged by the Dutch West India Company, purchased the island of Manhattan from the Canarsie, a small Lenape band, for \"the value of 60 guilders\" (about $900 in 2018). A disproved legend claims that Manhattan was purchased for $24 worth of glass beads.Following the purchase, New Amsterdam grew slowly. To attract settlers, the Dutch instituted the patroon system in 1628, whereby wealthy Dutchmen (patroons, or patrons) who brought 50 colonists to New Netherland would be awarded swaths of land, along with local political autonomy and rights to participate in the lucrative fur trade. This program had little success.Since 1621, the Dutch West India Company had operated as a monopoly in New Netherland, on authority granted by the Dutch States General. In 1639–1640, in an effort to bolster economic growth, the Dutch West India Company relinquished its monopoly over the fur trade, leading to growth in the production and trade of food, timber, tobacco, and slaves (particularly with the Dutch West Indies).In 1647, Peter Stuyvesant began his tenure as the last Director-General of New Netherland. During his tenure, the population of New Netherland grew from 2,000 to 8,000. Stuyvesant has been credited with improving law and order in the colony; however, he also earned a reputation as a despotic leader. He instituted regulations on liquor sales, attempted to assert control over the Dutch Reformed Church, and blocked other religious groups (including Quakers, Jews, and Lutherans) from establishing houses of worship. The Dutch West India Company would eventually attempt to ease tensions between Stuyvesant and residents of New Amsterdam.\\n\\n\\n=== English rule ===\\n\\nIn 1664, unable to summon any significant resistance, Stuyvesant surrendered New Amsterdam to English troops, led by Colonel Richard Nicolls, without bloodshed. The terms of the surrender permitted Dutch residents to remain in the colony and allowed for religious freedom. In 1667, during negotiations leading to the Treaty of Breda after the Second Anglo-Dutch War, the Dutch decided to keep the nascent plantation colony of what is now Suriname (on the northern South American coast) they had gained from the English; and in return, the English kept New Amsterdam. The fledgling settlement was promptly renamed \"New York\" after the Duke of York (the future King James II and VII), who would eventually be deposed in the Glorious Revolution. After the founding, the duke gave part of the colony to proprietors George Carteret and John Berkeley. Fort Orange, 150 miles (240 km) north on the Hudson River, was renamed Albany after James\\'s Scottish title. The transfer was confirmed in 1667 by the Treaty of Breda, which concluded the Second Anglo-Dutch War.On August 24, 1673, during the Third Anglo-Dutch War, Dutch captain Anthony Colve seized the colony of New York from the English at the behest of Cornelis Evertsen the Youngest and rechristened it \"New Orange\" after William III, the Prince of Orange. The Dutch would soon return the island to England under the Treaty of Westminster of November 1674.Several intertribal wars among the Native Americans and some epidemics brought on by contact with the Europeans caused sizeable population losses for the Lenape between the years 1660 and 1670. By 1700, the Lenape population had diminished to 200. New York experienced several yellow fever epidemics in the 18th century, losing ten percent of its population to the disease in 1702 alone.\\n\\n\\n=== Province of New York and slavery ===\\n\\nIn the early 18th century, New York grew in importance as a trading port while as a part of the colony of New York. It also became a center of slavery, with 42% of households enslaving Africans by 1730, the highest percentage outside Charleston, South Carolina. Most cases were that of domestic slavery, as a New York household then commonly enslaved few or several people. Others were hired out to work at labor. Slavery became integrally tied to New York\\'s economy through the labor of slaves throughout the port, and the banking and shipping industries trading with the American South. During construction in Foley Square in the 1990s, the African Burying Ground was discovered; the cemetery included 10,000 to 20,000 of graves of colonial-era Africans, some enslaved and some free.The 1735 trial and acquittal in Manhattan of John Peter Zenger, who had been accused of seditious libel after criticizing colonial governor William Cosby, helped to establish the freedom of the press in North America. In 1754, Columbia University was founded under charter by King George II as King\\'s College in Lower Manhattan.\\n\\n\\n=== American Revolution ===\\n\\nThe Stamp Act Congress met in New York in October 1765, as the Sons of Liberty organization emerged in the city and skirmished over the next ten years with British troops stationed there. The Battle of Long Island, the largest battle of the American Revolutionary War, was fought in August 1776 within the modern-day borough of Brooklyn. After the battle, in which the Americans were defeated, the British made the city their military and political base of operations in North America. The city was a haven for Loyalist refugees and escaped slaves who joined the British lines for freedom newly promised by the Crown for all fighters. As many as 10,000 escaped slaves crowded into the city during the British occupation. When the British forces evacuated at the close of the war in 1783, they transported 3,000 freedmen for resettlement in Nova Scotia. They resettled other freedmen in England and the Caribbean.\\nThe only attempt at a peaceful solution to the war took place at the Conference House on Staten Island between American delegates, including Benjamin Franklin, and British general Lord Howe on September 11, 1776. Shortly after the British occupation began, the Great Fire of New York occurred, a large conflagration on the West Side of Lower Manhattan, which destroyed about a quarter of the buildings in the city, including Trinity Church.In 1785, the assembly of the Congress of the Confederation made New York City the national capital shortly after the war. New York was the last capital of the U.S. under the Articles of Confederation and the first capital under the Constitution of the United States. New York City as the U.S. capital hosted several events of national scope in 1789—the first President of the United States, George Washington, was inaugurated; the first United States Congress and the Supreme Court of the United States each assembled for the first time; and the United States Bill of Rights was drafted, all at Federal Hall on Wall Street. In 1790, New York surpassed Philadelphia as the nation\\'s largest city. At the end of that year, pursuant to the Residence Act, the national capital was moved to Philadelphia.\\n\\n\\n=== 19th century ===\\n\\nOver the course of the nineteenth century, New York City\\'s population grew from 60,000 to 3.43 million. Under New York State\\'s abolition act of 1799, children of slave mothers were to be eventually liberated but to be held in indentured servitude until their mid-to-late twenties. Together with slaves freed by their masters after the Revolutionary War and escaped slaves, a significant free-Black population gradually developed in Manhattan. Under such influential United States founders as Alexander Hamilton and John Jay, the New York Manumission Society worked for abolition and established the African Free School to educate Black children. It was not until 1827 that slavery was completely abolished in the state, and free Blacks struggled afterward with discrimination. New York interracial abolitionist activism continued; among its leaders were graduates of the African Free School. New York city\\'s population jumped from 123,706 in 1820 to 312,710 by 1840, 16,000 of whom were Black.In the 19th century, the city was transformed by both commercial and residential development relating to its status as a national and international trading center, as well as by European immigration, respectively. The city adopted the Commissioners\\' Plan of 1811, which expanded the city street grid to encompass almost all of Manhattan. The 1825 completion of the Erie Canal through central New York connected the Atlantic port to the agricultural markets and commodities of the North American interior via the Hudson River and the Great Lakes. Local politics became dominated by Tammany Hall, a political machine supported by Irish and German immigrants.Several prominent American literary figures lived in New York during the 1830s and 1840s, including William Cullen Bryant, Washington Irving, Herman Melville, Rufus Wilmot Griswold, John Keese, Nathaniel Parker Willis, and Edgar Allan Poe. Public-minded members of the contemporaneous business elite lobbied for the establishment of Central Park, which in 1857 became the first landscaped park in an American city.\\nThe Great Irish Famine brought a large influx of Irish immigrants; more than 200,000 were living in New York by 1860, upwards of a quarter of the city\\'s population. There was also extensive immigration from the German provinces, where revolutions had disrupted societies, and Germans comprised another 25% of New York\\'s population by 1860.Democratic Party candidates were consistently elected to local office, increasing the city\\'s ties to the South and its dominant party. In 1861, Mayor Fernando Wood called upon the aldermen to declare independence from Albany and the United States after the South seceded, but his proposal was not acted on. Anger at new military conscription laws during the American Civil War (1861–1865), which spared wealthier men who could afford to pay a $300 (equivalent to $6,602 in 2021) commutation fee to hire a substitute, led to the Draft Riots of 1863, whose most visible participants were ethnic Irish working class.The draft riots deteriorated into attacks on New York\\'s elite, followed by attacks on Black New Yorkers and their property after fierce competition for a decade between Irish immigrants and Black people for work. Rioters burned the Colored Orphan Asylum to the ground, with more than 200 children escaping harm due to efforts of the New York Police Department, which was mainly made up of Irish immigrants. At least 120 people were killed. Eleven Black men were lynched over five days, and the riots forced hundreds of Blacks to flee the city for Williamsburg, Brooklyn, and New Jersey. The Black population in Manhattan fell below 10,000 by 1865, which it had last been in 1820. The White working class had established dominance. Violence by longshoremen against Black men was especially fierce in the docks area. It was one of the worst incidents of civil unrest in American history.In 1898, the City of New York was formed with the consolidation of Brooklyn (until then a separate city), the County of New York (which then included parts of the Bronx), the County of Richmond, and the western portion of the County of Queens. The opening of the subway in 1904, first built as separate private systems, helped bind the new city together. Throughout the first half of the 20th century, the city became a world center for industry, commerce, and communication.\\n\\n\\n=== 20th century ===\\n\\nIn 1904, the steamship General Slocum caught fire in the East River, killing 1,021 people on board. In 1911, the Triangle Shirtwaist Factory fire, the city\\'s worst industrial disaster, took the lives of 146 garment workers and spurred the growth of the International Ladies\\' Garment Workers\\' Union and major improvements in factory safety standards.New York\\'s non-White population was 36,620 in 1890. New York City was a prime destination in the early twentieth century for African Americans during the Great Migration from the American South, and by 1916, New York City had become home to the largest urban African diaspora in North America. The Harlem Renaissance of literary and cultural life flourished during the era of Prohibition. The larger economic boom generated construction of skyscrapers competing in height and creating an identifiable skyline.\\nNew York became the most populous urbanized area in the world in the early 1920s, overtaking London. The metropolitan area surpassed the 10 million mark in the early 1930s, becoming the first megacity in human history. The Great Depression saw the election of reformer Fiorello La Guardia as mayor and the fall of Tammany Hall after eighty years of political dominance.Returning World War II veterans created a post-war economic boom and the development of large housing tracts in eastern Queens and Nassau County as well as similar suburban areas in New Jersey. New York emerged from the war unscathed as the leading city of the world, with Wall Street leading America\\'s place as the world\\'s dominant economic power. The United Nations headquarters was completed in 1952, solidifying New York\\'s global geopolitical influence, and the rise of abstract expressionism in the city precipitated New York\\'s displacement of Paris as the center of the art world.The Stonewall riots were a series of spontaneous, violent protests by members of the gay community against a police raid that took place in the early morning hours of June 28, 1969, at the Stonewall Inn in the Greenwich Village neighborhood of Lower Manhattan. They are widely considered to constitute the single most important event leading to the gay liberation movement and the modern fight for LGBT rights. Wayne R. Dynes, author of the Encyclopedia of Homosexuality, wrote that drag queens were the only \"transgender folks around\" during the June 1969 Stonewall riots. The transgender community in New York City played a significant role in fighting for LGBT equality during the period of the Stonewall riots and thereafter.In the 1970s, job losses due to industrial restructuring caused New York City to suffer from economic problems and rising crime rates. While a resurgence in the financial industry greatly improved the city\\'s economic health in the 1980s, New York\\'s crime rate continued to increase through that decade and into the beginning of the 1990s. By the mid 1990s, crime rates started to drop dramatically due to revised police strategies, improving economic opportunities, gentrification, and new residents, both American transplants and new immigrants from Asia and Latin America. Important new sectors, such as Silicon Alley, emerged in the city\\'s economy.\\n\\n\\n=== 21st century ===\\n\\nNew York\\'s population reached all-time highs in the 2000 census and then again in the 2010 census.\\nNew York City suffered the bulk of the economic damage and largest loss of human life in the aftermath of the September 11, 2001, attacks. Two of the four airliners hijacked that day were flown into the twin towers of the World Trade Center, destroying the towers and killing 2,192 civilians, 343 firefighters, and 71 law enforcement officers. The North Tower became the tallest building ever to be destroyed anywhere then or subsequently.The area was rebuilt with a new One World Trade Center, a 9/11 memorial and museum, and other new buildings and infrastructure. The World Trade Center PATH station, which had opened on July 19, 1909, as the Hudson Terminal, was also destroyed in the attacks. A temporary station was built and opened on November 23, 2003. An 800,000-square-foot (74,000 m2) permanent rail station designed by Santiago Calatrava, the World Trade Center Transportation Hub, the city\\'s third-largest hub, was completed in 2016. The new One World Trade Center is the tallest skyscraper in the Western Hemisphere and the seventh-tallest building in the world by pinnacle height, with its spire reaching a symbolic 1,776 feet (541.3 m) in reference to the year of U.S. independence.The Occupy Wall Street protests in Zuccotti Park in the Financial District of Lower Manhattan began on September 17, 2011, receiving global attention and popularizing the Occupy movement against social and economic inequality worldwide.New York City was heavily affected by Hurricane Sandy in late October 2012. Sandy\\'s impacts included the flooding of the New York City Subway system, of many suburban communities, and of all road tunnels entering Manhattan except the Lincoln Tunnel. The New York Stock Exchange closed for two consecutive days. Numerous homes and businesses were destroyed by fire, including over 100 homes in Breezy Point, Queens. Large parts of the city and surrounding areas lost electricity for several days. Several thousand people in Midtown Manhattan were evacuated for six days due to a crane collapse at Extell\\'s One57. Bellevue Hospital Center and a few other large hospitals were closed and evacuated. Flooding at 140 West Street and another exchange disrupted voice and data communication in Lower Manhattan. At least 43 people lost their lives in New York City as a result of Sandy, and the economic losses in New York City were estimated to be roughly $19 billion. The disaster spawned long-term efforts towards infrastructural projects to counter climate change and rising seas.In March 2020, the first case of COVID-19 in the city was confirmed in Manhattan. The city rapidly replaced Wuhan, China to become the global epicenter of the pandemic during the early phase, before the infection became widespread across the world and the rest of the nation. As of March 2021, New York City had recorded over 30,000 deaths from COVID-19-related complications. In 2022, the LGBT community in New York City became the epicenter of the monkeypox outbreak in the Western Hemisphere, prompting New York Governor Kathy Hochul and New York City Mayor Eric Adams declared corresponding public health emergencies in the state and city, respectively, in July 2022.\\n\\n\\n== '], response=None, multi_responses=None, reference='During the American Revolution, New York City played a significant role as a military and political base for the British after the Battle of Long Island in 1776. The city served as a haven for Loyalist refugees and escaped slaves who joined the British lines. It was also the site of the only attempt at a peaceful resolution to the war at the Conference House on Staten Island. After the war, New York City became the national capital under the Articles of Confederation and hosted several important events, including the inauguration of George Washington as the first President of the United States.', rubric=None)])" + "EvaluationDataset(features=['user_input', 'reference_contexts', 'reference'], len=7)" ] }, - "execution_count": 17, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } diff --git a/src/ragas/testset/synthesizers/generate.py b/src/ragas/testset/synthesizers/generate.py index f386a6128..f8fc1363b 100644 --- a/src/ragas/testset/synthesizers/generate.py +++ b/src/ragas/testset/synthesizers/generate.py @@ -111,7 +111,40 @@ def generate_with_langchain_docs( raise_exceptions: bool = True, ) -> Testset: """ - Generates an evaluation dataset based on given scenarios and parameters. + Generates an evaluation dataset based on given Langchain documents and parameters. + + Parameters + ---------- + documents : Sequence[LCDocument] + A sequence of Langchain documents to use as source material + testset_size : int + The number of test samples to generate + transforms : Optional[Transforms], optional + Custom transforms to apply to the documents, by default None + transforms_llm : Optional[BaseRagasLLM], optional + LLM to use for transforms if different from instance LLM, by default None + transforms_embedding_model : Optional[BaseRagasEmbeddings], optional + Embedding model to use for transforms if different from instance model, by default None + query_distribution : Optional[QueryDistribution], optional + Distribution of query types to generate, by default None + run_config : Optional[RunConfig], optional + Configuration for the generation run, by default None + callbacks : Optional[Callbacks], optional + Callbacks to use during generation, by default None + with_debugging_logs : bool, optional + Whether to include debug logs, by default False + raise_exceptions : bool, optional + Whether to raise exceptions during generation, by default True + + Returns + ------- + Testset + The generated evaluation dataset + + Raises + ------ + ValueError + If no LLM or embedding model is provided either during initialization or as arguments """ # force the user to provide an llm and embedding client to prevent use of default LLMs From c5ca6d1044c39a259fb53b13c90268713ed7146a Mon Sep 17 00:00:00 2001 From: jjmachan Date: Fri, 1 Nov 2024 11:23:48 +0530 Subject: [PATCH 9/9] docs: documentation for site --- docs/howtos/integrations/_llamaindex.md | 279 ++++++++++------------ docs/howtos/integrations/llamaindex.ipynb | 198 +-------------- mkdocs.yml | 1 + 3 files changed, 126 insertions(+), 352 deletions(-) diff --git a/docs/howtos/integrations/_llamaindex.md b/docs/howtos/integrations/_llamaindex.md index 9092fe7f7..8419b259d 100644 --- a/docs/howtos/integrations/_llamaindex.md +++ b/docs/howtos/integrations/_llamaindex.md @@ -10,32 +10,27 @@ You will need an testset to evaluate your `QueryEngine` against. You can either Let's see how that works with Llamaindex - -```python # load the documents from llama_index.core import SimpleDirectoryReader documents = SimpleDirectoryReader("./nyc_wikipedia").load_data() -``` Now lets init the `TestsetGenerator` object with the corresponding generator and critic llms ```python -from ragas.testset.generator import TestsetGenerator -from ragas.testset.evolutions import simple, reasoning, multi_context +from ragas.testset import TestsetGenerator + from llama_index.llms.openai import OpenAI from llama_index.embeddings.openai import OpenAIEmbedding # generator with openai models -generator_llm = OpenAI(model="gpt-3.5-turbo-16k") -critic_llm = OpenAI(model="gpt-4") -embeddings = OpenAIEmbedding() +generator_llm = OpenAI(model="gpt-4o") +embeddings = OpenAIEmbedding(model="text-embedding-3-large") generator = TestsetGenerator.from_llama_index( - generator_llm=generator_llm, - critic_llm=critic_llm, - embeddings=embeddings, + llm=generator_llm, + embedding_model=embeddings, ) ``` @@ -46,23 +41,11 @@ Now you are all set to generate the dataset # generate testset testset = generator.generate_with_llamaindex_docs( documents, - test_size=5, - distributions={simple: 0.5, reasoning: 0.25, multi_context: 0.25}, + testset_size=5, ) ``` - embedding nodes: 0%| | 0/54 [00:00 - question - contexts - ground_truth - evolution_type - metadata - episode_done + user_input + reference_contexts + reference + synthesizer_name 0 - What cultural movement began in New York City ... - [ Others cite the end of the crack epidemic an... - The Harlem Renaissance - simple - [{'file_path': '/home/jjmachan/jjmachan/explod... - True + Why was New York named after the Duke of York? + [Etymology ==\n\nIn 1664, New York was named i... + New York was named after the Duke of York in 1... + AbstractQuerySynthesizer 1 - What is the significance of New York City's tr... - [ consisting of 51 council members whose distr... - New York City's transportation system is both ... - simple - [{'file_path': '/home/jjmachan/jjmachan/explod... - True + How did the early Europan exploraton and setle... + [History ==\n\n\n=== Early history ===\nIn the... + The early European exploration and settlement ... + AbstractQuerySynthesizer 2 - What factors led to the creation of Central Pa... - [ next ten years with British troops stationed... - Public-minded members of the contemporaneous b... - reasoning - [{'file_path': '/home/jjmachan/jjmachan/explod... - True + New York City population culture finance diver... + [New York City, the most populous city in the ... + New York City is a global cultural, financial,... + ComparativeAbstractQuerySynthesizer 3 - What was the impact of the Treaty of Breda on ... - [ British raids. In 1626, the Dutch colonial D... - The Treaty of Breda confirmed the transfer of ... - multi_context - [{'file_path': '/home/jjmachan/jjmachan/explod... - True + How do the economic aspects of New York City, ... + [New York City, the most populous city in the ... + New York City's economic aspects, such as its ... + ComparativeAbstractQuerySynthesizer 4 - What role did New York play in the American Re... - [ British raids. In 1626, the Dutch colonial D... - New York played a significant role in the Amer... - simple - [{'file_path': '/home/jjmachan/jjmachan/explod... - True + What role do biomedical research institutions ... + [Education ==\n\n \n\nNew York City has the la... + Biomedical research institutions in New York C... + SpecificQuerySynthesizer @@ -160,8 +131,7 @@ Since we already loaded the dataset into `documents` lets use that. ```python # build query engine -from llama_index.core import VectorStoreIndex, SimpleDirectoryReader -from llama_index.core.settings import Settings +from llama_index.core import VectorStoreIndex vector_index = VectorStoreIndex.from_documents(documents) @@ -174,24 +144,24 @@ Lets try an sample question from the generated testset to see if it is working ```python # convert it to pandas dataset df = testset.to_pandas() -df["question"][0] +df["user_input"][0] ``` - 'What cultural movement began in New York City and established the African-American literary canon in the United States?' + 'Why was New York named after the Duke of York?' ```python -response_vector = query_engine.query(df["question"][0]) +response_vector = query_engine.query(df["user_input"][0]) print(response_vector) ``` - The Harlem Renaissance was the cultural movement that began in New York City and established the African-American literary canon in the United States. + New York was named after the Duke of York because in 1664, the city was named in honor of the Duke of York, who later became King James II of England. ## Evaluating the `QueryEngine` @@ -210,54 +180,38 @@ Now lets import the metrics we will be using to evaluate ```python +# import metrics from ragas.metrics import ( - faithfulness, - answer_relevancy, - context_precision, - context_recall, + Faithfulness, + AnswerRelevancy, + ContextPrecision, + ContextRecall, ) -from ragas.metrics.critique import harmfulness +# init metrics with evaluator LLM +from ragas.llms import LlamaIndexLLMWrapper +evaluator_llm = LlamaIndexLLMWrapper(OpenAI(model="gpt-4o")) metrics = [ - faithfulness, - answer_relevancy, - context_precision, - context_recall, - harmfulness, + Faithfulness(llm=evaluator_llm), + AnswerRelevancy(llm=evaluator_llm), + ContextPrecision(llm=evaluator_llm), + ContextRecall(llm=evaluator_llm), ] ``` -now lets init the evaluator model - - -```python -from llama_index.llms.openai import OpenAI -from llama_index.embeddings.openai import OpenAIEmbedding - -# using GPT 3.5, use GPT 4 / 4-turbo for better accuracy -evaluator_llm = OpenAI(model="gpt-3.5-turbo") -``` - the `evaluate()` function expects a dict of "question" and "ground_truth" for metrics. You can easily convert the `testset` to that format ```python -# convert to HF dataset -ds = testset.to_dataset() - -ds_dict = ds.to_dict() -ds_dict["question"] -ds_dict["ground_truth"] +# convert to Ragas Evaluation Dataset +ragas_dataset = testset.to_evaluation_dataset() +ragas_dataset ``` - ['The Harlem Renaissance', - "New York City's transportation system is both complex and extensive, with a comprehensive mass transit system that accounts for one in every three users of mass transit in the United States. The New York City Subway system is the largest rapid transit system in the world, and the city has a high usage of public transport, with a majority of households not owning a car. Due to their reliance on mass transit, New Yorkers spend less of their household income on transportation compared to the national average.", - 'Public-minded members of the contemporaneous business elite lobbied for the establishment of Central Park', - 'The Treaty of Breda confirmed the transfer of New Amsterdam to English control and the renaming of the settlement as New York. The Duke of York, who would later become King James II and VII, played a significant role in the naming of New York City.', - 'New York played a significant role in the American Revolution. The Stamp Act Congress met in New York in October 1765, and the city became a center for the Sons of Liberty organization. Skirmishes and battles took place in and around New York, including the Battle of Long Island and the Battle of Saratoga. The city was occupied by British forces for much of the war, but it was eventually liberated by American troops in 1783.'] + EvaluationDataset(features=['user_input', 'reference_contexts', 'reference'], len=7) @@ -270,34 +224,17 @@ from ragas.integrations.llama_index import evaluate result = evaluate( query_engine=query_engine, metrics=metrics, - dataset=ds_dict, - llm=evaluator_llm, - embeddings=OpenAIEmbedding(), + dataset=ragas_dataset, ) ``` - Running Query Engine: 0%| | 0/5 [00:00 - question - contexts - answer - ground_truth + user_input + retrieved_contexts + reference_contexts + response + reference faithfulness answer_relevancy context_precision context_recall - harmfulness 0 - What cultural movement began in New York City ... - [=== 19th century ===\n\nOver the course of th... - The Harlem Renaissance of literary and cultura... - The Harlem Renaissance - 0.5 - 0.907646 - 0.5 + What events led to New York being named after ... + [New York City is the headquarters of the glob... + [Etymology ==\n\nIn 1664, New York was named i... + New York was named in honor of the Duke of Yor... + New York was named after the Duke of York in 1... + 1.000000 + 0.950377 + 1.0 1.0 - 0 1 - What is the significance of New York City's tr... - [== Transportation ==\n\nNew York City's compr... - New York City's transportation system is signi... - New York City's transportation system is both ... - 1.0 - 0.986921 - 1.0 + How early European explorers and Native Americ... + [=== Dutch rule ===\n\nA permanent European pr... + [History ==\n\n\n=== Early history ===\nIn the... + Early European explorers established a permane... + Early European explorers and Native Americans ... + 1.000000 + 0.896300 1.0 - 0 + 0.8 2 - What factors led to the creation of Central Pa... - [=== 19th century ===\n\nOver the course of th... - Prominent American literary figures lived in N... - Public-minded members of the contemporaneous b... - 1.0 - 0.805014 + New York City population economy challenges + [=== Wealth and income disparity ===\nNew York... + [New York City, the most populous city in the ... + New York City has faced challenges related to ... + New York City, as the most populous city in th... + 1.000000 + 0.915717 1.0 - 1.0 - 0 + 0.0 3 - What was the impact of the Treaty of Breda on ... - [=== Dutch rule ===\n\nA permanent European pr... - The Treaty of Breda resulted in the transfer o... - The Treaty of Breda confirmed the transfer of ... + How do the economic aspects of New York City, ... + [=== Wealth and income disparity ===\nNew York... + [New York City, the most populous city in the ... + The economic aspects of New York City, as a gl... + New York City's economic aspects as a global c... + 0.913043 + 0.929317 1.0 - 0.860931 + 0.0 + + + 4 + What are some of the cultural and architectura... + [==== Staten Island ====\nStaten Island (Richm... + [Geography ==\n\nDuring the Wisconsin glaciati... + Brooklyn is known for its cultural diversity, ... + Brooklyn is distinct within New York City due ... + 1.000000 + 0.902664 + 0.5 + 1.0 + + + 5 + What measures has New York City implemented to... + [==== International events ====\nIn terms of h... + [Environment ==\n\n \nEnvironmental issues in ... + New York City has implemented various measures... + New York City has implemented several measures... + 0.909091 + 1.000000 1.0 1.0 - 0 - 4 - What role did New York play in the American Re... + 6 + What role did New York City play during the Am... [=== Province of New York and slavery ===\n\nI... - New York served as a significant location duri... - New York played a significant role in the Amer... - 1.0 - 0.935846 + [History ==\n\n\n=== Early history ===\nIn the... + New York City served as a significant military... + During the American Revolution, New York City ... + 1.000000 + 1.000000 1.0 1.0 - 0 diff --git a/docs/howtos/integrations/llamaindex.ipynb b/docs/howtos/integrations/llamaindex.ipynb index 471940f3f..ddf677957 100644 --- a/docs/howtos/integrations/llamaindex.ipynb +++ b/docs/howtos/integrations/llamaindex.ipynb @@ -25,11 +25,9 @@ ] }, { - "cell_type": "code", - "execution_count": 1, + "cell_type": "markdown", "id": "096e5af0", "metadata": {}, - "outputs": [], "source": [ "# load the documents\n", "from llama_index.core import SimpleDirectoryReader\n", @@ -77,167 +75,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "fe03839d", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "32696e6eb3f04785a9251312a5c3e82d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Applying [SummaryExtractor, HeadlinesExtractor]: 0%| | 0/2 [00:00