From 1bc8965ae664072a52b504834fbc52a7375f03ed Mon Sep 17 00:00:00 2001
From: William Chen <chenweirongduke@gmail.com>
Date: Thu, 9 May 2024 17:38:53 -0500
Subject: [PATCH] ability to filter by dataset name and model name (#578)

---
 ts-client/src/ValorClient.ts        | 29 +++++++++
 ts-client/tests/ValorClient.test.ts | 93 +++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)

diff --git a/ts-client/src/ValorClient.ts b/ts-client/src/ValorClient.ts
index 3b535c9f3..fcb3b0661 100644
--- a/ts-client/src/ValorClient.ts
+++ b/ts-client/src/ValorClient.ts
@@ -581,6 +581,35 @@ export class ValorClient {
     });
   }
 
+  /**
+   * Fetches all evaluations associated to given models and dataset names
+   *
+   * @param modelNames names of the models
+   * @param datasetNames names of the datasets
+   * @param offset The start index of the evaluations to return. Used for pagination.
+   * @param limit The number of evaluations to return. Used for pagination.
+   * @param metricsToSortBy A map of metrics to sort the evaluations by.
+   *
+   * @returns {Promise<Evaluation[]>}
+   */
+  public async getEvaluationsByModelNamesAndDatasetNames(
+    modelNames: string[],
+    datasetNames: string[],
+    offset?: number,
+    limit?: number,
+    metricsToSortBy?: {
+      [key: string]: string | { [inner_key: string]: string };
+    }
+  ): Promise<Evaluation[]> {
+    return this.getEvaluations({
+      models: modelNames.join(','),
+      datasets: datasetNames.join(','),
+      offset: offset,
+      limit: limit,
+      metrics_to_sort_by: metricsToSortBy != null ? JSON.stringify(metricsToSortBy) : null
+    });
+  }
+
   /**
    * Adds ground truth annotations to a dataset
    *
diff --git a/ts-client/tests/ValorClient.test.ts b/ts-client/tests/ValorClient.test.ts
index 28d057529..26bfca3e9 100644
--- a/ts-client/tests/ValorClient.test.ts
+++ b/ts-client/tests/ValorClient.test.ts
@@ -257,6 +257,99 @@ test('evaluation methods', async () => {
   expect((await client.getEvaluationsByDatasetNames([datasetNames[0]])).length).toBe(2);
   expect((await client.getEvaluationsByDatasetNames(datasetNames)).length).toBe(4);
   expect((await client.getEvaluationsByDatasetNames(['no-such-dataset'])).length).toBe(0);
+  // check we can get evaluations by model names and dataset names
+  expect(
+    (await client.getEvaluationsByModelNamesAndDatasetNames(modelNames, datasetNames))
+      .length
+  ).toBe(4);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[0]],
+        datasetNames
+      )
+    ).length
+  ).toBe(2);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[0]],
+        [datasetNames[0]]
+      )
+    ).length
+  ).toBe(1);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[0]],
+        [datasetNames[1]]
+      )
+    ).length
+  ).toBe(1);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[1]],
+        datasetNames
+      )
+    ).length
+  ).toBe(2);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[1]],
+        [datasetNames[0]]
+      )
+    ).length
+  ).toBe(1);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[1]],
+        [datasetNames[1]]
+      )
+    ).length
+  ).toBe(1);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [...modelNames, 'fake', 'not-real'],
+        datasetNames
+      )
+    ).length
+  ).toBe(4);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [...modelNames, 'fake', 'not-real'],
+        [datasetNames[0]]
+      )
+    ).length
+  ).toBe(2);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(modelNames, [
+        ...datasetNames,
+        'fake',
+        'not-real'
+      ])
+    ).length
+  ).toBe(4);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[0]],
+        [...datasetNames, 'fake', 'not-real']
+      )
+    ).length
+  ).toBe(2);
+  expect(
+    (await client.getEvaluationsByModelNamesAndDatasetNames(['fake'], datasetNames))
+      .length
+  ).toBe(0);
+  expect(
+    (await client.getEvaluationsByModelNamesAndDatasetNames(modelNames, ['fake'])).length
+  ).toBe(0);
   // check pagination
   expect((await client.getEvaluationsByModelNames(modelNames, 2)).length).toBe(2);
   expect((await client.getEvaluationsByModelNames(modelNames, 3)).length).toBe(1);