ability to filter by dataset name and model name (#578)

Striveworks · May 9, 2024 · 1bc8965 · 1bc8965
1 parent 849a2bf
commit 1bc8965
Show file tree

Hide file tree

Showing 2 changed files with 122 additions and 0 deletions.
diff --git a/ts-client/src/ValorClient.ts b/ts-client/src/ValorClient.ts
@@ -581,6 +581,35 @@ export class ValorClient {
     });
   }
 
+  /**
+   * Fetches all evaluations associated to given models and dataset names
+   *
+   * @param modelNames names of the models
+   * @param datasetNames names of the datasets
+   * @param offset The start index of the evaluations to return. Used for pagination.
+   * @param limit The number of evaluations to return. Used for pagination.
+   * @param metricsToSortBy A map of metrics to sort the evaluations by.
+   *
+   * @returns {Promise<Evaluation[]>}
+   */
+  public async getEvaluationsByModelNamesAndDatasetNames(
+    modelNames: string[],
+    datasetNames: string[],
+    offset?: number,
+    limit?: number,
+    metricsToSortBy?: {
+      [key: string]: string | { [inner_key: string]: string };
+    }
+  ): Promise<Evaluation[]> {
+    return this.getEvaluations({
+      models: modelNames.join(','),
+      datasets: datasetNames.join(','),
+      offset: offset,
+      limit: limit,
+      metrics_to_sort_by: metricsToSortBy != null ? JSON.stringify(metricsToSortBy) : null
+    });
+  }
+
   /**
    * Adds ground truth annotations to a dataset
    *

diff --git a/ts-client/tests/ValorClient.test.ts b/ts-client/tests/ValorClient.test.ts
@@ -257,6 +257,99 @@ test('evaluation methods', async () => {
   expect((await client.getEvaluationsByDatasetNames([datasetNames[0]])).length).toBe(2);
   expect((await client.getEvaluationsByDatasetNames(datasetNames)).length).toBe(4);
   expect((await client.getEvaluationsByDatasetNames(['no-such-dataset'])).length).toBe(0);
+  // check we can get evaluations by model names and dataset names
+  expect(
+    (await client.getEvaluationsByModelNamesAndDatasetNames(modelNames, datasetNames))
+      .length
+  ).toBe(4);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[0]],
+        datasetNames
+      )
+    ).length
+  ).toBe(2);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[0]],
+        [datasetNames[0]]
+      )
+    ).length
+  ).toBe(1);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[0]],
+        [datasetNames[1]]
+      )
+    ).length
+  ).toBe(1);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[1]],
+        datasetNames
+      )
+    ).length
+  ).toBe(2);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[1]],
+        [datasetNames[0]]
+      )
+    ).length
+  ).toBe(1);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[1]],
+        [datasetNames[1]]
+      )
+    ).length
+  ).toBe(1);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [...modelNames, 'fake', 'not-real'],
+        datasetNames
+      )
+    ).length
+  ).toBe(4);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [...modelNames, 'fake', 'not-real'],
+        [datasetNames[0]]
+      )
+    ).length
+  ).toBe(2);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(modelNames, [
+        ...datasetNames,
+        'fake',
+        'not-real'
+      ])
+    ).length
+  ).toBe(4);
+  expect(
+    (
+      await client.getEvaluationsByModelNamesAndDatasetNames(
+        [modelNames[0]],
+        [...datasetNames, 'fake', 'not-real']
+      )
+    ).length
+  ).toBe(2);
+  expect(
+    (await client.getEvaluationsByModelNamesAndDatasetNames(['fake'], datasetNames))
+      .length
+  ).toBe(0);
+  expect(
+    (await client.getEvaluationsByModelNamesAndDatasetNames(modelNames, ['fake'])).length
+  ).toBe(0);
   // check pagination
   expect((await client.getEvaluationsByModelNames(modelNames, 2)).length).toBe(2);
   expect((await client.getEvaluationsByModelNames(modelNames, 3)).length).toBe(1);