diff --git a/evaluate/report/collection.go b/evaluate/report/collection.go index 4be27944..c533aa0d 100644 --- a/evaluate/report/collection.go +++ b/evaluate/report/collection.go @@ -20,6 +20,9 @@ type AssessmentPerModelPerLanguagePerRepository map[model.Model]map[language.Lan // AssessmentPerModel holds a collection of assessments per model. type AssessmentPerModel map[model.Model]metrics.Assessments +// AssessmentPerLanguagePerModel holds a collection of assessments per language and model. +type AssessmentPerLanguagePerModel map[language.Language]map[model.Model]metrics.Assessments + // NewAssessmentPerModelPerLanguagePerRepository returns a new AssessmentPerModelPerLanguagePerRepository initialized with an empty assessment for each combination. func NewAssessmentPerModelPerLanguagePerRepository(models []model.Model, languages []language.Language, repositories []string) AssessmentPerModelPerLanguagePerRepository { a := AssessmentPerModelPerLanguagePerRepository{} @@ -85,6 +88,26 @@ func (a AssessmentPerModelPerLanguagePerRepository) CollapseByModel() Assessment return perModel } +// CollapseByLanguage returns all assessments aggregated per language and model. +func (a AssessmentPerModelPerLanguagePerRepository) CollapseByLanguage() AssessmentPerLanguagePerModel { + assessments := AssessmentPerLanguagePerModel{} + _ = a.Walk(func(m model.Model, l language.Language, r string, a metrics.Assessments) error { + if _, ok := assessments[l]; !ok { + assessments[l] = map[model.Model]metrics.Assessments{} + } + + if _, ok := assessments[l][m]; !ok { + assessments[l][m] = metrics.NewAssessments() + } + + assessments[l][m].Add(a) + + return nil + }) + + return assessments +} + // WalkByScore walks the given assessment metrics by their score. func WalkByScore(assessmentsPerModel AssessmentPerModel, function func(model model.Model, assessment metrics.Assessments, score uint) error) error { models := maps.Keys(assessmentsPerModel) diff --git a/evaluate/report/collection_test.go b/evaluate/report/collection_test.go index f4f9b33c..11d44628 100644 --- a/evaluate/report/collection_test.go +++ b/evaluate/report/collection_test.go @@ -295,3 +295,89 @@ func TestAssessmentCollapseByModel(t *testing.T) { }, }) } + +func TestAssessmentCollapseByLanguage(t *testing.T) { + type testCase struct { + Name string + + Assessments AssessmentPerModelPerLanguagePerRepository + + ExpectedAssessmentPerLanguagePerModel AssessmentPerLanguagePerModel + } + + validate := func(t *testing.T, tc *testCase) { + t.Run(tc.Name, func(t *testing.T) { + actualAssessmentPerLanguagePerModel := tc.Assessments.CollapseByLanguage() + + assert.Equal(t, tc.ExpectedAssessmentPerLanguagePerModel, actualAssessmentPerLanguagePerModel) + }) + } + + modelA := modeltesting.NewMockModelNamed(t, "some-model-a") + modelB := modeltesting.NewMockModelNamed(t, "some-model-b") + + languageA := languagetesting.NewMockLanguageNamed(t, "some-language-a") + languageB := languagetesting.NewMockLanguageNamed(t, "some-language-b") + + validate(t, &testCase{ + Name: "Collapse", + + Assessments: AssessmentPerModelPerLanguagePerRepository{ + modelA: { + languageA: { + "some-repository-a": metrics.Assessments{ + metrics.AssessmentKeyResponseNoExcess: 1, + }, + "some-repository-b": metrics.Assessments{ + metrics.AssessmentKeyResponseNoExcess: 2, + }, + }, + languageB: { + "some-repository-a": metrics.Assessments{ + metrics.AssessmentKeyResponseNoExcess: 3, + }, + "some-repository-b": metrics.Assessments{ + metrics.AssessmentKeyResponseNoExcess: 4, + }, + }, + }, + modelB: { + languageA: { + "some-repository-a": metrics.Assessments{ + metrics.AssessmentKeyResponseNoExcess: 5, + }, + "some-repository-b": metrics.Assessments{ + metrics.AssessmentKeyResponseNoExcess: 6, + }, + }, + languageB: { + "some-repository-a": metrics.Assessments{ + metrics.AssessmentKeyResponseNoExcess: 7, + }, + "some-repository-b": metrics.Assessments{ + metrics.AssessmentKeyResponseNoExcess: 8, + }, + }, + }, + }, + + ExpectedAssessmentPerLanguagePerModel: AssessmentPerLanguagePerModel{ + languageA: map[model.Model]metrics.Assessments{ + modelA: { + metrics.AssessmentKeyResponseNoExcess: 3, + }, + modelB: { + metrics.AssessmentKeyResponseNoExcess: 11, + }, + }, + languageB: map[model.Model]metrics.Assessments{ + modelA: { + metrics.AssessmentKeyResponseNoExcess: 7, + }, + modelB: { + metrics.AssessmentKeyResponseNoExcess: 15, + }, + }, + }, + }) +}