Skip to content

Commit

Permalink
adding ability to count by wildcard (source_resource_type).
Browse files Browse the repository at this point in the history 
internal fields like `id`, `source_id`, `source_resource_id` and `source_resource_type` are now queryable via keyword type.
  • Loading branch information
@AnalogJ
AnalogJ committed Aug 23, 2023
1 parent dbc2f49 commit 3fe7291
Show file tree
Hide file tree
Showing 63 changed files with 1,277 additions and 853 deletions.
16 changes: 14 additions & 2 deletions backend/pkg/database/sqlite_repository_query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ const (
SearchParameterTypeQuantity SearchParameterType = "quantity"
SearchParameterTypeComposite SearchParameterType = "composite"
SearchParameterTypeSpecial SearchParameterType = "special"

SearchParameterTypeKeyword SearchParameterType = "keyword" //this is a literal/string primitive.

)

const TABLE_ALIAS = "fhir"
Expand Down Expand Up @@ -148,6 +151,12 @@ func (sr *SqliteRepository) sqlQueryResources(ctx context.Context, query models.
//populate the group by and order by clause with the count by values
query.Aggregations.OrderBy = "count(*) DESC"
query.Aggregations.GroupBy = query.Aggregations.CountBy

if query.Aggregations.GroupBy == "*" {
//we need to get the count of all resources, so we need to remove the group by clause and replace it by
// `source_resource_type` which will be the same for all resources
query.Aggregations.GroupBy = "source_resource_type"
}
}

//process order by clause
Expand Down Expand Up @@ -322,7 +331,7 @@ func ProcessSearchParameterValue(searchParameter SearchParameter, searchValueWit
SecondaryValues: map[string]interface{}{},
Value: searchValueWithPrefix,
}
if (searchParameter.Type == SearchParameterTypeString || searchParameter.Type == SearchParameterTypeUri) && len(searchParameterValue.Value.(string)) == 0 {
if (searchParameter.Type == SearchParameterTypeString || searchParameter.Type == SearchParameterTypeUri || searchParameter.Type == SearchParameterTypeKeyword) && len(searchParameterValue.Value.(string)) == 0 {
return searchParameterValue, fmt.Errorf("invalid search parameter value: (%s=%s)", searchParameter.Name, searchParameterValue.Value)
}

Expand Down Expand Up @@ -533,6 +542,9 @@ func SearchCodeToWhereClause(searchParam SearchParameter, searchParamValue Searc
}
return fmt.Sprintf("(%s)", clause), searchClauseNamedParams, nil

case SearchParameterTypeKeyword:
//setup the clause
return fmt.Sprintf("(%s = @%s)", searchParam.Name, NamedParameterWithSuffix(searchParam.Name, namedParameterSuffix)), searchClauseNamedParams, nil
case SearchParameterTypeReference:
return "", nil, fmt.Errorf("search parameter type %s not supported", searchParam.Type)
}
Expand Down Expand Up @@ -593,7 +605,7 @@ func ProcessAggregationParameter(aggregationFieldWithProperty string, searchPara
}

//primitive types should not have a modifier, we need to throw an error
if aggregationParameter.Type == SearchParameterTypeNumber || aggregationParameter.Type == SearchParameterTypeUri {
if aggregationParameter.Type == SearchParameterTypeNumber || aggregationParameter.Type == SearchParameterTypeUri || aggregationParameter.Type == SearchParameterTypeKeyword {
if len(aggregationParameter.Modifier) > 0 {
return aggregationParameter, fmt.Errorf("primitive aggregation parameter %s cannot have a property (%s)", aggregationParameter.Name, aggregationParameter.Modifier)
}
Expand Down
107 changes: 107 additions & 0 deletions backend/pkg/database/sqlite_repository_query_sql_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,41 @@ func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithPrimitiveOrderBy
})
}

func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithKeywordOrderByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})

//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")

sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{},
From: "CarePlan",
Aggregations: &models.QueryResourceAggregations{OrderBy: "id"},
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars

//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT fhir.*",
"FROM fhir_care_plan as fhir",
"WHERE (user_id = ?)",
"GROUP BY `fhir`.`id`",
"ORDER BY fhir.id ASC",
}, " "), sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"00000000-0000-0000-0000-000000000000",
})
}

func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithComplexOrderByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
Expand Down Expand Up @@ -254,6 +289,78 @@ func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithPrimitiveCountBy
})
}

func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithKeywordCountByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})

//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")

sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{
"activityCode": "test_code",
},
From: "CarePlan",
Aggregations: &models.QueryResourceAggregations{CountBy: "source_resource_type"},
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars

//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT fhir.source_resource_type as label, count(*) as value",
"FROM fhir_care_plan as fhir, json_each(fhir.activityCode) as activityCodeJson",
"WHERE ((activityCodeJson.value ->> '$.code' = ?)) AND (user_id = ?)",
"GROUP BY `fhir`.`source_resource_type`",
"ORDER BY count(*) DESC",
}, " "), sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"test_code", "00000000-0000-0000-0000-000000000000",
})
}

func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithWildcardCountByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
sqliteRepo.GormClient = sqliteRepo.GormClient.Session(&gorm.Session{DryRun: true})

//test
authContext := context.WithValue(context.Background(), pkg.ContextKeyTypeAuthUsername, "test_username")

sqlQuery, err := sqliteRepo.sqlQueryResources(authContext, models.QueryResource{
Select: []string{},
Where: map[string]interface{}{},
From: "CarePlan",
Aggregations: &models.QueryResourceAggregations{CountBy: "*"},
})
require.NoError(suite.T(), err)
var results []map[string]interface{}
statement := sqlQuery.Find(&results).Statement
sqlString := statement.SQL.String()
sqlParams := statement.Vars

//assert
require.NoError(suite.T(), err)
require.Equal(suite.T(),
strings.Join([]string{
"SELECT fhir.source_resource_type as label, count(*) as value",
"FROM fhir_care_plan as fhir",
"WHERE (user_id = ?)",
"GROUP BY `fhir`.`source_resource_type`",
"ORDER BY count(*) DESC",
}, " "), sqlString)
require.Equal(suite.T(), sqlParams, []interface{}{
"00000000-0000-0000-0000-000000000000",
})
}

func (suite *RepositorySqlTestSuite) TestQueryResources_SQL_WithComplexCountByAggregation() {
//setup
sqliteRepo := suite.TestRepository.(*SqliteRepository)
Expand Down
6 changes: 6 additions & 0 deletions backend/pkg/database/sqlite_repository_query_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ func TestProcessSearchParameter(t *testing.T) {
{"unknown:doesntmatter", map[string]string{"test": "string"}, SearchParameter{}, true}, //unknown search parameter shoudl throw error
{"unknown", map[string]string{"test": "string"}, SearchParameter{}, true}, //unknown search parameter shoudl throw error
{"test", map[string]string{"test": "faketype"}, SearchParameter{Type: "faketype", Name: "test", Modifier: ""}, false},
{"id", map[string]string{"id": "keyword"}, SearchParameter{Type: "keyword", Name: "id", Modifier: ""}, false},

{"given", map[string]string{"given": "string"}, SearchParameter{Type: "string", Name: "given", Modifier: ""}, false},
{"given:contains", map[string]string{"given": "string"}, SearchParameter{Type: "string", Name: "given", Modifier: "contains"}, false},
Expand Down Expand Up @@ -112,6 +113,8 @@ func TestProcessSearchParameterValue(t *testing.T) {
{SearchParameter{Type: "quantity", Name: "valueQuantity", Modifier: ""}, "ap5.4|http://unitsofmeasure.org|mg|additional", SearchParameterValue{Value: float64(5.4), Prefix: "ap", SecondaryValues: map[string]interface{}{"valueQuantitySystem": "http://unitsofmeasure.org", "valueQuantityCode": "mg|additional"}}, false},
{SearchParameter{Type: "quantity", Name: "valueQuantity", Modifier: ""}, "5.4||", SearchParameterValue{Value: float64(5.4), Prefix: "", SecondaryValues: map[string]interface{}{}}, false},
{SearchParameter{Type: "quantity", Name: "valueQuantity", Modifier: ""}, "", SearchParameterValue{}, true},

{SearchParameter{Type: "keyword", Name: "id", Modifier: ""}, "1234", SearchParameterValue{Value: "1234", SecondaryValues: map[string]interface{}{}}, false},
}

//test && assert
Expand Down Expand Up @@ -155,6 +158,8 @@ func TestSearchCodeToWhereClause(t *testing.T) {
{SearchParameter{Type: "token", Name: "code", Modifier: ""}, SearchParameterValue{Value: "ha125", Prefix: "", SecondaryValues: map[string]interface{}{"codeSystem": "http://acme.org/conditions/codes"}}, "0_0", "(codeJson.value ->> '$.code' = @code_0_0 AND codeJson.value ->> '$.system' = @codeSystem_0_0)", map[string]interface{}{"code_0_0": "ha125", "codeSystem_0_0": "http://acme.org/conditions/codes"}, false},
{SearchParameter{Type: "token", Name: "code", Modifier: ""}, SearchParameterValue{Value: "ha125", Prefix: "", SecondaryValues: map[string]interface{}{}}, "0_0", "(codeJson.value ->> '$.code' = @code_0_0)", map[string]interface{}{"code_0_0": "ha125"}, false},
{SearchParameter{Type: "token", Name: "identifier", Modifier: "otype"}, SearchParameterValue{Value: "MR|446053", Prefix: "", SecondaryValues: map[string]interface{}{"identifierSystem": "http://terminology.hl7.org/CodeSystem/v2-0203"}}, "0_0", "(identifierJson.value ->> '$.code' = @identifier_0_0 AND identifierJson.value ->> '$.system' = @identifierSystem_0_0)", map[string]interface{}{"identifier_0_0": "MR|446053", "identifierSystem_0_0": "http://terminology.hl7.org/CodeSystem/v2-0203"}, false},

{SearchParameter{Type: "keyword", Name: "id", Modifier: ""}, SearchParameterValue{Value: "1234", Prefix: "", SecondaryValues: map[string]interface{}{}}, "0_0", "(id = @id_0_0)", map[string]interface{}{"id_0_0": "1234"}, false},
}

//test && assert
Expand All @@ -181,6 +186,7 @@ func TestSearchCodeToFromClause(t *testing.T) {
}{
{SearchParameter{Type: "number", Name: "probability", Modifier: ""}, "", false},
{SearchParameter{Type: "date", Name: "issueDate", Modifier: ""}, "", false},
{SearchParameter{Type: "keyword", Name: "id", Modifier: ""}, "", false},
{SearchParameter{Type: "token", Name: "hello", Modifier: ""}, "json_each(fhir.hello) as helloJson", false},
}

Expand Down
29 changes: 17 additions & 12 deletions backend/pkg/models/database/fhir_account.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,18 +54,23 @@ type FhirAccount struct {

func (s *FhirAccount) GetSearchParameters() map[string]string {
searchParameters := map[string]string{
"identifier": "token",
"language": "token",
"lastUpdated": "date",
"name": "string",
"owner": "reference",
"period": "date",
"profile": "reference",
"status": "token",
"subject": "reference",
"tag": "token",
"text": "string",
"type": "special",
"id": "keyword",
"identifier": "token",
"language": "token",
"lastUpdated": "date",
"name": "string",
"owner": "reference",
"period": "date",
"profile": "reference",
"source_id": "keyword",
"source_resource_id": "keyword",
"source_resource_type": "keyword",
"source_uri": "keyword",
"status": "token",
"subject": "reference",
"tag": "token",
"text": "string",
"type": "special",
}
return searchParameters
}
Expand Down
41 changes: 23 additions & 18 deletions backend/pkg/models/database/fhir_adverse_event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,24 +72,29 @@ type FhirAdverseEvent struct {

func (s *FhirAdverseEvent) GetSearchParameters() map[string]string {
searchParameters := map[string]string{
"actuality": "token",
"category": "token",
"date": "date",
"event": "token",
"language": "token",
"lastUpdated": "date",
"location": "reference",
"profile": "reference",
"recorder": "reference",
"resultingcondition": "reference",
"seriousness": "token",
"severity": "token",
"study": "reference",
"subject": "reference",
"substance": "reference",
"tag": "token",
"text": "string",
"type": "special",
"actuality": "token",
"category": "token",
"date": "date",
"event": "token",
"id": "keyword",
"language": "token",
"lastUpdated": "date",
"location": "reference",
"profile": "reference",
"recorder": "reference",
"resultingcondition": "reference",
"seriousness": "token",
"severity": "token",
"source_id": "keyword",
"source_resource_id": "keyword",
"source_resource_type": "keyword",
"source_uri": "keyword",
"study": "reference",
"subject": "reference",
"substance": "reference",
"tag": "token",
"text": "string",
"type": "special",
}
return searchParameters
}
Expand Down
45 changes: 25 additions & 20 deletions backend/pkg/models/database/fhir_allergy_intolerance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,26 +148,31 @@ type FhirAllergyIntolerance struct {

func (s *FhirAllergyIntolerance) GetSearchParameters() map[string]string {
searchParameters := map[string]string{
"asserter": "reference",
"category": "token",
"clinicalStatus": "token",
"code": "token",
"criticality": "token",
"date": "date",
"identifier": "token",
"language": "token",
"lastDate": "date",
"lastUpdated": "date",
"manifestation": "token",
"onset": "date",
"profile": "reference",
"recorder": "reference",
"route": "token",
"severity": "token",
"tag": "token",
"text": "string",
"type": "special",
"verificationStatus": "token",
"asserter": "reference",
"category": "token",
"clinicalStatus": "token",
"code": "token",
"criticality": "token",
"date": "date",
"id": "keyword",
"identifier": "token",
"language": "token",
"lastDate": "date",
"lastUpdated": "date",
"manifestation": "token",
"onset": "date",
"profile": "reference",
"recorder": "reference",
"route": "token",
"severity": "token",
"source_id": "keyword",
"source_resource_id": "keyword",
"source_resource_type": "keyword",
"source_uri": "keyword",
"tag": "token",
"text": "string",
"type": "special",
"verificationStatus": "token",
}
return searchParameters
}
Expand Down
49 changes: 27 additions & 22 deletions backend/pkg/models/database/fhir_appointment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,28 +84,33 @@ type FhirAppointment struct {

func (s *FhirAppointment) GetSearchParameters() map[string]string {
searchParameters := map[string]string{
"actor": "reference",
"appointmentType": "token",
"basedOn": "reference",
"date": "date",
"identifier": "token",
"language": "token",
"lastUpdated": "date",
"location": "reference",
"partStatus": "token",
"practitioner": "reference",
"profile": "reference",
"reasonCode": "token",
"reasonReference": "reference",
"serviceCategory": "token",
"serviceType": "token",
"slot": "reference",
"specialty": "token",
"status": "token",
"supportingInfo": "reference",
"tag": "token",
"text": "string",
"type": "special",
"actor": "reference",
"appointmentType": "token",
"basedOn": "reference",
"date": "date",
"id": "keyword",
"identifier": "token",
"language": "token",
"lastUpdated": "date",
"location": "reference",
"partStatus": "token",
"practitioner": "reference",
"profile": "reference",
"reasonCode": "token",
"reasonReference": "reference",
"serviceCategory": "token",
"serviceType": "token",
"slot": "reference",
"source_id": "keyword",
"source_resource_id": "keyword",
"source_resource_type": "keyword",
"source_uri": "keyword",
"specialty": "token",
"status": "token",
"supportingInfo": "reference",
"tag": "token",
"text": "string",
"type": "special",
}
return searchParameters
}
Expand Down

0 comments on commit 3fe7291

Please sign in to comment.