Fixture Export Fixes, Re-export and Clean Up (MeasureAuthoringTool#1129)

* re-exported core set of fixtures. fixed fixture loading and some errored tests

* fixed user_id in fixture exporting. reexported core_measures. fixed unit tests

* moved CMS347v4 from special set to deprecated set because it cannot be found

.rubocop_todo.yml

@@ -1184,3 +1184,7 @@ Style/ZeroLengthPredicate:
 # URISchemes: http, https
 Metrics/LineLength:
   Max: 3564
+
+Lint/UriEscapeUnescape:
+  Exclude:
+    - 'test/vcr_setup.rb'

lib/tasks/fixture_helper.rb

@@ -16,10 +16,10 @@ def add_collection(collection)
     fixture_json = JSON.parse(File.read(json_fixture_file))
     next if fixture_json.empty?
     # Value_sets are arrays of objects, unlike measures etc, so we need to iterate in that case.
-    fixture_json = [fixture_json] unless fixture_json.is_a? Array
+    fixture_json = [fixture_json] unless fixture_json.is_a?(Array)
     fixture_json.each do |fj|
       convert_times(fj)
-      insert_mongoid_ids(fj)
+      convert_mongoid_ids(fj)
       fix_binary_data(fj)
       begin
         Mongoid.default_client[collection_name].insert_one(fj)
@@ -40,17 +40,16 @@ def convert_times(json)
   end
 end
 
-def insert_mongoid_ids(json)
-  return nil unless json.is_a?(Hash)
-  json.each_pair do |k,v|
-    if v && v.is_a?(Hash)
-      if v["$oid"]
+def convert_mongoid_ids(json)
+  if json.is_a?(Array)
+    json.each { |val| convert_mongoid_ids(val) }
+  elsif json.is_a?(Hash)
+    json.each_pair do |k,v|
+      if v && v.is_a?(Hash) && v["$oid"]
         json[k] = BSON::ObjectId.from_string(v["$oid"])
       else
-        insert_mongoid_ids(v)
+        convert_mongoid_ids(v)
       end
-    elsif %w[_id bundle_id user_id].include?(k)
-      json[k] = BSON::ObjectId.from_string(v) unless v.nil?
     end
   end
 end

lib/util/fixture_exporter.rb

@@ -1,7 +1,5 @@
 # An abstract class that provides fixture exporting functionality
 class FixtureExporter
-  # In order to avoid storing details of real users, a test-specific user fixture exists.
-  @bonnie_fixtures_user_id = '501fdba3044a111b98000001'
 
   def initialize(user, measure: nil, records: nil)
     @user = user
@@ -83,7 +81,7 @@ def find_component_measures
 
   def as_transformed_hash(mongoid_doc)
     doc = make_hash_and_apply_any_transforms(mongoid_doc)
-    doc['user_id'] = @bonnie_fixtures_user_id if doc['user_id'].present?
+    doc['user_id'] = bonnie_fixtures_user_id if doc.key? 'user_id'
     return doc
   end
 
@@ -143,9 +141,19 @@ class FrontendFixtureExporter < FixtureExporter
   alias export_value_sets export_value_sets_as_map
   alias export_records export_records_as_array
 
+  def initialize(user, measure: nil, records: nil)
+    super(user, measure: measure, records: records)
+  end
+
   def make_hash_and_apply_any_transforms(mongoid_doc)
     return JSON.parse(mongoid_doc.to_json, max_nesting: 1000)
   end
+
+  def bonnie_fixtures_user_id
+    # In order to avoid storing details of real users, a test-specific user fixture exists.
+    # for front end tests this should be just a string
+    '501fdba3044a111b98000001'
+  end
 end
 
 # Used to export fixtures for use in backend tests (where fixtures are loaded into mongo)
@@ -158,4 +166,9 @@ def make_hash_and_apply_any_transforms(mongoid_doc)
     objectid_to_oids(doc)
     return JSON.parse(doc.to_json, max_nesting: 1000)
   end
+
+  def bonnie_fixtures_user_id
+    # In order to avoid storing details of real users, a test-specific user fixture exists.
+    { '$oid': '501fdba3044a111b98000001' }
+  end
 end

spec/javascripts/cql_calculator_spec.js.coffee

@@ -236,7 +236,7 @@ describe 'cqlCalculator', ->
         result = @cql_calculator.calculate(@measure.get('populations').first(), patient)
 
         # there will be a single result in the episode_results
-        expect(result.get('episode_results')).toEqual({'5a58f529942c6d5479457abc': { IPP: 1, DENOM: 1, NUMER: 1}})
+        expect(result.get('episode_results')).toEqual({'5aeb7763b848463d625b33d2': { IPP: 1, DENOM: 1, NUMER: 1}})
         # NUMER should be the only not relevant population
         expect(result.get('population_relevance')).toEqual({ IPP: true, DENOM: true, NUMER: true })
 

spec/javascripts/fixtures/json/README

@@ -69,33 +69,30 @@ bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_meas
 CMS761 - measure with "Participation"
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMS761,bonnie-fixtures@mitre.org,CMS761v0]
 
-CMS460 - OpioidData, HUGE data statement in library
+!! NEEDS NEW PACKAGE - CMS460 - OpioidData, HUGE data statement in library
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMS460,bonnie-fixtures@mitre.org,CMS460v0]
 
 CMSv9999 - measure with "let" in common library function
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMSv9999,bonnie-fixtures@mitre.org,CMSv9999,Patient,Blank]
 
-CMS341v1 - measure with multiple libraries that each have population statements
+CMS334v1 - measure with multiple libraries that each have population statements
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMS334v1,bonnie-fixtures@mitre.org,CMS334v1]
 
-CMS529v0 - Hybrid Measure with Supplemental Data Elements
+!! NEEDS TO BE REUPLOADED WITH SDE CALC - CMS529v0 - Hybrid Measure with Supplemental Data Elements
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMS529v0,bonnie-fixtures@mitre.org,CMS529v0]
 
-CMS54321v0 - Measure that contains new QDM 5.4 elements
+!! CAN NOT BE FOUND - CMS54321v0 - Measure that contains new QDM 5.4 elements
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMSv54321,bonnie-fixtures@mitre.org,CMSv54321]
 
-CMS136v7 - Proportion measure with two population sets with Medication Active and Dispensed
+!! NEEDS NEW PACKAGE (136v8 doesnt work either) - CMS136v7 - Proportion measure with two population sets with Medication Active and Dispensed
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMS136,bonnie-fixtures@mitre.org,CMS136v7]
 
-CMS890v0 - Composite Measure
+!! TODO update later - CMS890v0 - Composite Measure
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMS890,bonnie-fixtures@mitre.org,CMS890v0]
 
 CMS10v0 - Measure from Jira test BONNIE-980
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/CMS10,bonnie-fixtures@mitre.org,CMS10v0]
 
-cms136 - proportion measure with two population sets with medication active and dispensed
-bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/cms136,bonnie-fixtures@mitre.org,cms136v7]
-
 cms146 - episode of care based measure with medication ordered. includes the the hospice library.
 bundle exec rake bonnie:fixtures:generate_frontend_cql_fixtures[cms,special_measures/cms146,bonnie-fixtures@mitre.org,cms146v6]
 

spec/javascripts/fixtures/json/measure_data/core_measures/CMS134/CMS134v6.json

@@ -1,6 +1,7 @@
 {
   "_id": "5a57eb19942c6d1e61d33552",
   "bundle_id": "5a57e977942c6d1e61d32f14",
+  "calculate_sdes": null,
   "category": "Miscellaneous",
   "cms_id": "CMS134v6",
   "complexity": {
@@ -131,6 +132,11 @@
       }
     ]
   },
+  "component": false,
+  "component_hqmf_set_ids": [
+
+  ],
+  "composite": false,
   "continuous_variable": false,
   "cql": [
     "library DiabetesMedicalAttentionforNephropathy version '6.1.003'\r\n\r\nusing QDM version '5.3'\r\n\r\ninclude Hospice version '0.1.000' called Hospice\r\ninclude MATGlobalCommonFunctions version '1.0.000' called Global\r\n\r\nvalueset \"ONC Administrative Sex\": 'urn:oid:2.16.840.1.113762.1.4.1'\r\nvalueset \"Race\": 'urn:oid:2.16.840.1.114222.4.11.836'\r\nvalueset \"Ethnicity\": 'urn:oid:2.16.840.1.114222.4.11.837'\r\nvalueset \"Payer\": 'urn:oid:2.16.840.1.114222.4.11.3591'\r\nvalueset \"ACE Inhibitor or ARB\": 'urn:oid:2.16.840.1.113883.3.526.3.1139'\r\nvalueset \"Annual Wellness Visit\": 'urn:oid:2.16.840.1.113883.3.526.3.1240'\r\nvalueset \"Diabetes\": 'urn:oid:2.16.840.1.113883.3.464.1003.103.12.1001'\r\nvalueset \"Diabetic Nephropathy\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1004'\r\nvalueset \"Dialysis Education\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1016'\r\nvalueset \"Dialysis Services\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1013'\r\nvalueset \"ESRD Monthly Outpatient Services\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1014'\r\nvalueset \"Face-to-Face Interaction\": 'urn:oid:2.16.840.1.113883.3.464.1003.101.12.1048'\r\nvalueset \"Glomerulonephritis and Nephrotic Syndrome\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1018'\r\nvalueset \"Home Healthcare Services\": 'urn:oid:2.16.840.1.113883.3.464.1003.101.12.1016'\r\nvalueset \"Hypertensive Chronic Kidney Disease\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1017'\r\nvalueset \"Kidney Transplant\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1012'\r\nvalueset \"Office Visit\": 'urn:oid:2.16.840.1.113883.3.464.1003.101.12.1001'\r\nvalueset \"Other Services Related to Dialysis\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1015'\r\nvalueset \"Preventive Care Services - Established Office Visit, 18 and Up\": 'urn:oid:2.16.840.1.113883.3.464.1003.101.12.1025'\r\nvalueset \"Preventive Care Services-Initial Office Visit, 18 and Up\": 'urn:oid:2.16.840.1.113883.3.464.1003.101.12.1023'\r\nvalueset \"Proteinuria\": 'urn:oid:2.16.840.1.113883.3.526.3.1003'\r\nvalueset \"Urine Protein Tests\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1024'\r\nvalueset \"Vascular Access for Dialysis\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1011'\r\nvalueset \"Kidney Failure\": 'urn:oid:2.16.840.1.113883.3.464.1003.109.12.1028'\r\n\r\nparameter \"Measurement Period\" Interval<DateTime>\r\n\r\ncontext Patient\r\n\r\ndefine \"SDE Ethnicity\":\r\n\t[\"Patient Characteristic Ethnicity\": \"Ethnicity\"]\r\n\r\ndefine \"SDE Payer\":\r\n\t[\"Patient Characteristic Payer\": \"Payer\"]\r\n\r\ndefine \"SDE Race\":\r\n\t[\"Patient Characteristic Race\": \"Race\"]\r\n\r\ndefine \"SDE Sex\":\r\n\t[\"Patient Characteristic Sex\": \"ONC Administrative Sex\"]\r\n\r\ndefine \"Renal Procedures\":\r\n\t( [\"Procedure, Performed\": \"Kidney Transplant\"]\r\n\t\t\tunion [\"Procedure, Performed\": \"Vascular Access for Dialysis\"]\r\n\t\t\tunion [\"Procedure, Performed\": \"Dialysis Services\"]\r\n\t)\r\n\r\ndefine \"Renal Interventions\":\r\n\t[\"Intervention, Performed\": \"Other Services Related to Dialysis\"]\r\n\t\tunion [\"Intervention, Performed\": \"Dialysis Education\"]\r\n\r\ndefine \"Qualifying Encounters\":\r\n\t( [\"Encounter, Performed\": \"Office Visit\"]\r\n\t\tunion [\"Encounter, Performed\": \"Face-to-Face Interaction\"]\r\n\t\tunion [\"Encounter, Performed\": \"Preventive Care Services - Established Office Visit, 18 and Up\"]\r\n\t\tunion [\"Encounter, Performed\": \"Preventive Care Services-Initial Office Visit, 18 and Up\"]\r\n\t\tunion [\"Encounter, Performed\": \"Home Healthcare Services\"]\r\n\t\tunion [\"Encounter, Performed\": \"Annual Wellness Visit\"] ) ValidEncounter\r\n\t\twhere ValidEncounter.relevantPeriod during \"Measurement Period\"\r\n\r\ndefine \"Protein Urea Lab Test\":\r\n\t[\"Laboratory Test, Performed\": \"Urine Protein Tests\"] ProteinUrea\r\n\t\twhere ProteinUrea.result is not null\r\n\r\ndefine \"Nephropathy Diagnoses\":\r\n\t( [\"Diagnosis\": \"Hypertensive Chronic Kidney Disease\"]\r\n\t\tunion [\"Diagnosis\": \"Kidney Failure\"]\r\n\t\tunion [\"Diagnosis\": \"Glomerulonephritis and Nephrotic Syndrome\"]\r\n\t\tunion [\"Diagnosis\": \"Diabetic Nephropathy\"]\r\n\t\tunion [\"Diagnosis\": \"Proteinuria\"] ) NephropathyDiagnoses\r\n\t\twhere NephropathyDiagnoses.prevalencePeriod overlaps \"Measurement Period\"\r\n\r\ndefine \"ESRD Encounter\":\r\n\t[\"Encounter, Performed\": \"ESRD Monthly Outpatient Services\"]\r\n\r\ndefine \"Active ACEI or ARB Medications\":\r\n\t[\"Medication, Active\": \"ACE Inhibitor or ARB\"] ACEIorARB\r\n\t\twhere ACEIorARB.relevantPeriod overlaps \"Measurement Period\"\r\n\r\ndefine \"Initial Population\":\r\n\texists [\"Patient Characteristic Birthdate\"] BirthDate\r\n\t\twhere Global.\"CalendarAgeInYearsAt\"(BirthDate.birthDatetime, start of \"Measurement Period\")>= 18\r\n\t\t\tand Global.\"CalendarAgeInYearsAt\"(BirthDate.birthDatetime, start of \"Measurement Period\")< 75\r\n\t\t\tand exists ( \"Qualifying Encounters\" )\r\n\t\t\tand exists ( [\"Diagnosis\": \"Diabetes\"] DiabetesDx\r\n\t\t\t\t\twhere DiabetesDx.prevalencePeriod overlaps \"Measurement Period\"\r\n\t\t\t)\r\n\r\ndefine \"Denominator\":\r\n\t\"Initial Population\"\r\n\r\ndefine \"Numerator\":\r\n\texists ( \"Active ACEI or ARB Medications\" )\r\n\t\tor exists ( \"Nephropathy Diagnoses\" )\r\n\t\tor exists ( \"Nephropathy Screenings\" )\r\n\r\ndefine \"In Hospice\":\r\n\tHospice.\"Has Hospice\"\r\n\r\ndefine \"Denominator Exclusions\":\r\n\t\"In Hospice\"\r\n\r\ndefine \"Nephropathy Screenings\":\r\n\t( \"Renal Procedures\"\r\n\t\tunion \"Renal Interventions\"\r\n\t\tunion \"ESRD Encounter\"\r\n\t\tunion \"Protein Urea Lab Test\" ) NephropathyScreenings\r\n\t\twhere NephropathyScreenings.relevantPeriod during \"Measurement Period\"\r\n",
@@ -14030,7 +14036,7 @@
   "title": "Diabetes: Medical Attention for Nephropathy",
   "type": "ep",
   "updated_at": "2018-02-15T19:54:14.839Z",
-  "user_id": "5a57e975942c6d1e61d32f13",
+  "user_id": "501fdba3044a111b98000001",
   "value_set_oid_version_objects": [
     {
       "oid": "2.16.840.1.113762.1.4.1",

spec/javascripts/fixtures/json/measure_data/core_measures/CMS158/CMS158v6.json

@@ -1,6 +1,7 @@
 {
   "_id": "5a57eb37942c6d1e61d33813",
   "bundle_id": "5a57e977942c6d1e61d32f14",
+  "calculate_sdes": null,
   "category": "Miscellaneous",
   "cms_id": "CMS158v6",
   "complexity": {
@@ -51,6 +52,11 @@
       }
     ]
   },
+  "component": false,
+  "component_hqmf_set_ids": [
+
+  ],
+  "composite": false,
   "continuous_variable": false,
   "cql": [
     "library PregnantwomenthathadHBsAgtesting version '6.0.001'\r\n\r\nusing QDM version '5.3'\r\n\r\nvalueset \"ONC Administrative Sex\": 'urn:oid:2.16.840.1.113762.1.4.1'\r\nvalueset \"Race\": 'urn:oid:2.16.840.1.114222.4.11.836'\r\nvalueset \"Ethnicity\": 'urn:oid:2.16.840.1.114222.4.11.837'\r\nvalueset \"Payer\": 'urn:oid:2.16.840.1.114222.4.11.3591'\r\nvalueset \"Delivery - Diagnosis\": 'urn:oid:2.16.840.1.113883.3.67.1.101.1.278'\r\nvalueset \"Delivery - Procedure\": 'urn:oid:2.16.840.1.113762.1.4.1078.5'\r\nvalueset \"Female\": 'urn:oid:2.16.840.1.113883.3.560.100.2'\r\nvalueset \"HBsAg\": 'urn:oid:2.16.840.1.113883.3.67.1.101.1.279'\r\nvalueset \"Hepatitis B\": 'urn:oid:2.16.840.1.113883.3.67.1.101.1.269'\r\n\r\nparameter \"Measurement Period\" Interval<DateTime>\r\n\r\ncontext Patient\r\n\r\ndefine \"SDE Ethnicity\":\r\n\t[\"Patient Characteristic Ethnicity\": \"Ethnicity\"]\r\n\r\ndefine \"SDE Payer\":\r\n\t[\"Patient Characteristic Payer\": \"Payer\"]\r\n\r\ndefine \"SDE Race\":\r\n\t[\"Patient Characteristic Race\": \"Race\"]\r\n\r\ndefine \"SDE Sex\":\r\n\t[\"Patient Characteristic Sex\": \"ONC Administrative Sex\"]\r\n\r\ndefine \"Most Recent Delivery\":\r\n\tLast([\"Procedure, Performed\": \"Delivery - Procedure\"] DeliveryProcedure\r\n\t\t\twhere DeliveryProcedure.relevantPeriod during \"Measurement Period\"\r\n\t\t\tsort by start of relevantPeriod\r\n\t)\r\n\r\ndefine \"Denominator Exceptions\":\r\n\texists ( [\"Diagnosis\": \"Hepatitis B\"] HepB\r\n\t\t\twith \"Most Recent Delivery Overlaps Diagnosis\" DeliveryOverlapsDiagnosis\r\n\t\t\t\tsuch that HepB.prevalencePeriod starts 365 days or less before start of DeliveryOverlapsDiagnosis.relevantPeriod\r\n\t\t\t\t\tor HepB.prevalencePeriod ends 365 days or less before start of DeliveryOverlapsDiagnosis.relevantPeriod\r\n\t)\r\n\r\ndefine \"Most Recent Delivery Overlaps Diagnosis\":\r\n\t\"Most Recent Delivery\" MostRecentDelivery\r\n\t\twhere exists ( [\"Diagnosis\": \"Delivery - Diagnosis\"] DeliveryDiagnosis\r\n\t\t\t\twhere MostRecentDelivery.relevantPeriod overlaps DeliveryDiagnosis.prevalencePeriod\r\n\t\t)\r\n\r\ndefine \"Initial Population\":\r\n\t\"Most Recent Delivery Overlaps Diagnosis\" is not null\r\n\t\tand AgeInYearsAt(start of \"Measurement Period\")>= 12\r\n\t\tand exists ( [\"Patient Characteristic Sex\": \"Female\"] )\r\n\r\ndefine \"Numerator\":\r\n\texists ( [\"Laboratory Test, Performed\": \"HBsAg\"] LabTest\r\n\t\t\twith \"Most Recent Delivery Overlaps Diagnosis\" DeliveryOverlapsDiagnosis\r\n\t\t\t\tsuch that LabTest.relevantPeriod starts 280 days or less before start of DeliveryOverlapsDiagnosis.relevantPeriod\r\n\t)\r\n\r\ndefine \"Denominator\":\r\n\t\"Initial Population\"\r\n"
@@ -5064,7 +5070,7 @@
   "title": "Pregnant women that had HBsAg testing",
   "type": "ep",
   "updated_at": "2018-01-11T22:54:47.980Z",
-  "user_id": "5a57e975942c6d1e61d32f13",
+  "user_id": "501fdba3044a111b98000001",
   "value_set_oid_version_objects": [
     {
       "oid": "2.16.840.1.113762.1.4.1",