/
qaStudentResultsCheck.go
222 lines (198 loc) · 6.53 KB
/
qaStudentResultsCheck.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
package reports
import (
"encoding/csv"
"fmt"
"github.com/nsip/dev-nrt/helper"
"github.com/nsip/dev-nrt/records"
"github.com/tidwall/gjson"
"github.com/tidwall/sjson"
)
//
// capture the connected graph of any
// detected codeframe issue
//
type studentresultsCodeframeIssue struct {
eor *records.EventOrientedRecord
testRefId string
testLocalId string
testletRefId string
testletLocalId string
itemRefId string
itemLocalId string
itemSubRefIds map[string]struct{}
itemSubLocalId string
substitue bool
}
type QaStudentResultsCheck struct {
baseReport // embed common setup capability
cfh helper.CodeframeHelper
issues []*studentresultsCodeframeIssue
}
//
// Checks that items-testlets-tests seen in responses
// are all in the codeframe
//
// NOTE: This is a Collector report, it harvests data from the stream but
// only writes it out once all data has passed through.
//
//
func QaStudentResultsCheckReport(cfh helper.CodeframeHelper) *QaStudentResultsCheck {
r := QaStudentResultsCheck{cfh: cfh}
r.initialise("./config/QaStudentResultsCheck.toml")
r.printStatus()
return &r
}
//
// implement the EventPipe interface, core work of the
// report engine.
//
func (r *QaStudentResultsCheck) ProcessEventRecords(in chan *records.EventOrientedRecord) chan *records.EventOrientedRecord {
out := make(chan *records.EventOrientedRecord)
go func() {
defer close(out)
// open the csv file writer, and set the header
w := csv.NewWriter(r.outF)
defer r.outF.Close()
w.Write(r.config.header)
defer w.Flush()
for eor := range in {
if !r.config.activated { // only process if activated
out <- eor
continue
}
//
// check for codeframe errors
//
r.validate(eor)
out <- eor
}
//
// Write out issues collected
//
for _, issue := range r.issues {
//
// generate any calculated fields required
//
issue.eor.CalculatedFields = r.calculateFields(issue)
//
// now loop through the ouput definitions to create a
// row of results
//
var result string
var row []string = make([]string, 0, len(r.config.queries))
for _, query := range r.config.queries {
result = issue.eor.GetValueString(query)
row = append(row, result)
}
// write the row to the output file
if err := w.Write(row); err != nil {
fmt.Println("Warning: error writing record to csv:", r.config.name, err)
}
}
}()
return out
}
//
// generates a block of json that can be added to the
// record containing values that are not in the original data
//
//
func (r *QaStudentResultsCheck) calculateFields(issue *studentresultsCodeframeIssue) []byte {
json := issue.eor.CalculatedFields
// add details of any issues found
json, _ = sjson.SetBytes(json, "CalculatedFields.NAPTestlet.RefId", issue.testletRefId)
json, _ = sjson.SetBytes(json, "CalculatedFields.NAPTestlet.TestletContent.NAPTestletLocalId", issue.testletLocalId)
json, _ = sjson.SetBytes(json, "CalculatedFields.NAPTestItem.RefId", issue.itemRefId)
json, _ = sjson.SetBytes(json, "CalculatedFields.NAPTestItem.TestItemContent.NAPTestItemLocalId", issue.itemLocalId)
json, _ = sjson.SetBytes(json, "CalculatedFields.ErrorType", "item->testlet->test not found in codeframe")
//
// this will be true if a substitute item has been used as the primary reference in the
// codeframe - in effect the item/substitute refernces have been created the wrong way round
//
if issue.substitue {
json, _ = sjson.SetBytes(json, "CalculatedFields.SubstituteItemRefId", issue.itemSubRefIds)
json, _ = sjson.SetBytes(json, "CalculatedFields.SubstituteItemLocalId", issue.itemSubLocalId)
json, _ = sjson.SetBytes(json, "CalculatedFields.ErrorType", "codeframe acyclical use of substitute item")
}
return json
}
//
// checks integrity of codeframe assets in the response member of the record
// iterates each item-testlet-test triplet and tests against the data in the
// codeframe helper.
// Any discrepancies in the members or the linkage will be reported.
// Mutiple issues may be detected in the same response.
//
//
func (r *QaStudentResultsCheck) validate(eor *records.EventOrientedRecord) {
//
// iterate response testlets
//
gjson.GetBytes(eor.NAPStudentResponseSet, "NAPStudentResponseSet.TestletList.Testlet").
ForEach(func(key, value gjson.Result) bool {
cfi := &studentresultsCodeframeIssue{}
containers := make(map[string][]string, 0)
//
// get testlet identifiers
//
cfi.testletRefId = value.Get("NAPTestletRefId").String()
cfi.testletLocalId = value.Get("NAPTestletLocalId").String()
//
// now iterate testlet item responses
//
value.Get("ItemResponseList.ItemResponse").
ForEach(func(key, value gjson.Result) bool {
//
// get item identifiers
//
cfi.itemRefId = value.Get("NAPTestItemRefId").String()
cfi.itemLocalId = value.Get("NAPTestItemLocalId").String()
//
// first we need ot see if this is a substitute item
//
if cfi.itemSubRefIds, cfi.substitue = r.cfh.IsSubstituteItem(cfi.itemRefId); cfi.substitue {
//
// if it is then use the substitute to look up the
// codeframe validity, codeframe itself doesn;t know about
// substitute items
//
for subRefId := range cfi.itemSubRefIds {
for k, v := range r.cfh.GetContainersForItem(subRefId) {
containers[k] = v // add testlet-test reference to lookup
}
}
cfi.itemSubLocalId = r.cfh.GetCodeframeObjectValueString(cfi.itemRefId, "NAPTestItem.TestItemContent.NAPTestItemLocalId")
} else {
//
// if not we check the validity of the item as given
//
containers = r.cfh.GetContainersForItem(cfi.itemRefId)
}
// get test identifiers
//
cfi.testRefId = eor.GetValueString("NAPTest.RefId")
cfi.testLocalId = eor.GetValueString("NAPTest.TestContent.NAPTestLocalId")
//
// now we check that the available combinations of test and testlet
// in the codeframe for this item have at least one match with the
// combination captured in this response
//
for testlet, tests := range containers {
for _, test := range tests {
if testlet == cfi.testletRefId && test == cfi.testRefId {
return true // keep iterating - gjson equiv. of continue - to next item response
}
}
}
//
// if we got here then our item has an issue
// add to the issues list
//
cfi.eor = eor
r.issues = append(r.issues, cfi)
//
return true // keep iterating, move on to next item response
})
return true // keep iterating
})
}