/
dbToTextRun.go
351 lines (301 loc) · 11.6 KB
/
dbToTextRun.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
// Copyright (c) 2016 OpenM++
// This code is licensed under the MIT license (see LICENSE.txt for details)
package main
import (
"database/sql"
"errors"
"os"
"path/filepath"
"strconv"
"github.com/openmpp/go/ompp/config"
"github.com/openmpp/go/ompp/db"
"github.com/openmpp/go/ompp/helper"
"github.com/openmpp/go/ompp/omppLog"
)
// copy model run from database into text json and csv files
func dbToTextRun(modelName string, modelDigest string, runOpts *config.RunOptions) error {
// open source database connection and check is it valid
cs, dn := db.IfEmptyMakeDefault(modelName, runOpts.String(dbConnStrArgKey), runOpts.String(dbDriverArgKey))
srcDb, _, err := db.Open(cs, dn, false)
if err != nil {
return err
}
defer srcDb.Close()
if err := db.CheckOpenmppSchemaVersion(srcDb); err != nil {
return err
}
// get model metadata
modelDef, err := db.GetModel(srcDb, modelName, modelDigest)
if err != nil {
return err
}
modelName = modelDef.Model.Name // set model name: it can be empty and only model digest specified
// find model run metadata by id, run digest or name
runId, runDigest, runName, isFirst, isLast := runIdDigestNameFromOptions(runOpts)
if runId < 0 || runId == 0 && runName == "" && runDigest == "" && !isFirst && !isLast {
return errors.New("dbcopy invalid argument(s) run id: " + runOpts.String(runIdArgKey) + ", run name: " + runOpts.String(runNameArgKey) + ", run digest: " + runOpts.String(runDigestArgKey))
}
runRow, e := findModelRunByIdDigestName(srcDb, modelDef.Model.ModelId, runId, runDigest, runName, isFirst, isLast)
if e != nil {
return e
}
if runRow == nil {
return errors.New("model run not found: " + runOpts.String(runIdArgKey) + " " + runOpts.String(runNameArgKey) + " " + runOpts.String(runDigestArgKey))
}
// check is this run belong to the model
if runRow.ModelId != modelDef.Model.ModelId {
return errors.New("model run " + strconv.Itoa(runRow.RunId) + " " + runRow.Name + " " + runRow.RunDigest + " does not belong to model " + modelName + " " + modelDigest)
}
// run must be completed: status success, error or exit
if !db.IsRunCompleted(runRow.Status) {
return errors.New("model run not completed: " + strconv.Itoa(runRow.RunId) + " " + runRow.Name)
}
// get full model run metadata
meta, err := db.GetRunFullText(srcDb, runRow, "")
if err != nil {
return err
}
// create new "root" output directory for model run metadata
// for csv files this "root" combined as root/run.1234.runName
var outDir, csvName string
switch {
case runId > 0:
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".run."+strconv.Itoa(runId))
case runDigest != "":
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".run."+helper.CleanPath(runDigest))
case runName == "" && isFirst:
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".first.run")
csvName = "first.run"
case runName == "" && isLast:
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".last.run")
csvName = "last.run"
default:
// if not run id and not digest then run name
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".run."+helper.CleanPath(runRow.Name))
}
err = os.MkdirAll(outDir, 0750)
if err != nil {
return err
}
// use of run and set id's in directory names:
// do this by default or if use id name = true
// only if use id name = false then do not use id's in directory names
isUseIdNames := !runOpts.IsExist(useIdNamesArgKey) || runOpts.Bool(useIdNamesArgKey)
// write model run metadata into json, parameters and output result values into csv files
dblFmt := runOpts.String(doubleFormatArgKey)
isIdCsv := runOpts.Bool(useIdCsvArgKey)
isWriteUtf8bom := runOpts.Bool(useUtf8CsvArgKey)
if err = toRunText(srcDb, modelDef, meta, outDir, csvName, dblFmt, isIdCsv, isWriteUtf8bom, isUseIdNames); err != nil {
return err
}
// pack model run metadata and results into zip
if runOpts.Bool(zipArgKey) {
zipPath, err := helper.PackZip(outDir, "")
if err != nil {
return err
}
omppLog.Log("Packed ", zipPath)
}
return nil
}
// toRunListText write all model runs parameters and output tables into csv files, each run in separate subdirectory
func toRunListText(
dbConn *sql.DB, modelDef *db.ModelMeta, outDir string, doubleFmt string, isIdCsv bool, isWriteUtf8bom bool, isUseIdNames bool) error {
// get all successfully completed model runs
rl, err := db.GetRunFullTextList(dbConn, modelDef.Model.ModelId, true, "")
if err != nil {
return err
}
// read all run parameters, output accumulators and expressions and dump it into csv files
for k := range rl {
err = toRunText(dbConn, modelDef, &rl[k], outDir, "", doubleFmt, isIdCsv, isWriteUtf8bom, isUseIdNames)
if err != nil {
return err
}
}
return nil
}
// toRunText write model run metadata, parameters and output tables into csv files, in separate subdirectory
// by default file name and directory name include run id: modelName.run.1234.RunName
// user can explicitly disable it by IdNames=false
func toRunText(
dbConn *sql.DB,
modelDef *db.ModelMeta,
meta *db.RunMeta,
outDir string,
csvName string,
doubleFmt string,
isIdCsv bool,
isWriteUtf8bom bool,
isUseIdNames bool) error {
// convert db rows into "public" format
runId := meta.Run.RunId
omppLog.Log("Model run ", runId, " ", meta.Run.Name)
pub, err := meta.ToPublic(dbConn, modelDef)
if err != nil {
return err
}
// create run subdir under model dir
switch {
case csvName == "" && !isUseIdNames:
csvName = "run." + helper.CleanPath(pub.Name)
case csvName == "" && isUseIdNames:
csvName = "run." + strconv.Itoa(runId) + "." + helper.CleanPath(pub.Name)
}
csvDir := filepath.Join(outDir, csvName)
err = os.MkdirAll(csvDir, 0750)
if err != nil {
return err
}
paramLt := &db.ReadParamLayout{ReadLayout: db.ReadLayout{FromId: runId}}
// write all parameters into csv file
for j := range modelDef.Param {
paramLt.Name = modelDef.Param[j].Name
cLst, _, err := db.ReadParameter(dbConn, modelDef, paramLt)
if err != nil {
return err
}
if cLst.Len() <= 0 { // parameter data must exist for all parameters
return errors.New("missing run parameter values " + paramLt.Name + " run id: " + strconv.Itoa(paramLt.FromId))
}
var pc db.CellParam
err = toCsvCellFile(
csvDir, modelDef, paramLt.Name, false, pc, cLst, doubleFmt, isIdCsv, "", isWriteUtf8bom, "", "")
if err != nil {
return err
}
}
// write output tables into csv file, if the table included in run results
tblLt := &db.ReadTableLayout{ReadLayout: db.ReadLayout{FromId: runId}}
for j := range modelDef.Table {
// check if table exist in model run results
var isFound bool
for k := range meta.Table {
isFound = meta.Table[k].TableHid == modelDef.Table[j].TableHid
if isFound {
break
}
}
if !isFound {
continue // skip table: it is suppressed and not in run results
}
// write output table expression values into csv file
tblLt.Name = modelDef.Table[j].Name
tblLt.IsAccum = false
tblLt.IsAllAccum = false
cLst, _, err := db.ReadOutputTable(dbConn, modelDef, tblLt)
if err != nil {
return err
}
var ec db.CellExpr
err = toCsvCellFile(
csvDir, modelDef, tblLt.Name, false, ec, cLst, doubleFmt, isIdCsv, "", isWriteUtf8bom, "", "")
if err != nil {
return err
}
// write output table accumulators into csv file
tblLt.IsAccum = true
tblLt.IsAllAccum = false
cLst, _, err = db.ReadOutputTable(dbConn, modelDef, tblLt)
if err != nil {
return err
}
var ac db.CellAcc
err = toCsvCellFile(
csvDir, modelDef, tblLt.Name, false, ac, cLst, doubleFmt, isIdCsv, "", isWriteUtf8bom, "", "")
if err != nil {
return err
}
// write all accumulators view into csv file
tblLt.IsAccum = true
tblLt.IsAllAccum = true
cLst, _, err = db.ReadOutputTable(dbConn, modelDef, tblLt)
if err != nil {
return err
}
var al db.CellAllAcc
err = toCsvCellFile(
csvDir, modelDef, tblLt.Name, false, al, cLst, doubleFmt, isIdCsv, "", isWriteUtf8bom, "", "")
if err != nil {
return err
}
}
// save model run metadata into json
if err := helper.ToJsonFile(filepath.Join(outDir, modelDef.Model.Name+"."+csvName+".json"), pub); err != nil {
return err
}
return nil
}
// check dbcopy run options and return only one of: run id, run digest or name to find model run.
func runIdDigestNameFromOptions(runOpts *config.RunOptions) (int, string, string, bool, bool) {
// from dbcopy options get model run id and/or run digest and/or run name
runId := runOpts.Int(runIdArgKey, 0)
runDigest := runOpts.String(runDigestArgKey)
runName := runOpts.String(runNameArgKey)
isFirst := runOpts.Bool(runFirstArgKey)
isLast := runOpts.Bool(runLastArgKey)
// conflicting options: use run id if positive else use run digest if not empty else run name
if runOpts.IsExist(runIdArgKey) && (runOpts.IsExist(runNameArgKey) || runOpts.IsExist(runDigestArgKey)) {
if runId > 0 {
if runName != "" {
omppLog.Log("dbcopy options conflict. Using run id: ", runId, " ignore run name: ", runName)
}
if runDigest != "" {
omppLog.Log("dbcopy options conflict. Using run id: ", runId, " ignore run digest: ", runDigest)
}
runName = ""
runDigest = ""
} else {
if runDigest != "" {
omppLog.Log("dbcopy options conflict. Using run digest: ", runDigest, " ignore run id: ", runId)
if runName != "" {
omppLog.Log("dbcopy options conflict. Using run digest: ", runDigest, " ignore run name: ", runName)
runName = ""
}
} else {
omppLog.Log("dbcopy options conflict. Using run name: ", runName, " ignore run id: ", runId)
}
runId = 0
}
}
if runName != "" && runDigest != "" {
omppLog.Log("dbcopy options conflict. Using run digest: ", runDigest, " ignore run name: ", runName)
runName = ""
}
if isFirst && isLast {
omppLog.Log("dbcopy options conflict: '-", runFirstArgKey, "' flag should not be combined with '-", runLastArgKey)
isFirst = false
}
if isFirst && (runId > 0 || runDigest != "") {
omppLog.Log("dbcopy options conflict: '-", runFirstArgKey, "' flag should not be combined with '-", runIdArgKey, "' or '-", runDigestArgKey, "'")
isFirst = false
}
if isLast && (runId > 0 || runDigest != "") {
omppLog.Log("dbcopy options conflict: '-", runLastArgKey, "' flag should not be combined with '-", runIdArgKey, "' or '-", runDigestArgKey, "'")
isLast = false
}
if isLast && (runId > 0 || runDigest != "") {
omppLog.Log("dbcopy options conflict: '-", runLastArgKey, "' flag should not be combined with '-", runIdArgKey, "' or '-", runDigestArgKey, "'")
isLast = false
}
return runId, runDigest, runName, isFirst, isLast
}
// find model run metadata by id, run digest, run name or last run, retun run_lst db row or nil if model run not found.
func findModelRunByIdDigestName(dbConn *sql.DB, modelId, runId int, runDigest, runName string, isFirst, isLast bool) (*db.RunRow, error) {
switch {
case runId > 0:
return db.GetRun(dbConn, runId)
case runDigest != "":
return db.GetRunByDigest(dbConn, runDigest)
case isLast && runName != "":
return db.GetLastRunByName(dbConn, modelId, runName)
case isLast && runName == "":
return db.GetLastRun(dbConn, modelId)
case isFirst && runName == "":
return db.GetFirstRun(dbConn, modelId)
default:
// if not run id and not run digest and not last run and not first run any name
// then first run by name
return db.GetRunByName(dbConn, modelId, runName)
}
}