/
dbToTextTask.go
270 lines (230 loc) · 8.07 KB
/
dbToTextTask.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
// Copyright (c) 2016 OpenM++
// This code is licensed under the MIT license (see LICENSE.txt for details)
package main
import (
"database/sql"
"errors"
"os"
"path/filepath"
"strconv"
"github.com/openmpp/go/ompp/config"
"github.com/openmpp/go/ompp/db"
"github.com/openmpp/go/ompp/helper"
"github.com/openmpp/go/ompp/omppLog"
)
// copy modeling task metadata and run history from database into text json and csv files
func dbToTextTask(modelName string, modelDigest string, runOpts *config.RunOptions) error {
// get task name and id
taskName := runOpts.String(taskNameArgKey)
taskId := runOpts.Int(taskIdArgKey, 0)
// conflicting options: use task id if positive else use task name
if runOpts.IsExist(taskNameArgKey) && runOpts.IsExist(taskIdArgKey) {
if taskId > 0 {
omppLog.Log("dbcopy options conflict. Using task id: ", taskId, " ignore task name: ", taskName)
taskName = ""
} else {
omppLog.Log("dbcopy options conflict. Using task name: ", taskName, " ignore task id: ", taskId)
taskId = 0
}
}
if taskId < 0 || taskId == 0 && taskName == "" {
return errors.New("dbcopy invalid argument(s) for task id: " + runOpts.String(taskIdArgKey) + " and/or task name: " + runOpts.String(taskNameArgKey))
}
// use of run and set id's in directory names:
// do this by default or if use id name = true
// only if use id name = false then do not use id's in directory names
isUseIdNames := !runOpts.IsExist(useIdNamesArgKey) || runOpts.Bool(useIdNamesArgKey)
// open source database connection and check is it valid
cs, dn := db.IfEmptyMakeDefault(modelName, runOpts.String(dbConnStrArgKey), runOpts.String(dbDriverArgKey))
srcDb, _, err := db.Open(cs, dn, false)
if err != nil {
return err
}
defer srcDb.Close()
if err := db.CheckOpenmppSchemaVersion(srcDb); err != nil {
return err
}
// get model metadata
modelDef, err := db.GetModel(srcDb, modelName, modelDigest)
if err != nil {
return err
}
modelName = modelDef.Model.Name // set model name: it can be empty and only model digest specified
// get task metadata by id or name
var taskRow *db.TaskRow
var outDir string
if taskId > 0 {
if taskRow, err = db.GetTask(srcDb, taskId); err != nil {
return err
}
if taskRow == nil {
return errors.New("modeling task not found, task id: " + strconv.Itoa(taskId))
}
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".task."+strconv.Itoa(taskId))
} else {
if taskRow, err = db.GetTaskByName(srcDb, modelDef.Model.ModelId, taskName); err != nil {
return err
}
if taskRow == nil {
return errors.New("modeling task not found: " + taskName)
}
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".task."+taskName)
}
meta, err := db.GetTaskFull(srcDb, taskRow, "") // get task full metadata, including task run history
if err != nil {
return err
}
// create new output directory for task metadata
err = os.MkdirAll(outDir, 0750)
if err != nil {
return err
}
// write task metadata into json file
if err = toTaskJson(srcDb, modelDef, meta, outDir, isUseIdNames); err != nil {
return err
}
// save runs from model run history
var runIdLst []int
var isRunNotFound, isRunNotCompleted bool
dblFmt := runOpts.String(doubleFormatArgKey)
isIdCsv := runOpts.Bool(useIdCsvArgKey)
isWriteUtf8bom := runOpts.Bool(useUtf8CsvArgKey)
for j := range meta.TaskRun {
nextRun:
for k := range meta.TaskRun[j].TaskRunSet {
// check is this run id already processed
runId := meta.TaskRun[j].TaskRunSet[k].RunId
for i := range runIdLst {
if runId == runIdLst[i] {
continue nextRun
}
}
runIdLst = append(runIdLst, runId)
// find model run metadata by id
runRow, err := db.GetRun(srcDb, runId)
if err != nil {
return err
}
if runRow == nil {
isRunNotFound = true
continue // skip: run not found
}
// run must be completed: status success, error or exit
if !db.IsRunCompleted(runRow.Status) {
isRunNotCompleted = true
continue // skip: run not completed
}
rm, err := db.GetRunFullText(srcDb, runRow, "") // get full model run metadata
if err != nil {
return err
}
// write model run metadata into json, parameters and output result values into csv files
if err = toRunText(srcDb, modelDef, rm, outDir, dblFmt, isIdCsv, isWriteUtf8bom, isUseIdNames); err != nil {
return err
}
}
}
// find workset by set id and save it's metadata to json and workset parameters to csv
var wsIdLst []int
var isSetNotFound, isSetNotReadOnly bool
var fws = func(dbConn *sql.DB, setId int) error {
// check is workset already processed
for i := range wsIdLst {
if setId == wsIdLst[i] {
return nil
}
}
wsIdLst = append(wsIdLst, setId)
// get workset by id
wsRow, err := db.GetWorkset(dbConn, setId)
if err != nil {
return err
}
if wsRow == nil { // exit: workset not found
isSetNotFound = true
return nil
}
if !wsRow.IsReadonly { // exit: workset not readonly
isSetNotReadOnly = true
return nil
}
wm, err := db.GetWorksetFull(dbConn, wsRow, "") // get full workset metadata
if err != nil {
return err
}
// write workset metadata into json and parameter values into csv files
if err = toWorksetText(dbConn, modelDef, wm, outDir, dblFmt, isIdCsv, isWriteUtf8bom, isUseIdNames); err != nil {
return err
}
return nil
}
// save task body worksets
for k := range meta.Set {
if err = fws(srcDb, meta.Set[k]); err != nil {
return err
}
}
// save worksets from model run history
for j := range meta.TaskRun {
for k := range meta.TaskRun[j].TaskRunSet {
if err = fws(srcDb, meta.TaskRun[j].TaskRunSet[k].SetId); err != nil {
return err
}
}
}
// display warnings if any worksets not found or not readonly
// display warnings if any model runs not exists or not completed
if isSetNotFound {
omppLog.Log("Warning: task ", meta.Task.Name, " workset(s) not found, copy of task incomplete")
}
if isSetNotReadOnly {
omppLog.Log("Warning: task ", meta.Task.Name, " workset(s) not readonly, copy of task incomplete")
}
if isRunNotFound {
omppLog.Log("Warning: task ", meta.Task.Name, " model run(s) not found, copy of task run history incomplete")
}
if isRunNotCompleted {
omppLog.Log("Warning: task ", meta.Task.Name, " model run(s) not completed, copy of task run history incomplete")
}
// pack worksets metadata json and csv files into zip
if runOpts.Bool(zipArgKey) {
zipPath, err := helper.PackZip(outDir, "")
if err != nil {
return err
}
omppLog.Log("Packed ", zipPath)
}
return nil
}
// toTaskListJson convert all successfully completed tasks and tasks run history to json and write into json files
func toTaskListJson(dbConn *sql.DB, modelDef *db.ModelMeta, outDir string, isUseIdNames bool) error {
// get all modeling tasks and successfully completed tasks run history
tl, err := db.GetTaskFullList(dbConn, modelDef.Model.ModelId, true, "")
if err != nil {
return err
}
// read each task metadata and write into json files
for k := range tl {
if err := toTaskJson(dbConn, modelDef, &tl[k], outDir, isUseIdNames); err != nil {
return err
}
}
return nil
}
// toTaskJson convert modeling task and task run history to json and write into json file
func toTaskJson(dbConn *sql.DB, modelDef *db.ModelMeta, meta *db.TaskMeta, outDir string, isUseIdNames bool) error {
// convert db rows into "public" format
omppLog.Log("Modeling task ", meta.Task.TaskId, " ", meta.Task.Name)
pub, err := meta.ToPublic(dbConn, modelDef)
if err != nil {
return err
}
// save modeling task metadata into json
var fname string
if !isUseIdNames {
fname = modelDef.Model.Name + ".task." + helper.CleanPath(meta.Task.Name) + ".json"
} else {
fname = modelDef.Model.Name + ".task." + strconv.Itoa(meta.Task.TaskId) + "." + helper.CleanPath(meta.Task.Name) + ".json"
}
return helper.ToJsonFile(filepath.Join(outDir, fname), pub)
}