-
Notifications
You must be signed in to change notification settings - Fork 1
/
dbToTextWorkset.go
234 lines (200 loc) · 6.61 KB
/
dbToTextWorkset.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
// Copyright (c) 2016 OpenM++
// This code is licensed under the MIT license (see LICENSE.txt for details)
package main
import (
"database/sql"
"errors"
"os"
"path/filepath"
"strconv"
"time"
"github.com/openmpp/go/ompp/config"
"github.com/openmpp/go/ompp/db"
"github.com/openmpp/go/ompp/helper"
"github.com/openmpp/go/ompp/omppLog"
)
// copy workset from database into text json and csv files
func dbToTextWorkset(modelName string, modelDigest string, runOpts *config.RunOptions) error {
// get workset name and id
setName := runOpts.String(setNameArgKey)
setId := runOpts.Int(setIdArgKey, 0)
// conflicting options: use set id if positive else use set name
if runOpts.IsExist(setNameArgKey) && runOpts.IsExist(setIdArgKey) {
if setId > 0 {
omppLog.Log("dbcopy options conflict. Using set id: ", setId, " ignore set name: ", setName)
setName = ""
} else {
omppLog.Log("dbcopy options conflict. Using set name: ", setName, " ignore set id: ", setId)
setId = 0
}
}
if setId < 0 || setId == 0 && setName == "" {
return errors.New("dbcopy invalid argument(s) for set id: " + runOpts.String(setIdArgKey) + " and/or set name: " + runOpts.String(setNameArgKey))
}
// open source database connection and check is it valid
cs, dn := db.IfEmptyMakeDefaultReadOnly(modelName, runOpts.String(fromSqliteArgKey), runOpts.String(dbConnStrArgKey), runOpts.String(dbDriverArgKey))
srcDb, _, err := db.Open(cs, dn, false)
if err != nil {
return err
}
defer srcDb.Close()
if err := db.CheckOpenmppSchemaVersion(srcDb); err != nil {
return err
}
// get model metadata
modelDef, err := db.GetModel(srcDb, modelName, modelDigest)
if err != nil {
return err
}
modelName = modelDef.Model.Name // set model name: it can be empty and only model digest specified
// "root" directory for workset metadata
// later this "root" combined with modelName.set.name or modelName.set.id
outDir := ""
if runOpts.IsExist(paramDirArgKey) {
outDir = filepath.Clean(runOpts.String(paramDirArgKey))
} else {
if setId > 0 {
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".set."+strconv.Itoa(setId))
} else {
outDir = filepath.Join(runOpts.String(outputDirArgKey), modelName+".set."+setName)
}
}
fileCreated := make(map[string]bool)
// get workset metadata by id or name
var wsRow *db.WorksetRow
if setId > 0 {
if wsRow, err = db.GetWorkset(srcDb, setId); err != nil {
return err
}
if wsRow == nil {
return errors.New("workset not found, set id: " + strconv.Itoa(setId))
}
} else {
if wsRow, err = db.GetWorksetByName(srcDb, modelDef.Model.ModelId, setName); err != nil {
return err
}
if wsRow == nil {
return errors.New("workset not found: " + setName)
}
}
wm, err := db.GetWorksetFull(srcDb, wsRow, "") // get full workset metadata
if err != nil {
return err
}
// check: workset must be readonly
if !wm.Set.IsReadonly {
return errors.New("workset must be readonly: " + strconv.Itoa(wsRow.SetId) + " " + wsRow.Name)
}
// create new output directory for workset metadata
if !theCfg.isKeepOutputDir {
if ok := dirDeleteAndLog(outDir); !ok {
return errors.New("Error: unable to delete: " + outDir)
}
}
if err = os.MkdirAll(outDir, 0750); err != nil {
return err
}
// for single run or single workset output to text
// do not use of run and set id's in directory names by default
// only use id's in directory names if:
// dbcopy option use id name = true or user specified run id or workset id
isUseIdNames := runOpts.Bool(useIdNamesArgKey)
// write workset metadata into json and parameter values into csv files
if err = toWorksetText(srcDb, modelDef, wm, outDir, fileCreated, isUseIdNames); err != nil {
return err
}
// pack worksets metadata json and csv files into zip
if runOpts.Bool(zipArgKey) {
zipPath, err := helper.PackZip(outDir, !theCfg.isKeepOutputDir, "")
if err != nil {
return err
}
omppLog.Log("Packed ", zipPath)
}
return nil
}
// toWorksetListText write all readonly worksets into csv files, each set in separate subdirectory
func toWorksetListText(
dbConn *sql.DB,
modelDef *db.ModelMeta,
outDir string,
fileCreated map[string]bool,
isUseIdNames bool) error {
// get all readonly worksets
wl, err := db.GetWorksetFullList(dbConn, modelDef.Model.ModelId, true, "")
if err != nil {
return err
}
// read all workset parameters and dump it into csv files
for k := range wl {
err = toWorksetText(dbConn, modelDef, &wl[k], outDir, fileCreated, isUseIdNames)
if err != nil {
return err
}
}
return nil
}
// toWorksetText write workset metadata into json file
// and parameters into csv files, in separate subdirectory
// by default file name and directory name include set id: modelName.set.1234.SetName
// user can explicitly disable it by IdNames=false
func toWorksetText(
dbConn *sql.DB,
modelDef *db.ModelMeta,
meta *db.WorksetMeta,
outDir string,
fileCreated map[string]bool,
isUseIdNames bool) error {
// convert db rows into "public" format
setId := meta.Set.SetId
omppLog.Log("Workset ", setId, " ", meta.Set.Name)
pub, err := meta.ToPublic(dbConn, modelDef)
if err != nil {
return err
}
// create workset subdir under output dir
var csvName string
if !isUseIdNames {
csvName = "set." + helper.CleanPath(pub.Name)
} else {
csvName = "set." + strconv.Itoa(setId) + "." + helper.CleanPath(pub.Name)
}
csvDir := filepath.Join(outDir, csvName)
if !theCfg.isKeepOutputDir {
if ok := dirDeleteAndLog(csvDir); !ok {
return errors.New("Error: unable to delete: " + csvDir)
}
}
if err = os.MkdirAll(csvDir, 0750); err != nil {
return err
}
// write all parameters into csv files
nP := len(pub.Param)
omppLog.Log(" Parameters: ", nP)
logT := time.Now().Unix()
for j := 0; j < nP; j++ {
cvtParam := &db.CellParamConverter{
ModelDef: modelDef,
Name: pub.Param[j].Name,
IsIdCsv: theCfg.isIdCsv,
DoubleFmt: theCfg.doubleFmt,
}
paramLt := db.ReadParamLayout{
ReadLayout: db.ReadLayout{
Name: pub.Param[j].Name,
FromId: setId,
},
IsFromSet: true,
}
logT = omppLog.LogIfTime(logT, logPeriod, " ", j, " of ", nP, ": ", paramLt.Name)
err = toCellCsvFile(dbConn, modelDef, paramLt, cvtParam, fileCreated, csvDir, "", "")
if err != nil {
return err
}
}
// save model workset metadata into json
if err := helper.ToJsonFile(filepath.Join(outDir, modelDef.Model.Name+"."+csvName+".json"), pub); err != nil {
return err
}
return nil
}