/
textToDbWorkset.go
375 lines (311 loc) · 11.6 KB
/
textToDbWorkset.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
// Copyright (c) 2016 OpenM++
// This code is licensed under the MIT license (see LICENSE.txt for details)
package main
import (
"database/sql"
"errors"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/openmpp/go/ompp/config"
"github.com/openmpp/go/ompp/db"
"github.com/openmpp/go/ompp/helper"
"github.com/openmpp/go/ompp/omppLog"
)
// copy workset from text json and csv files into database
func textToDbWorkset(modelName string, modelDigest string, runOpts *config.RunOptions) error {
// validate parameters
if modelName == "" {
return errors.New("invalid (empty) model name")
}
// get workset name and id
setName := runOpts.String(setNameArgKey)
setId := runOpts.Int(setIdArgKey, 0)
if setId < 0 || setId == 0 && setName == "" {
return errors.New("dbcopy invalid argument(s) for set id: " + runOpts.String(setIdArgKey) + " and/or set name: " + runOpts.String(setNameArgKey))
}
// root for workset data: input directory or name of input.zip
// it is parameter directory (if specified) or input directory/modelName.set.id
// for csv files this "root" combined with sub-directory: root/set.id.setName or root/set.setName
inpDir := ""
if runOpts.IsExist(paramDirArgKey) {
inpDir = filepath.Clean(runOpts.String(paramDirArgKey))
} else {
if setId > 0 {
inpDir = filepath.Join(runOpts.String(inputDirArgKey), modelName+".set."+strconv.Itoa(setId))
} else {
inpDir = filepath.Join(runOpts.String(inputDirArgKey), modelName+".set."+setName)
}
}
// unzip if required and use unzipped directory as "root" input diretory
if runOpts.Bool(zipArgKey) {
base := filepath.Base(inpDir)
omppLog.Log("Unpack ", base, ".zip")
outDir := runOpts.String(outputDirArgKey)
if outDir == "" {
outDir = filepath.Dir(inpDir)
}
if err := helper.UnpackZip(inpDir+".zip", outDir); err != nil {
return err
}
inpDir = filepath.Join(outDir, base)
}
// get workset metadata json path and csv directory by set id or set name or both
var metaPath string
var csvDir string
if runOpts.IsExist(setNameArgKey) && runOpts.IsExist(setIdArgKey) { // both: set id and name
metaPath = filepath.Join(inpDir,
modelName+".set."+strconv.Itoa(setId)+"."+helper.CleanPath(setName)+".json")
if _, err := os.Stat(metaPath); err != nil { // clear path to indicate metadata json file does not exist
metaPath = ""
}
csvDir = filepath.Join(inpDir,
"set."+strconv.Itoa(setId)+"."+helper.CleanPath(setName))
if _, err := os.Stat(csvDir); err != nil { // clear path to indicate csv directory does not exist
csvDir = ""
}
} else { // set id or set name only
// make path search patterns for metadata json and csv directory
var cp string
if runOpts.IsExist(setNameArgKey) && !runOpts.IsExist(setIdArgKey) { // set name only
cp = "set.*" + helper.CleanPath(setName)
}
if !runOpts.IsExist(setNameArgKey) && runOpts.IsExist(setIdArgKey) { // set id only
cp = "set." + strconv.Itoa(setId) + ".*"
}
mp := modelName + "." + cp + ".json"
// find path to metadata json by pattern
fl, err := filepath.Glob(inpDir + "/" + mp)
if err != nil {
return err
}
if len(fl) >= 1 {
metaPath = fl[0]
if len(fl) > 1 {
omppLog.Log("found multiple workset metadata json files, using: " + filepath.Base(metaPath))
}
}
// csv directory:
// if metadata json file exist then check if csv directory for that json file
if metaPath != "" {
d, f := filepath.Split(metaPath)
c := strings.TrimSuffix(strings.TrimPrefix(f, modelName+"."), ".json")
if len(c) <= 4 { // expected csv directory: set.4.w or set.w
csvDir = ""
} else {
csvDir = filepath.Join(d, c)
if _, err := os.Stat(csvDir); err != nil {
csvDir = ""
}
}
} else { // metadata json file not exist: search for csv directory by pattern
fl, err := filepath.Glob(inpDir + "/" + cp)
if err != nil {
return err
}
if len(fl) >= 1 {
csvDir = fl[0]
if len(fl) > 1 {
omppLog.Log("found multiple workset csv directories, using: " + filepath.Base(csvDir))
}
}
}
}
// check results: metadata json file or csv directory must exist
if metaPath == "" && csvDir == "" {
return errors.New("no workset metadata json file and no csv directory, workset: " + strconv.Itoa(setId) + " " + setName)
}
// get connection string and driver name
cs := runOpts.String(toDbConnStrArgKey)
dn := runOpts.String(toDbDriverArgKey)
if dn == "" && runOpts.IsExist(dbDriverArgKey) {
dn = runOpts.String(dbDriverArgKey)
}
cs, dn = db.IfEmptyMakeDefault(modelName, cs, dn)
// open destination database and check is it valid
dstDb, _, err := db.Open(cs, dn, true)
if err != nil {
return err
}
defer dstDb.Close()
if err := db.CheckOpenmppSchemaVersion(dstDb); err != nil {
return err
}
// get model metadata
modelDef, err := db.GetModel(dstDb, modelName, modelDigest)
if err != nil {
return err
}
// get full list of languages
langDef, err := db.GetLanguages(dstDb)
if err != nil {
return err
}
// read from metadata json and csv files and update target database
encName := runOpts.String(encodingArgKey)
dstSetName := runOpts.String(setNewNameArgKey)
dstId, err := fromWorksetTextToDb(dstDb, modelDef, langDef, setName, dstSetName, metaPath, csvDir, encName)
if err != nil {
return err
}
if dstId <= 0 {
return errors.New("workset not found or empty: " + strconv.Itoa(setId) + " " + setName)
}
return nil
}
// fromWorksetTextListToDb read all worksets parameters from csv and json files,
// convert it to db cells and insert into database
// update set id's and base run id's with actual id in database
func fromWorksetTextListToDb(
dbConn *sql.DB, modelDef *db.ModelMeta, langDef *db.LangMeta, inpDir string, encodingName string,
) error {
// get list of workset json files
fl, err := filepath.Glob(inpDir + "/" + modelDef.Model.Name + ".set.*.json")
if err != nil {
return err
}
if len(fl) <= 0 {
return nil // no worksets
}
// for each file:
// read workset metadata, update workset in target database
// read csv files from workset csv subdir and update parameter values
for k := range fl {
// check if workset subdir exist
d, f := filepath.Split(fl[k])
csvDir := strings.TrimSuffix(strings.TrimPrefix(f, modelDef.Model.Name+"."), ".json")
if len(csvDir) <= 4 { // expected csv directory: set.4.q or set.q
csvDir = ""
} else {
csvDir = filepath.Join(d, csvDir)
if _, err := os.Stat(csvDir); err != nil {
csvDir = ""
}
}
// update or insert workset metadata and parameters from csv if csv directory exist
_, err := fromWorksetTextToDb(dbConn, modelDef, langDef, "", "", fl[k], csvDir, encodingName)
if err != nil {
return err
}
}
return nil
}
// fromWorksetTextToDb read workset metadata from json file,
// read all parameters from csv files, convert it to db cells and insert into database
// update set id's and base run id's with actual id in destination database
// it return source workset id (set id from metadata json file) and destination set id
func fromWorksetTextToDb(
dbConn *sql.DB, modelDef *db.ModelMeta, langDef *db.LangMeta, srcSetName string, dstSetName string, metaPath string, csvDir string, encodingName string,
) (int, error) {
// if no metadata file and no csv directory then exit: nothing to do
if metaPath == "" && csvDir == "" {
return 0, nil // no workset
}
// get workset metadata:
// model name and set name must be specified as parameter or inside of metadata json
var pub db.WorksetPub
if metaPath == "" && csvDir != "" { // no metadata json file, only csv directory
pub.Name = srcSetName
pub.ModelName = modelDef.Model.Name
}
if metaPath != "" { // read metadata json file
isExist, err := helper.FromJsonFile(metaPath, &pub)
if err != nil {
return 0, err
}
if !isExist { // metadata from json is empty
if csvDir == "" { // if metadata json empty and no csv directory then exit: no data
return 0, nil
}
// metadata empty but there is csv directory: use expected model name and set name
pub.Name = srcSetName
pub.ModelName = modelDef.Model.Name
}
}
if pub.Name == "" {
return 0, errors.New("workset name is empty and metadata json file not found or empty")
}
srcSetName = pub.Name
// if only csv directory specified:
// make list of parameters based on csv file names
// assume only one parameter sub-value in csv file
if metaPath == "" && csvDir != "" {
fl, err := filepath.Glob(csvDir + "/*.csv")
if err != nil {
return 0, err
}
pub.Param = make([]db.ParamRunSetPub, len(fl))
for j := range fl {
fn := filepath.Base(fl[j])
fn = fn[:len(fn)-4] // remove .csv extension
pub.Param[j].Name = fn
pub.Param[j].SubCount = 1 // only one sub-value
}
}
// save workset metadata as "read-write" and after importing all parameters set it as "readonly"
// save workset metadata parameters list, make it empty and use add parameters to update metadata and values from csv
isReadonly := pub.IsReadonly
pub.IsReadonly = false
paramLst := append([]db.ParamRunSetPub{}, pub.Param...)
pub.Param = []db.ParamRunSetPub{}
// rename destination workset
if dstSetName != "" {
pub.Name = dstSetName
}
// destination: convert from "public" format into destination db rows
// display warning if base run not found in destination database
ws, err := pub.FromPublic(dbConn, modelDef)
if err != nil {
return 0, err
}
if ws.Set.BaseRunId <= 0 && pub.BaseRunDigest != "" {
omppLog.Log("Warning: workset ", ws.Set.Name, ", base run not found by digest ", pub.BaseRunDigest)
}
// if destination workset exists then make it read-write and delete all existing parameters from workset
wsRow, err := db.GetWorksetByName(dbConn, modelDef.Model.ModelId, ws.Set.Name)
if err != nil {
return 0, err
}
if wsRow != nil {
err = db.UpdateWorksetReadonly(dbConn, wsRow.SetId, false) // make destination workset read-write
if err != nil {
return 0, errors.New("failed to clear workset read-only status: " + strconv.Itoa(wsRow.SetId) + " " + wsRow.Name + " " + err.Error())
}
err = db.DeleteWorksetAllParameters(dbConn, wsRow.SetId) // delete all parameters from workset
if err != nil {
return 0, errors.New("failed to delete workset " + strconv.Itoa(wsRow.SetId) + " " + wsRow.Name + " " + err.Error())
}
}
// create empty workset metadata or update existing workset metadata
err = ws.UpdateWorkset(dbConn, modelDef, true, langDef)
if err != nil {
return 0, err
}
dstId := ws.Set.SetId // actual set id from destination database
// read all workset parameters and copy into destination database
omppLog.Log("Workset ", srcSetName, " into: ", dstId, " "+ws.Set.Name)
// read all workset parameters from csv files
for j := range paramLst {
// read parameter values from csv file
var cell db.CellParam
cLst, err := fromCsvFile(csvDir, modelDef, paramLst[j].Name, paramLst[j].SubCount, &cell, encodingName)
if err != nil {
return 0, err
}
if cLst == nil || cLst.Len() <= 0 {
return 0, errors.New("workset: " + ws.Set.Name + " parameter empty: " + paramLst[j].Name)
}
// insert or update parameter values in workset
_, err = ws.UpdateWorksetParameter(dbConn, modelDef, true, ¶mLst[j], cLst, langDef)
if err != nil {
return 0, err
}
}
// update workset readonly status with actual value
err = db.UpdateWorksetReadonly(dbConn, dstId, isReadonly)
if err != nil {
return 0, err
}
return dstId, nil
}