Skip to content

Commit

Permalink
fix(db): no wal mode with SQLite3 backend to avoid database is locked
Browse files Browse the repository at this point in the history
  • Loading branch information
@kotakanbe
kotakanbe committed Jul 13, 2018
1 parent 3d59a58 commit c60a1b8
Show file tree
Hide file tree
Showing 11 changed files with 48 additions and 54 deletions.
8 changes: 4 additions & 4 deletions commands/fetch-alpine.go
View file
Expand Up @@ -96,7 +96,7 @@ func (p *FetchAlpineCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
}

if len(f.Args()) == 0 {
log15.Error("Specify versions to fetch.")
log15.Error("Specify versions to fetch")
log15.Error(p.Usage())
return subcommands.ExitUsageError
}
Expand All @@ -114,17 +114,17 @@ func (p *FetchAlpineCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
driver, locked, err := db.NewDB(c.Alpine, c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL)
if err != nil {
if locked {
log15.Error("Failed to Open DB. Close DB connection before fetching", "err", err)
log15.Error("Failed to open DB. Close DB connection before fetching", "err", err)
return subcommands.ExitFailure
}
log15.Error("%s", err)
log15.Error("Failed to open DB", "err", err)
return subcommands.ExitFailure
}
defer driver.CloseDB()

results, err := fetcher.FetchAlpineFiles(vers)
if err != nil {
log15.Error("Failed to fetch files.", "err", err)
log15.Error("Failed to fetch files", "err", err)
return subcommands.ExitFailure
}

Expand Down
10 changes: 5 additions & 5 deletions commands/fetch-amazon.go
View file
Expand Up @@ -95,23 +95,23 @@ func (p *FetchAmazonCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
driver, locked, err := db.NewDB(c.Amazon, c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL)
if err != nil {
if locked {
log15.Error("Failed to Open DB. Close DB connection before fetching", "err", err)
log15.Error("Failed to open DB. Close DB connection before fetching", "err", err)
return subcommands.ExitFailure
}
log15.Error("%s", err)
log15.Error("Failed to open DB", "err", err)
return subcommands.ExitFailure
}
defer driver.CloseDB()

result, err := fetcher.FetchAmazonFile()
if err != nil {
log15.Error("Failed to fetch files.", "err", err)
log15.Error("Failed to fetch files", "err", err)
return subcommands.ExitFailure
}

amazonRSS := models.AmazonRSS{}
if err = xml.Unmarshal(result.Body, &amazonRSS); err != nil {
log15.Error("Failed to unmarshal.", "url", result.URL, "err", err)
log15.Error("Failed to unmarshal", "url", result.URL, "err", err)
return subcommands.ExitUsageError
}
defs := models.ConvertAmazonToModel(&amazonRSS)
Expand All @@ -125,7 +125,7 @@ func (p *FetchAmazonCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf

log15.Info(fmt.Sprintf("%d CVEs", len(defs)))
if err := driver.InsertOval(&root, models.FetchMeta{}); err != nil {
log15.Error("Failed to insert meta.", "err", err)
log15.Error("Failed to insert meta", "err", err)
return subcommands.ExitFailure
}

Expand Down
14 changes: 7 additions & 7 deletions commands/fetch-debian.go
View file
Expand Up @@ -103,10 +103,10 @@ func (p *FetchDebianCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
driver, locked, err := db.NewDB(c.Debian, c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL)
if err != nil {
if locked {
log15.Error("Failed to Open DB. Close DB connection before fetching", "err", err)
log15.Error("Failed to open DB. Close DB connection before fetching", "err", err)
return subcommands.ExitFailure
}
log15.Error("%s", err)
log15.Error("Failed to open DB", "err", err)
return subcommands.ExitFailure
}
defer driver.CloseDB()
Expand All @@ -123,14 +123,14 @@ func (p *FetchDebianCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf

results, err := fetcher.FetchDebianFiles(vers)
if err != nil {
log15.Error("Failed to fetch files.", "err", err)
log15.Error("Failed to fetch files", "err", err)
return subcommands.ExitFailure
}

for _, r := range results {
ovalroot := oval.Root{}
if err = xml.Unmarshal(r.Body, &ovalroot); err != nil {
log15.Error("Failed to unmarshal.", "url", r.URL, "err", err)
log15.Error("Failed to unmarshal", "url", r.URL, "err", err)
return subcommands.ExitUsageError
}
log15.Info("Fetched", "URL", r.URL, "OVAL definitions", len(ovalroot.Definitions.Definitions))
Expand All @@ -141,7 +141,7 @@ func (p *FetchDebianCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
var t time.Time
t, err = time.Parse(timeformat, strings.Split(ovalroot.Generator.Timestamp, ".")[0])
if err != nil {
log15.Error("Failed to parse timestamp.", "url", r.URL, "err", err)
log15.Error("Failed to parse timestamp", "url", r.URL, "err", err)
return subcommands.ExitUsageError
}

Expand All @@ -159,11 +159,11 @@ func (p *FetchDebianCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
}

if err := driver.InsertOval(&root, fmeta); err != nil {
log15.Error("Failed to insert oval.", "err", err)
log15.Error("Failed to insert oval", "err", err)
return subcommands.ExitFailure
}
if err := driver.InsertFetchMeta(fmeta); err != nil {
log15.Error("Failed to insert meta.", "err", err)
log15.Error("Failed to insert meta", "err", err)
return subcommands.ExitFailure
}
}
Expand Down
12 changes: 6 additions & 6 deletions commands/fetch-oracle.go
View file
Expand Up @@ -97,24 +97,24 @@ func (p *FetchOracleCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
driver, locked, err := db.NewDB(c.Oracle, c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL)
if err != nil {
if locked {
log15.Error("Failed to Open DB. Close DB connection before fetching", "err", err)
log15.Error("Failed to open DB. Close DB connection before fetching", "err", err)
return subcommands.ExitFailure
}
log15.Error("%s", err)
log15.Error("Failed to open DB", "err", err)
return subcommands.ExitFailure
}
defer driver.CloseDB()

results, err := fetcher.FetchOracleFiles()
if err != nil {
log15.Error("Failed to fetch files.", "err", err)
log15.Error("Failed to fetch files", "err", err)
return subcommands.ExitFailure
}

for _, r := range results {
ovalroot := oval.Root{}
if err = xml.Unmarshal(r.Body, &ovalroot); err != nil {
log15.Error("Failed to unmarshal.", "url", r.URL, "err", err)
log15.Error("Failed to unmarshal", "url", r.URL, "err", err)
return subcommands.ExitUsageError
}
log15.Info("Fetched", "URL", r.URL, "OVAL definitions", len(ovalroot.Definitions.Definitions))
Expand All @@ -136,12 +136,12 @@ func (p *FetchOracleCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
for _, root := range roots {
root.Timestamp = time.Now()
if err := driver.InsertOval(&root, fmeta); err != nil {
log15.Error("Failed to insert oval.", "err", err)
log15.Error("Failed to insert oval", "err", err)
return subcommands.ExitFailure
}
}
if err := driver.InsertFetchMeta(fmeta); err != nil {
log15.Error("Failed to insert meta.", "err", err)
log15.Error("Failed to insert meta", "err", err)
return subcommands.ExitFailure
}
}
Expand Down
12 changes: 6 additions & 6 deletions commands/fetch-redhat.go
View file
Expand Up @@ -106,10 +106,10 @@ func (p *FetchRedHatCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
driver, locked, err := db.NewDB(c.RedHat, c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL)
if err != nil {
if locked {
log15.Error("Failed to Open DB. Close DB connection before fetching", "err", err)
log15.Error("Failed to open DB. Close DB connection before fetching", "err", err)
return subcommands.ExitFailure
}
log15.Error("%s", err)
log15.Error("Failed to open DB", "err", err)
return subcommands.ExitFailure
}

Expand All @@ -130,14 +130,14 @@ func (p *FetchRedHatCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf

results, err := fetcher.FetchRedHatFiles(vers)
if err != nil {
log15.Error("Failed to fetch files.", "err", err)
log15.Error("Failed to fetch files", "err", err)
return subcommands.ExitFailure
}

for _, r := range results {
ovalroot := oval.Root{}
if err = xml.Unmarshal(r.Body, &ovalroot); err != nil {
log15.Error("Failed to unmarshal.", "url", r.URL, "err", err)
log15.Error("Failed to unmarshal", "url", r.URL, "err", err)
return subcommands.ExitUsageError
}
log15.Info("Fetched", "URL", r.URL, "OVAL definitions", len(ovalroot.Definitions.Definitions))
Expand All @@ -163,11 +163,11 @@ func (p *FetchRedHatCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
}

if err := driver.InsertOval(&root, fmeta); err != nil {
log15.Error("Failed to insert oval.", "err", err)
log15.Error("Failed to insert oval", "err", err)
return subcommands.ExitFailure
}
if err := driver.InsertFetchMeta(fmeta); err != nil {
log15.Error("Failed to insert meta.", "err", err)
log15.Error("Failed to insert meta", "err", err)
return subcommands.ExitFailure
}
}
Expand Down
16 changes: 8 additions & 8 deletions commands/fetch-suse.go
View file
Expand Up @@ -142,24 +142,24 @@ func (p *FetchSUSECmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfac
driver, locked, err := db.NewDB(suseType, c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL)
if err != nil {
if locked {
log15.Error("Failed to Open DB. Close DB connection before fetching", "err", err)
log15.Error("Failed to open DB. Close DB connection before fetching", "err", err)
return subcommands.ExitFailure
}
log15.Error("%s", err)
log15.Error("Failed to open DB", "err", err)
return subcommands.ExitFailure
}

var results []fetcher.FetchResult
if p.OVALPath == "" {
results, err = fetcher.FetchSUSEFiles(suseType, vers)
if err != nil {
log15.Error("Failed to fetch files.", "err", err)
log15.Error("Failed to fetch files", "err", err)
return subcommands.ExitFailure
}
} else {
dat, err := ioutil.ReadFile(p.OVALPath)
if err != nil {
log15.Error("Failed to read file.", "err", err)
log15.Error("Failed to read file", "err", err)
return subcommands.ExitFailure
}
results = []fetcher.FetchResult{{
Expand All @@ -171,15 +171,15 @@ func (p *FetchSUSECmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfac
for _, r := range results {
ovalroot := oval.Root{}
if err = xml.Unmarshal(r.Body, &ovalroot); err != nil {
log15.Error("Failed to unmarshal.", "url", r.URL, "err", err)
log15.Error("Failed to unmarshal", "url", r.URL, "err", err)
return subcommands.ExitUsageError
}
log15.Info("Fetched", "URL", r.URL, "OVAL definitions", len(ovalroot.Definitions.Definitions))

var timeformat = "2006-01-02T15:04:05"
t, err := time.Parse(timeformat, ovalroot.Generator.Timestamp)
if err != nil {
log15.Error("Failed to parse time.", "err", err)
log15.Error("Failed to parse time", "err", err)
return subcommands.ExitFailure
}
ss := strings.Split(r.URL, "/")
Expand All @@ -192,13 +192,13 @@ func (p *FetchSUSECmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interfac
for _, root := range roots {
root.Timestamp = time.Now()
if err := driver.InsertOval(&root, fmeta); err != nil {
log15.Error("Failed to insert oval.", "err", err)
log15.Error("Failed to insert oval", "err", err)
return subcommands.ExitFailure
}
}

if err := driver.InsertFetchMeta(fmeta); err != nil {
log15.Error("Failed to insert meta.", "err", err)
log15.Error("Failed to insert meta", "err", err)
return subcommands.ExitFailure
}
}
Expand Down
12 changes: 6 additions & 6 deletions commands/fetch-ubuntu.go
View file
Expand Up @@ -102,10 +102,10 @@ func (p *FetchUbuntuCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
driver, locked, err := db.NewDB(c.Ubuntu, c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL)
if err != nil {
if locked {
log15.Error("Failed to Open DB. Close DB connection before fetching", "err", err)
log15.Error("Failed to open DB. Close DB connection before fetching", "err", err)
return subcommands.ExitFailure
}
log15.Error("%s", err)
log15.Error("Failed to open DB", "err", err)
return subcommands.ExitFailure
}

Expand All @@ -121,14 +121,14 @@ func (p *FetchUbuntuCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf

results, err := fetcher.FetchUbuntuFiles(vers)
if err != nil {
log15.Error("Failed to fetch files.", "err", err)
log15.Error("Failed to fetch files", "err", err)
return subcommands.ExitFailure
}

for _, r := range results {
ovalroot := oval.Root{}
if err = xml.Unmarshal(r.Body, &ovalroot); err != nil {
log15.Error("Failed to unmarshal.", "url", r.URL, "err", err)
log15.Error("Failed to unmarshal", "url", r.URL, "err", err)
return subcommands.ExitUsageError
}
log15.Info("Fetched", "URL", r.URL, "OVAL definitions", len(ovalroot.Definitions.Definitions))
Expand All @@ -155,11 +155,11 @@ func (p *FetchUbuntuCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interf
}

if err := driver.InsertOval(&root, fmeta); err != nil {
log15.Error("Failed to insert oval.", "err", err)
log15.Error("Failed to insert OVAL", "err", err)
return subcommands.ExitFailure
}
if err := driver.InsertFetchMeta(fmeta); err != nil {
log15.Error("Failed to insert meta.", "err", err)
log15.Error("Failed to insert meta", "err", err)
return subcommands.ExitFailure
}
}
Expand Down
4 changes: 2 additions & 2 deletions commands/select.go
View file
Expand Up @@ -98,10 +98,10 @@ func (p *SelectCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
driver, locked, err := db.NewDB(f.Args()[0], c.Conf.DBType, c.Conf.DBPath, c.Conf.DebugSQL)
if err != nil {
if locked {
log15.Error("Failed to Open DB. Close DB connection before select: %s", err)
log15.Error("Failed to open DB. Close DB connection before select", "err", err)
return subcommands.ExitFailure
}
log15.Error("%s", err)
log15.Error("Failed to open DB", "err", err)
return subcommands.ExitFailure
}

Expand Down
2 changes: 1 addition & 1 deletion commands/server.go
View file
Expand Up @@ -91,7 +91,7 @@ func (p *ServerCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}

log15.Info("Starting HTTP Server...")
if err := server.Start(p.logDir); err != nil {
log15.Error("Failed to start server.", "err", err)
log15.Error("Failed to start server", "err", err)
return subcommands.ExitFailure
}
return subcommands.ExitSuccess
Expand Down
6 changes: 0 additions & 6 deletions db/rdb/rdb.go
View file
Expand Up @@ -112,12 +112,6 @@ func (d *Driver) OpenDB(dbType, dbPath string, debugSQL bool) (locked bool, err
return false, fmt.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %s", dbType, dbPath, err)
}
d.conn.LogMode(debugSQL)

if dbType == DialectSqlite3 {
if err := d.conn.Exec("PRAGMA journal_mode=WAL;").Error; err != nil {
return false, err
}
}
return false, nil
}

Expand Down
6 changes: 3 additions & 3 deletions fetcher/util.go
View file
Expand Up @@ -84,7 +84,7 @@ func fetchFeedFileConcurrently(reqs []fetchRequest) (results []FetchResult, err
return results, fmt.Errorf("Timeout Fetching")
}
}
log15.Info("Finished to fetch OVAL definitions.")
log15.Info("Finished to fetch OVAL definitions")
if 0 < len(errs) {
return results, fmt.Errorf("%s", errs)
}
Expand Down Expand Up @@ -151,12 +151,12 @@ func getFileSize(req fetchRequest) int {
defer resp.Body.Close()

if resp.Header.Get("Accept-Ranges") != "bytes" {
log15.Warn("Not supported range access.")
log15.Warn("Not supported range access")
}

// the value -1 indicates that the length is unknown.
if resp.ContentLength <= 0 {
log15.Info("Failed to get content length.")
log15.Info("Failed to get content length")
return 0
}
return int(resp.ContentLength)
Expand Down

0 comments on commit c60a1b8

Please sign in to comment.