Skip to content

Commit

Permalink
begin boltdb => badger migration
Browse files Browse the repository at this point in the history
  • Loading branch information
@shawnps
shawnps committed Feb 10, 2020
1 parent df965ca commit 9ec2869
Show file tree
Hide file tree
Showing 107 changed files with 24,781 additions and 123 deletions.
106 changes: 66 additions & 40 deletions handlers/check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,17 @@ import (
"net/http"
"strings"

"github.com/boltdb/bolt"
"github.com/dgraph-io/badger"
"github.com/gojp/goreportcard/download"
)

const (
// DBPath is the relative (or absolute) path to the bolt database file
DBPath string = "goreportcard.db"

// RepoBucket is the bucket in which repos will be cached in the bolt DB
RepoBucket string = "repos"

// MetaBucket is the bucket containing the names of the projects with the
// top 100 high scores, and other meta information
MetaBucket string = "meta"

// RepoPrefix is the badger prefix for repos
RepoPrefix string = "repos-"
)

// CheckHandler handles the request for checking a repo
Expand Down Expand Up @@ -53,18 +50,35 @@ func CheckHandler(w http.ResponseWriter, r *http.Request) {
w.Write(b)
}

func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error {
func updateHighScores(txn *badger.Txn, resp checksResp, repo string) error {
// check if we need to update the high score list
if resp.Files < 100 {
// only repos with >= 100 files are considered for the high score list
return nil
}

var scoreBytes []byte
// start updating high score list
scoreBytes := mb.Get([]byte("scores"))
if scoreBytes == nil {
item, err := txn.Get([]byte("scores"))
if err != nil && err != badger.ErrKeyNotFound {
return err
}

if item == nil {
scoreBytes, _ = json.Marshal([]ScoreHeap{})
}

if item != nil {
err = item.Value(func(val []byte) error {
err = json.Unmarshal(val, &scoreBytes)
if err != nil {
return fmt.Errorf("could not unmarshal high scores: %v", err)
}

return nil
})
}

scores := &ScoreHeap{}
json.Unmarshal(scoreBytes, scores)

Expand All @@ -74,64 +88,81 @@ func updateHighScores(mb *bolt.Bucket, resp checksResp, repo string) error {
// we do not have 50 high scores yet
return nil
}

// if this repo is already in the list, remove the original entry:
for i := range *scores {
if strings.ToLower((*scores)[i].Repo) == strings.ToLower(repo) {
heap.Remove(scores, i)
break
}
}

// now we can safely push it onto the heap
heap.Push(scores, scoreItem{
Repo: repo,
Score: resp.Average * 100.0,
Files: resp.Files,
})

if len(*scores) > 50 {
// trim heap if it's grown to over 50
*scores = (*scores)[1:51]
}
scoreBytes, err := json.Marshal(&scores)

scoreBytes, err = json.Marshal(&scores)
if err != nil {
return err
}
return mb.Put([]byte("scores"), scoreBytes)

return txn.Set([]byte("scores"), scoreBytes)
}

func updateReposCount(mb *bolt.Bucket, repo string) (err error) {
func updateReposCount(txn *badger.Txn, repo string) error {
log.Printf("New repo %q, adding to repo count...", repo)
totalInt := 0
total := mb.Get([]byte("total_repos"))
if total != nil {
err = json.Unmarshal(total, &totalInt)
if err != nil {
return fmt.Errorf("could not unmarshal total repos count: %v", err)
}
item, err := txn.Get([]byte("total_repos"))
if err != nil && err != badger.ErrKeyNotFound {
return err
}

if item != nil {
err = item.Value(func(val []byte) error {
err = json.Unmarshal(val, &totalInt)
if err != nil {
return fmt.Errorf("could not unmarshal total repos count: %v", err)
}

return nil
})
}

totalInt++ // increase repo count
total, err = json.Marshal(totalInt)
total, err := json.Marshal(totalInt)
if err != nil {
return fmt.Errorf("could not marshal total repos count: %v", err)
}
mb.Put([]byte("total_repos"), total)

err = txn.Set([]byte("total_repos"), total)
if err != nil {
return err
}
log.Println("Repo count is now", totalInt)

return nil
}

type recentItem struct {
Repo string
}

func updateRecentlyViewed(mb *bolt.Bucket, repo string) error {
if mb == nil {
return fmt.Errorf("meta bucket not found")
}
b := mb.Get([]byte("recent"))
if b == nil {
b, _ = json.Marshal([]recentItem{})
func updateRecentlyViewed(txn *badger.Txn, repo string) error {
var recent []recentItem
item, err := txn.Get([]byte("recent"))
if item != nil {
item.Value(func(val []byte) error {
return json.Unmarshal(val, &recent)
})
}
recent := []recentItem{}
json.Unmarshal(b, &recent)

// add it to the slice, if it is not in there already
for i := range recent {
Expand All @@ -149,23 +180,18 @@ func updateRecentlyViewed(mb *bolt.Bucket, repo string) error {
if err != nil {
return err
}
return mb.Put([]byte("recent"), b)

return txn.Set([]byte("recent"), b)
}

//func updateMetadata(tx *bolt.Tx, resp checksResp, repo string, isNewRepo bool, oldScore *float64) error {
func updateMetadata(tx *bolt.Tx, resp checksResp, repo string, isNewRepo bool) error {
// fetch meta-bucket
mb := tx.Bucket([]byte(MetaBucket))
if mb == nil {
return fmt.Errorf("high score bucket not found")
}
func updateMetadata(txn *badger.Txn, resp checksResp, repo string, isNewRepo bool) error {
// update total repos count
if isNewRepo {
err := updateReposCount(mb, repo)
err := updateReposCount(txn, repo)
if err != nil {
return err
}
}

return updateHighScores(mb, resp, repo)
return updateHighScores(txn, resp, repo)
}
81 changes: 44 additions & 37 deletions handlers/checks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@ package handlers

import (
"encoding/json"
"errors"
"fmt"
"log"
"time"

"github.com/boltdb/bolt"
"github.com/dgraph-io/badger"
"github.com/dustin/go-humanize"
"github.com/gojp/goreportcard/check"
"github.com/gojp/goreportcard/download"
Expand All @@ -26,29 +25,34 @@ func dirName(repo string) string {
}

func getFromCache(repo string) (checksResp, error) {
// try and fetch from boltdb
db, err := bolt.Open(DBPath, 0600, &bolt.Options{Timeout: 3 * time.Second})
// try and fetch from badger
db, err := badger.Open(badger.DefaultOptions("/tmp/badger"))
if err != nil {
return checksResp{}, fmt.Errorf("failed to open bolt database during GET: %v", err)
log.Fatal(err)
}
defer db.Close()

resp := checksResp{}
err = db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(RepoBucket))
if b == nil {
return errors.New("No repo bucket")
err = db.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(RepoPrefix + repo))
if err != nil && err != badger.ErrKeyNotFound {
return err
}
cached := b.Get([]byte(repo))
if cached == nil {

if item == nil {
return notFoundError{repo}
}

err = json.Unmarshal(cached, &resp)
if err != nil {
return fmt.Errorf("failed to parse JSON for %q in cache", repo)
}
return nil
err = item.Value(func(val []byte) error {
err = json.Unmarshal(val, &resp)
if err != nil {
return fmt.Errorf("failed to parse JSON for %q in cache", repo)
}

return nil
})

return err
})

if err != nil {
Expand Down Expand Up @@ -120,23 +124,29 @@ func newChecksResp(repo string, forceRefresh bool) (checksResp, error) {
return checksResp{}, fmt.Errorf("could not marshal json: %v", err)
}

// write to boltdb
db, err := bolt.Open(DBPath, 0755, &bolt.Options{Timeout: 1 * time.Second})
// write to badger
db, err := badger.Open(badger.DefaultOptions("/tmp/badger"))
if err != nil {
return checksResp{}, fmt.Errorf("could not open bolt db: %v", err)
log.Fatal(err)
}
defer db.Close()

// is this a new repo? if so, increase the count in the high scores bucket later
isNewRepo := false
var oldRepoBytes []byte
err = db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte(RepoBucket))
if b == nil {
return fmt.Errorf("repo bucket not found")
err = db.View(func(txn *badger.Txn) error {
item, err := txn.Get([]byte(RepoPrefix + repo))
if err != nil {
return err
}
oldRepoBytes = b.Get([]byte(repo))
return nil

err = item.Value(func(val []byte) error {
oldRepoBytes = val

return nil
})

return err
})
if err != nil {
log.Println("ERROR getting repo from repo bucket:", err)
Expand All @@ -146,21 +156,16 @@ func newChecksResp(repo string, forceRefresh bool) (checksResp, error) {

// if this is a new repo, or the user force-refreshed, update the cache
if isNewRepo || forceRefresh {
err = db.Update(func(tx *bolt.Tx) error {
err = db.Update(func(txn *badger.Txn) error {
log.Printf("Saving repo %q to cache...", repo)

b := tx.Bucket([]byte(RepoBucket))
if b == nil {
return fmt.Errorf("repo bucket not found")
}

// save repo to cache
err = b.Put([]byte(repo), respBytes)
err = txn.Set([]byte(RepoPrefix+repo), respBytes)
if err != nil {
return err
}

return updateMetadata(tx, resp, repo, isNewRepo)
return updateMetadata(txn, resp, repo, isNewRepo)
})

if err != nil {
Expand All @@ -169,11 +174,13 @@ func newChecksResp(repo string, forceRefresh bool) (checksResp, error) {

}

db.Update(func(tx *bolt.Tx) error {
// fetch meta-bucket
mb := tx.Bucket([]byte(MetaBucket))
return updateRecentlyViewed(mb, repo)
err = db.Update(func(txn *badger.Txn) error {
return updateRecentlyViewed(txn, repo)
})

if err != nil {
log.Printf("ERROR: could not update recently viewed: %v", err)
}

return resp, nil
}
Loading

0 comments on commit 9ec2869

Please sign in to comment.