Move repo store from in memory to redis

.travis.yml

@@ -10,6 +10,7 @@ cache:
     - src/ui/node_modules
 services:
   - docker
+  - redis-server
 # UI dependencies
 before_install:
   - export CHROME_BIN=chromium-browser

docker-compose.yml

@@ -13,7 +13,7 @@ services:
     tty: true
     build: ./dev_env/api
     ports:
-      - 8080:8080
+    - 8080:8080
     volumes:
       - ./src/api:/go/src/github.com/kubernetes-helm/monocular/src/api
       # Config example file
@@ -22,3 +22,14 @@ services:
       - $HOME/.kube/:/root/.kube
     environment:
       - ENVIRONMENT=development
+  redis:
+    image: bitnami/redis:4.0
+    environment:
+      - ALLOW_EMPTY_PASSWORD=yes
+    volumes:
+      - 'redis_data:/bitnami/redis'
+    ports:
+      - 6379:6379
+volumes:
+  redis_data:
+    driver: local

docs/config.example.yaml

@@ -29,3 +29,7 @@ tillerPortForward: true
 
 # Configure cache refresh interval in sec
 cacheRefreshInterval: 3600
+
+# Configure Redis server
+redis:
+  host: redis:6379

src/api/Makefile

@@ -8,7 +8,7 @@ REPO_PATH := github.com/kubernetes-helm/${SHORT_NAME}
 DEV_ENV_IMAGE := quay.io/deis/go-dev:v0.22.0
 SWAGGER_IMAGE := quay.io/goswagger/swagger:0.6.0
 DEV_ENV_WORK_DIR := /go/src/${REPO_PATH}/src/api
-DEV_ENV_PREFIX := docker run --rm -e GO15VENDOREXPERIMENT=1 -v ${CURDIR}:${DEV_ENV_WORK_DIR} -w ${DEV_ENV_WORK_DIR}
+DEV_ENV_PREFIX := docker run --rm -e GO15VENDOREXPERIMENT=1 -v ${CURDIR}:${DEV_ENV_WORK_DIR} -w ${DEV_ENV_WORK_DIR} --net=host
 DEV_ENV_CMD := ${DEV_ENV_PREFIX} ${DEV_ENV_IMAGE}
 SWAGGER_CMD := docker run --rm -e GOPATH=/go -v ${CURDIR}:${DEV_ENV_WORK_DIR} -w ${DEV_ENV_WORK_DIR} ${SWAGGER_IMAGE}
 SWAGGER_GEN_DIR := swagger

src/api/chartpackagesort/chartpackagesort_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestSortedByName(t *testing.T) {
-	chartsImplementation := mocks.NewMockCharts()
+	chartsImplementation := mocks.NewMockCharts(mocks.MockedMethods{})
 	charts, err := chartsImplementation.All()
 	assert.NoErr(t, err)
 	// Randomize slice before sorting
@@ -28,7 +28,7 @@ func TestSortedByName(t *testing.T) {
 }
 
 func TestSortedBySemver(t *testing.T) {
-	chartsImplementation := mocks.NewMockCharts()
+	chartsImplementation := mocks.NewMockCharts(mocks.MockedMethods{})
 	charts, err := chartsImplementation.All()
 	chart := charts[0]
 	assert.NoErr(t, err)
@@ -51,7 +51,7 @@ func TestSortedBySemver(t *testing.T) {
 
 // If it is not a valid semver, it still sorts
 func TestSortedBySemverWrongVersion(t *testing.T) {
-	chartsImplementation := mocks.NewMockCharts()
+	chartsImplementation := mocks.NewMockCharts(mocks.MockedMethods{})
 	charts, err := chartsImplementation.All()
 	assert.NoErr(t, err)
 	// Bogus versions

src/api/config/cache.go

@@ -0,0 +1,45 @@
+package config
+
+import (
+	"sync"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/albrow/zoom"
+)
+
+const defaultHost = "localhost:6379"
+
+var (
+	pool *zoom.Pool
+	once sync.Once
+)
+
+type redisConfig struct {
+	Host string
+}
+
+// GetRedisPool returns a pool of Zoom connections
+func GetRedisPool() *zoom.Pool {
+	once.Do(func() {
+		pool = newRedisPool()
+	})
+	return pool
+}
+
+func newRedisPool() *zoom.Pool {
+	config := getRedisConf()
+	return zoom.NewPool(config.Host)
+}
+
+func getRedisConf() redisConfig {
+	config, err := GetConfig()
+	if err != nil {
+		log.Fatalf("unable to read config")
+	}
+	redis := config.Redis
+	// Set default Redis host
+	if redis.Host == "" {
+		redis.Host = defaultHost
+	}
+	return redis
+}

src/api/config/cache_test.go

@@ -0,0 +1,38 @@
+package config
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/arschles/assert"
+)
+
+func TestGetRedisPool(t *testing.T) {
+	currentConfig = Configuration{}
+	pool := GetRedisPool()
+	assert.NotNil(t, pool, "Redis Pool")
+}
+
+func Test_getRedisConf(t *testing.T) {
+	tests := []struct {
+		name           string
+		configFileName string
+		expectedHost   string
+	}{
+		{"No Redis config", "noredis_config.yaml", defaultHost},
+		{"Blank Redis config", "emptyredis_config.yaml", defaultHost},
+		{"Custom Redis config", "config.yaml", "myredis:1234"},
+	}
+	configFileOrig := configFile
+	defer func() { configFile = configFileOrig }()
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			currentConfig = Configuration{}
+			configFile = func() string {
+				return filepath.Join("./testdata", tt.configFileName)
+			}
+			conf := getRedisConf()
+			assert.Equal(t, conf.Host, tt.expectedHost, tt.name)
+		})
+	}
+}

src/api/config/config.go

@@ -24,6 +24,7 @@ type Configuration struct {
 	ReleasesEnabled      bool  `yaml:"releasesEnabled"`
 	TillerPortForward    bool  `yaml:"tillerPortForward"`
 	CacheRefreshInterval int64 `yaml:"cacheRefreshInterval"`
+	Redis                redisConfig
 	Initialized          bool
 }
 

src/api/config/config_test.go

@@ -9,6 +9,7 @@ import (
 )
 
 func TestGetConfig(t *testing.T) {
+	currentConfig = Configuration{}
 	configFileOrig := configFile
 	defer func() { configFile = configFileOrig }()
 	configFile = func() string {
@@ -47,10 +48,10 @@ func TestGetConfigFromFile(t *testing.T) {
 	if len(config.Repos) == 0 {
 		t.Error("Repositories not present")
 	}
-	assert.Equal(t, config.Repos[0].Name, "repoName", "First repo")
-	assert.Equal(t, config.Repos[1].Name, "repoName2", "Second repo")
-	assert.Equal(t, config.Repos[0].URL, "http://myrepobucket", "Repo URL")
-	assert.Equal(t, config.Repos[1].URL, "http://myrepobucket2", "Repo URL")
+	assert.Equal(t, *config.Repos[0].Name, "repoName", "First repo")
+	assert.Equal(t, *config.Repos[1].Name, "repoName2", "Second repo")
+	assert.Equal(t, *config.Repos[0].URL, "http://myrepobucket", "Repo URL")
+	assert.Equal(t, *config.Repos[1].URL, "http://myrepobucket2", "Repo URL")
 	assert.Equal(t, config.Repos[0].Source, "http://github.com/my-repo", "Repo Source")
 	assert.Equal(t, config.Repos[1].Source, "", "Repo Source")
 

src/api/config/repos/repos.go

@@ -5,33 +5,28 @@ import (
 	"os"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/kubernetes-helm/monocular/src/api/data/pointerto"
+	"github.com/kubernetes-helm/monocular/src/api/swagger/models"
 
 	yaml "gopkg.in/yaml.v2"
 )
 
-// Repos is an array of Repo
-type Repos []Repo
+// Repos is an array of models.Repo
+type Repos []models.Repo
 
 type reposYAML struct {
 	Repos Repos
 }
 
-// Repo is a map name => URL
-type Repo struct {
-	Name   string
-	URL    string
-	Source string
-}
-
 var official = Repos{
-	Repo{
-		Name:   "stable",
-		URL:    "https://kubernetes-charts.storage.googleapis.com",
+	{
+		Name:   pointerto.String("stable"),
+		URL:    pointerto.String("https://kubernetes-charts.storage.googleapis.com"),
 		Source: "https://github.com/kubernetes/charts/tree/master/stable",
 	},
-	Repo{
-		Name:   "incubator",
-		URL:    "https://kubernetes-charts-incubator.storage.googleapis.com",
+	{
+		Name:   pointerto.String("incubator"),
+		URL:    pointerto.String("https://kubernetes-charts-incubator.storage.googleapis.com"),
 		Source: "https://github.com/kubernetes/charts/tree/master/incubator",
 	},
 }

src/api/config/repos/repos_test.go

@@ -5,19 +5,21 @@ import (
 	"testing"
 
 	"github.com/arschles/assert"
+	"github.com/kubernetes-helm/monocular/src/api/data/pointerto"
+	"github.com/kubernetes-helm/monocular/src/api/swagger/models"
 )
 
 var configFileOk = filepath.Join("..", "testdata", "config.yaml")
 var configFileNotOk = filepath.Join("..", "testdata", "bogus_config.yaml")
 var configFileNoRepos = filepath.Join("..", "testdata", "norepos_config.yaml")
 
 func TestOfficial(t *testing.T) {
-	offRepo := []Repo{
+	offRepo := []models.Repo{
 		{
-			Name: "stable",
+			Name: pointerto.String("stable"),
 		},
 		{
-			Name: "incubator",
+			Name: pointerto.String("incubator"),
 		},
 	}
 	for i, repo := range official {
@@ -41,15 +43,15 @@ func TestEnabledFileWithoutRepos(t *testing.T) {
 func TestEnabledReposInFile(t *testing.T) {
 	repos, err := Enabled(configFileOk)
 	assert.NoErr(t, err)
-	offRepo := []Repo{
+	offRepo := []models.Repo{
 		{
-			Name:   "repoName",
-			URL:    "http://myrepobucket",
+			Name:   pointerto.String("repoName"),
+			URL:    pointerto.String("http://myrepobucket"),
 			Source: "http://github.com/my-repo",
 		},
 		{
-			Name: "repoName2",
-			URL:  "http://myrepobucket2",
+			Name: pointerto.String("repoName2"),
+			URL:  pointerto.String("http://myrepobucket2"),
 		},
 	}
 

src/api/config/testdata/config.yaml

@@ -12,3 +12,5 @@ cors:
     - "x-xsrf-token"
 releasesEnabled: true
 cacheRefreshInterval: 3600
+redis:
+  host: myredis:1234

src/api/config/testdata/emptyredis_config.yaml

@@ -0,0 +1,2 @@
+redis:
+  host: ""

src/api/config/testdata/noredis_config.yaml

@@ -0,0 +1 @@
+otheroption: foo

src/api/data/cache/auto_refresher_test.go

@@ -5,12 +5,15 @@ import (
 	"time"
 
 	"github.com/arschles/assert"
-	"github.com/kubernetes-helm/monocular/src/api/config/repos"
+	"github.com/kubernetes-helm/monocular/src/api/data/pointerto"
+	"github.com/kubernetes-helm/monocular/src/api/swagger/models"
 )
 
 func TestNewRefreshData(t *testing.T) {
-	repos := repos.Repos{}
-	chartsImplementation := NewCachedCharts(repos)
+	setupTestRepoCache(nil)
+	defer teardownTestRepoCache()
+
+	chartsImplementation := NewCachedCharts()
 	// Setup background index refreshes
 	freshness := time.Duration(3600) * time.Second
 	job := NewRefreshChartsData(chartsImplementation, freshness, "test-run", false)
@@ -22,13 +25,16 @@ func TestNewRefreshData(t *testing.T) {
 }
 
 func TestNewRefreshDataError(t *testing.T) {
-	repos := repos.Repos{
-		repos.Repo{
-			Name: "waps",
-			URL:  "./localhost",
+	repos := []models.Repo{
+		{
+			Name: pointerto.String("waps"),
+			URL:  pointerto.String("./localhost"),
 		},
 	}
-	chartsImplementation := NewCachedCharts(repos)
+	setupTestRepoCache(&repos)
+	defer teardownTestRepoCache()
+
+	chartsImplementation := NewCachedCharts()
 	freshness := time.Duration(3600) * time.Second
 	job := NewRefreshChartsData(chartsImplementation, freshness, "test-run", true)
 	assert.Equal(t, job.FirstRun(), true, "First run")