Merge branch 'master' into bryan/CA-test-reorg

.clabot

@@ -56,6 +56,7 @@
     "mbouzi",
     "AlexeyALeonov",
     "Qweder93",
-    "cpustejovsky"
+    "cpustejovsky",
+    "grafael"
 	]
 }

Dockerfile.jenkins

@@ -1,4 +1,4 @@
-FROM golang:1.13.3
+FROM golang:1.13.4
 
 # Postgres
 
@@ -53,7 +53,7 @@ RUN export JAXB=${ANDROID_HOME}/tools/bin/jaxb_lib/activation.jar:${ANDROID_HOME
 	&& sed -i '/^eval set -- $DEFAULT_JVM_OPTS.*/i CLASSPATH='$JAXB':$CLASSPATH' ${ANDROID_HOME}/tools/bin/avdmanager
 
 ENV PATH ${PATH}:${ANDROID_HOME}/tools:${ANDROID_HOME}/tools/bin:${ANDROID_HOME}/platform-tools
- 
+
 # accept all licenses
 RUN yes | sdkmanager  --licenses
 RUN touch /root/.android/repositories.cfg

Makefile

@@ -1,4 +1,4 @@
-GO_VERSION ?= 1.13.3
+GO_VERSION ?= 1.13.4
 GOOS ?= linux
 GOARCH ?= amd64
 GOPATH ?= $(shell go env GOPATH)
@@ -294,7 +294,7 @@ BINARIES      := $(foreach C,$(COMPONENTLIST),$(foreach O,$(OSARCHLIST),$C_$O))
 binaries: ${BINARIES} ## Build certificates, gateway, identity, inspector, linksharing, satellite, storagenode, uplink, and versioncontrol binaries (jenkins)
 
 .PHONY: sign-windows-installer
-sign-windows-installer: 
+sign-windows-installer:
 	storj-sign release/${TAG}/storagenode_windows_amd64.msi
 
 .PHONY: libuplink

cmd/satellite/main.go

@@ -222,7 +222,7 @@ func cmdMigrationRun(cmd *cobra.Command, args []string) (err error) {
 	log := zap.L()
 	db, err := satellitedb.New(log.Named("db migration"), runCfg.Database)
 	if err != nil {
-		return errs.New("Error createing new master database connection for satellitedb migration: %+v", err)
+		return errs.New("Error creating new master database connection for satellitedb migration: %+v", err)
 	}
 	defer func() {
 		err = errs.Combine(err, db.Close())

cmd/storj-sim/main.go

@@ -27,6 +27,9 @@ type Flags struct {
 	// Connection string for the postgres database to use for storj-sim processes
 	Postgres string
 	Redis    string
+
+	// Value of first redis db
+	RedisStartDB int
 }
 
 var printCommands bool
@@ -59,7 +62,8 @@ func main() {
 	rootCmd.PersistentFlags().BoolVarP(&flags.IsDev, "dev", "", false, "use configuration values tuned for development")
 
 	rootCmd.PersistentFlags().StringVarP(&flags.Postgres, "postgres", "", os.Getenv("STORJ_SIM_POSTGRES"), "connection string for postgres (defaults to STORJ_SIM_POSTGRES)")
-	rootCmd.PersistentFlags().StringVarP(&flags.Redis, "redis", "", os.Getenv("STORJ_SIM_REDIS"), "connection string for redis (defaults to STORJ_SIM_REDIS)")
+	rootCmd.PersistentFlags().StringVarP(&flags.Redis, "redis", "", os.Getenv("STORJ_SIM_REDIS"), "connection string for redis e.g. 127.0.0.1:6379 (defaults to STORJ_SIM_REDIS)")
+	rootCmd.PersistentFlags().IntVarP(&flags.RedisStartDB, "redis-startdb", "", 0, "value of first redis db (defaults to 0)")
 
 	networkCmd := &cobra.Command{
 		Use:   "network",

cmd/storj-sim/network.go

@@ -270,35 +270,35 @@ func newNetwork(flags *Flags) (*Processes, error) {
 
 	var satellites []*Process
 	for i := 0; i < flags.SatelliteCount; i++ {
-		process := processes.New(Info{
+		apiProcess := processes.New(Info{
 			Name:       fmt.Sprintf("satellite/%d", i),
 			Executable: "satellite",
 			Directory:  filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
 			Address:    net.JoinHostPort(host, port(satellitePeer, i, publicGRPC)),
 		})
-		satellites = append(satellites, process)
+		satellites = append(satellites, apiProcess)
 
 		consoleAuthToken := "secure_token"
 
 		redisAddress := flags.Redis
-		redisPortBase := i * 2
+		redisPortBase := flags.RedisStartDB + i*2
 		if redisAddress == "" {
 			redisAddress = redisServers[i].Address
 			redisPortBase = 0
-			process.WaitForStart(redisServers[i])
+			apiProcess.WaitForStart(redisServers[i])
 		}
 
-		process.Arguments = withCommon(process.Directory, Arguments{
+		apiProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
 			"setup": {
-				"--identity-dir", process.Directory,
+				"--identity-dir", apiProcess.Directory,
 				"--console.address", net.JoinHostPort(host, port(satellitePeer, i, publicHTTP)),
 				"--console.static-dir", filepath.Join(storjRoot, "web/satellite/"),
 				// TODO: remove console.auth-token after vanguard release
 				"--console.auth-token", consoleAuthToken,
 				"--marketing.base-url", "",
 				"--marketing.address", net.JoinHostPort(host, port(satellitePeer, i, privateHTTP)),
 				"--marketing.static-dir", filepath.Join(storjRoot, "web/marketing/"),
-				"--server.address", process.Address,
+				"--server.address", apiProcess.Address,
 				"--server.private-address", net.JoinHostPort(host, port(satellitePeer, i, privateGRPC)),
 
 				"--live-accounting.storage-backend", "redis://" + redisAddress + "?db=" + strconv.Itoa(redisPortBase),
@@ -317,12 +317,12 @@ func newNetwork(flags *Flags) (*Processes, error) {
 		})
 
 		if flags.Postgres != "" {
-			process.Arguments["setup"] = append(process.Arguments["setup"],
+			apiProcess.Arguments["setup"] = append(apiProcess.Arguments["setup"],
 				"--database", pgutil.ConnstrWithSchema(flags.Postgres, fmt.Sprintf("satellite/%d", i)),
 				"--metainfo.database-url", pgutil.ConnstrWithSchema(flags.Postgres, fmt.Sprintf("satellite/%d/meta", i)),
 			)
 		}
-		process.ExecBefore["run"] = func(process *Process) error {
+		apiProcess.ExecBefore["run"] = func(process *Process) error {
 			return readConfigString(&process.Address, process.Directory, "server.address")
 		}
 
@@ -331,7 +331,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
 			Executable: "satellite",
 			Directory:  filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
 		})
-		migrationProcess.Arguments = withCommon(process.Directory, Arguments{
+		migrationProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
 			"run": {
 				"migration",
 				"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugMigrationHTTP)),
@@ -344,7 +344,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
 			Directory:  filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
 			Address:    "",
 		})
-		coreProcess.Arguments = withCommon(process.Directory, Arguments{
+		coreProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
 			"run": {
 				"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugPeerHTTP)),
 			},
@@ -356,15 +356,15 @@ func newNetwork(flags *Flags) (*Processes, error) {
 			Executable: "satellite",
 			Directory:  filepath.Join(processes.Directory, "satellite", fmt.Sprint(i)),
 		})
-		repairProcess.Arguments = withCommon(process.Directory, Arguments{
+		repairProcess.Arguments = withCommon(apiProcess.Directory, Arguments{
 			"run": {
 				"repair",
 				"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugRepairerHTTP)),
 			},
 		})
 		repairProcess.WaitForExited(migrationProcess)
 
-		process.WaitForExited(migrationProcess)
+		apiProcess.WaitForExited(migrationProcess)
 	}
 
 	// Create gateways for each satellite

docs/blueprints/storage-node-downtime-tracking.md

@@ -213,7 +213,7 @@ Data Science could use this approach to more nicely calculate statistics however
     1. Implement [_estimating offline time_ part](#estimating-offline-time)<sup>1</sup>.
 
     <sup>1</sup> These subtasks can be done in parallel.
-1. Wire the new chore to the `satellite.Peer`.
+1. Wire the new chore to the `satellite.Core`.
 1. Remove the implementation of the current uptime disqualification.
   - `satellite/satellitedb.Overlaycache.UpdateUptime`: Remove update disqualified field due to lower uptime reputation.
    - `satellite/satellitedb.Overlaycache.populateUpdateNodeStats`: Remove update disqualified field due to lower uptime reputation.

internal/testblobs/slow.go

@@ -84,6 +84,12 @@ func (slow *SlowBlobs) Delete(ctx context.Context, ref storage.BlobRef) error {
 	return slow.blobs.Delete(ctx, ref)
 }
 
+// DeleteWithStorageFormat deletes the blob with the namespace, key, and format version
+func (slow *SlowBlobs) DeleteWithStorageFormat(ctx context.Context, ref storage.BlobRef, formatVer storage.FormatVersion) error {
+	slow.sleep()
+	return slow.blobs.DeleteWithStorageFormat(ctx, ref, formatVer)
+}
+
 // Stat looks up disk metadata on the blob file
 func (slow *SlowBlobs) Stat(ctx context.Context, ref storage.BlobRef) (storage.BlobInfo, error) {
 	slow.sleep()

internal/testplanet/satellite.go

@@ -55,7 +55,7 @@ import (
 
 // SatelliteSystem contains all the processes needed to run a full Satellite setup
 type SatelliteSystem struct {
-	Peer     *satellite.Peer
+	Core     *satellite.Core
 	API      *satellite.API
 	Repairer *satellite.Repairer
 
@@ -176,15 +176,15 @@ func (system *SatelliteSystem) URL() storj.NodeURL {
 
 // Close closes all the subsystems in the Satellite system
 func (system *SatelliteSystem) Close() error {
-	return errs.Combine(system.API.Close(), system.Peer.Close(), system.Repairer.Close())
+	return errs.Combine(system.API.Close(), system.Core.Close(), system.Repairer.Close())
 }
 
 // Run runs all the subsystems in the Satellite system
 func (system *SatelliteSystem) Run(ctx context.Context) (err error) {
 	group, ctx := errgroup.WithContext(ctx)
 
 	group.Go(func() error {
-		return errs2.IgnoreCanceled(system.Peer.Run(ctx))
+		return errs2.IgnoreCanceled(system.Core.Run(ctx))
 	})
 	group.Go(func() error {
 		return errs2.IgnoreCanceled(system.API.Run(ctx))
@@ -435,9 +435,9 @@ func (planet *Planet) newSatellites(count int) ([]*SatelliteSystem, error) {
 // before we split out the API. In the short term this will help keep all the tests passing
 // without much modification needed. However long term, we probably want to rework this
 // so it represents how the satellite will run when it is made up of many prrocesses.
-func createNewSystem(log *zap.Logger, peer *satellite.Peer, api *satellite.API, repairerPeer *satellite.Repairer) *SatelliteSystem {
+func createNewSystem(log *zap.Logger, peer *satellite.Core, api *satellite.API, repairerPeer *satellite.Repairer) *SatelliteSystem {
 	system := &SatelliteSystem{
-		Peer:     peer,
+		Core:     peer,
 		API:      api,
 		Repairer: repairerPeer,
 	}

satellite/accounting/bucketusage.go

@@ -4,77 +4,11 @@
 package accounting
 
 import (
-	"context"
 	"time"
 
 	"github.com/skyrings/skyring-common/tools/uuid"
 )
 
-// BucketUsage is bucket usage rollup repository
-//
-// architecture: Database
-type BucketUsage interface {
-	Get(ctx context.Context, id uuid.UUID) (*BucketRollup, error)
-	GetPaged(ctx context.Context, cursor *BucketRollupCursor) ([]BucketRollup, error)
-	Create(ctx context.Context, rollup BucketRollup) (*BucketRollup, error)
-	Delete(ctx context.Context, id uuid.UUID) error
-}
-
-// Order is sorting order can be asc or desc
-type Order string
-
-const (
-	// Asc ascending sort order
-	Asc Order = "asc"
-	// Desc descending sort order
-	Desc Order = "desc"
-)
-
-// BucketRollupCursor encapsulates cursor based page
-type BucketRollupCursor struct {
-	BucketID uuid.UUID
-	Before   time.Time
-	After    time.Time
-
-	Order Order
-
-	PageSize int
-	Next     *BucketRollupCursor
-}
-
-// BucketRollup holds usage rollup info
-type BucketRollup struct {
-	ID       uuid.UUID
-	BucketID uuid.UUID
-
-	RollupEndTime time.Time
-
-	RemoteStoredData uint64
-	InlineStoredData uint64
-	RemoteSegments   uint
-	InlineSegments   uint
-	Objects          uint
-	MetadataSize     uint64
-
-	RepairEgress uint64
-	GetEgress    uint64
-	AuditEgress  uint64
-}
-
-// BucketBandwidthRollup contains data about bandwidth rollup
-type BucketBandwidthRollup struct {
-	BucketName string
-	ProjectID  uuid.UUID
-
-	IntervalStart   time.Time
-	IntervalSeconds uint
-	Action          uint
-
-	Inline    uint64
-	Allocated uint64
-	Settled   uint64
-}
-
 // BucketStorageTally holds data about a bucket tally
 type BucketStorageTally struct {
 	BucketName    string