Merge pull request #9 from uber/upstream_patches

Upstream patches
uber-archive · Jan 11, 2017 · 9c0860b · 9c0860b
2 parents e60e28d + eba5c0d
commit 9c0860b
Show file tree

Hide file tree

Showing 48 changed files with 3,104 additions and 1,231 deletions.
diff --git a/clients/metadata/metadata_cassandra.go b/clients/metadata/metadata_cassandra.go
@@ -102,6 +102,7 @@ const (
 	columnHostName                       = "hostname"
 	columnInputHostUUID                  = "input_host_uuid"
 	columnOriginZone                     = "origin_zone"
+	columnRemoteExtentPrimaryStore       = "remote_extent_primary_store"
 	columnLastAddress                    = "last_address"
 	columnLastEnqueueTime                = "last_enqueue_time"
 	columnLastSequence                   = "last_sequence"
@@ -1926,6 +1927,7 @@ const (
 		columnStoreUUIDS + `: ?, ` +
 		columnInputHostUUID + `: ?, ` +
 		columnOriginZone + `: ?, ` +
+		columnRemoteExtentPrimaryStore + `: ?, ` +
 		columnStatus + `: ?}`
 
 	sqlInsertDstExent = `INSERT INTO ` + tableDestinationExtents + ` (` +
@@ -1970,6 +1972,7 @@ const (
 		columnStoreUUIDS + `: ?, ` +
 		columnInputHostUUID + `: ?, ` +
 		columnOriginZone + `: ?, ` +
+		columnRemoteExtentPrimaryStore + `: ?, ` +
 		columnStatus + `: ?, ` +
 		columnArchivalLocation + `: ?` +
 		`}`
@@ -2084,6 +2087,7 @@ func (s *CassandraMetadataService) createExtentImpl(extent *shared.Extent, exten
 		extent.GetStoreUUIDs(),
 		extent.GetInputHostUUID(),
 		extent.GetOriginZone(),
+		extent.GetRemoteExtentPrimaryStore(),
 		extentStatus,
 		replicaStatsList,
 		consumerGroupVisibility,
@@ -2101,6 +2105,7 @@ func (s *CassandraMetadataService) createExtentImpl(extent *shared.Extent, exten
 		extent.GetStoreUUIDs(),
 		extent.GetInputHostUUID(),
 		extent.GetOriginZone(),
+		extent.GetRemoteExtentPrimaryStore(),
 		extentStatus,
 		replicaStatsList,
 	)
@@ -2122,6 +2127,7 @@ func (s *CassandraMetadataService) createExtentImpl(extent *shared.Extent, exten
 			extent.GetStoreUUIDs(),
 			extent.GetInputHostUUID(),
 			extent.GetOriginZone(),
+			extent.GetRemoteExtentPrimaryStore(),
 			extentStatus,
 			replicaStats,
 		)
@@ -2190,6 +2196,7 @@ func (s *CassandraMetadataService) deleteExtentImpl(extent *shared.Extent, exten
 		extent.GetStoreUUIDs(),
 		extent.GetInputHostUUID(),
 		extent.GetOriginZone(),
+		extent.GetRemoteExtentPrimaryStore(),
 		shared.ExtentStatus_DELETED,
 		extentStatsMap,
 		extentStats.ConsumerGroupVisibility,
@@ -2208,6 +2215,7 @@ func (s *CassandraMetadataService) deleteExtentImpl(extent *shared.Extent, exten
 		extent.GetStoreUUIDs(),
 		extent.GetInputHostUUID(),
 		extent.GetOriginZone(),
+		extent.GetRemoteExtentPrimaryStore(),
 		shared.ExtentStatus_DELETED,
 		extentStatsMap,
 		defaultDeleteTTLSeconds,
@@ -2227,6 +2235,7 @@ func (s *CassandraMetadataService) deleteExtentImpl(extent *shared.Extent, exten
 				extent.GetStoreUUIDs(),
 				extent.GetInputHostUUID(),
 				extent.GetOriginZone(),
+				extent.GetRemoteExtentPrimaryStore(),
 				shared.ExtentStatus_DELETED,
 				replicaStats,
 				defaultDeleteTTLSeconds,
@@ -2252,6 +2261,7 @@ func (s *CassandraMetadataService) updateExtent(extentStats *shared.ExtentStats,
 		extent.GetStoreUUIDs(),
 		extent.GetInputHostUUID(),
 		extent.GetOriginZone(),
+		extent.GetRemoteExtentPrimaryStore(),
 		newStatus,
 		newArchivalLocation,
 		extent.GetDestinationUUID(),
@@ -2266,6 +2276,7 @@ func (s *CassandraMetadataService) updateExtent(extentStats *shared.ExtentStats,
 		extent.GetStoreUUIDs(),
 		extent.GetInputHostUUID(),
 		extent.GetOriginZone(),
+		extent.GetRemoteExtentPrimaryStore(),
 		newStatus,
 		newArchivalLocation,
 		extent.GetDestinationUUID(),
@@ -2282,6 +2293,7 @@ func (s *CassandraMetadataService) updateExtent(extentStats *shared.ExtentStats,
 			extent.GetStoreUUIDs(),
 			extent.GetInputHostUUID(),
 			extent.GetOriginZone(),
+			extent.GetRemoteExtentPrimaryStore(),
 			newStatus,
 			newArchivalLocation,
 			storeID,
@@ -2316,6 +2328,10 @@ func (s *CassandraMetadataService) UpdateExtentStats(ctx thrift.Context, request
 	}
 
 	extent := extentStatsResult.ExtentStats.Extent
+	if request.IsSetRemoteExtentPrimaryStore() {
+		extent.RemoteExtentPrimaryStore = common.StringPtr(request.GetRemoteExtentPrimaryStore())
+	}
+
 	if len(request.GetArchivalLocation()) == 0 {
 		request.ArchivalLocation = common.StringPtr(extentStatsResult.ExtentStats.GetArchivalLocation())
 	}
@@ -2442,11 +2458,12 @@ func uuidSliceToStringSlice(i interface{}) []string {
 func convertExtentStats(extentMap map[string]interface{}, extentStatsMap map[string]map[string]interface{}) *shared.ExtentStats {
 	result := &shared.ExtentStats{
 		Extent: &shared.Extent{
-			ExtentUUID:      common.StringPtr(toUUIDString(extentMap[columnUUID])),
-			DestinationUUID: common.StringPtr(toUUIDString(extentMap[columnDestinationUUID])),
-			StoreUUIDs:      uuidSliceToStringSlice(extentMap[columnStoreUUIDS]),
-			InputHostUUID:   common.StringPtr(toUUIDString(extentMap[columnInputHostUUID])),
-			OriginZone:      common.StringPtr(toString(extentMap[columnOriginZone])),
+			ExtentUUID:               common.StringPtr(toUUIDString(extentMap[columnUUID])),
+			DestinationUUID:          common.StringPtr(toUUIDString(extentMap[columnDestinationUUID])),
+			StoreUUIDs:               uuidSliceToStringSlice(extentMap[columnStoreUUIDS]),
+			InputHostUUID:            common.StringPtr(toUUIDString(extentMap[columnInputHostUUID])),
+			OriginZone:               common.StringPtr(toString(extentMap[columnOriginZone])),
+			RemoteExtentPrimaryStore: common.StringPtr(toString(extentMap[columnRemoteExtentPrimaryStore])),
 		},
 		Status:           common.MetadataExtentStatusPtr(shared.ExtentStatus(toInt(extentMap[columnStatus]))),
 		ArchivalLocation: common.StringPtr(toString(extentMap[columnArchivalLocation])),

diff --git a/clients/metadata/metadata_cassandra_test.go b/clients/metadata/metadata_cassandra_test.go
@@ -563,10 +563,11 @@ func (s *CassandraSuite) TestExtentCRU() {
 		extentUUID := uuid.New()
 		storeIds := []string{uuid.New(), uuid.New(), uuid.New()}
 		extent := &shared.Extent{
-			ExtentUUID:      common.StringPtr(extentUUID),
-			DestinationUUID: common.StringPtr(dest.GetDestinationUUID()),
-			StoreUUIDs:      storeIds,
-			InputHostUUID:   common.StringPtr(uuid.New()),
+			ExtentUUID:               common.StringPtr(extentUUID),
+			DestinationUUID:          common.StringPtr(dest.GetDestinationUUID()),
+			StoreUUIDs:               storeIds,
+			InputHostUUID:            common.StringPtr(uuid.New()),
+			RemoteExtentPrimaryStore: common.StringPtr(uuid.New()),
 		}
 		createExtent := &shared.CreateExtentRequest{Extent: extent}
 		t0 := time.Now().UnixNano() / int64(time.Millisecond)
@@ -622,10 +623,11 @@ func (s *CassandraSuite) TestExtentCRU() {
 		s.Equal(extentStatsOrig.GetStatusUpdatedTimeMillis(), extentStats.GetExtentStats().GetStatusUpdatedTimeMillis())
 
 		updateExtent := &m.UpdateExtentStatsRequest{
-			DestinationUUID:  common.StringPtr(extent.GetDestinationUUID()),
-			ExtentUUID:       common.StringPtr(extent.GetExtentUUID()),
-			Status:           common.MetadataExtentStatusPtr(shared.ExtentStatus_ARCHIVED),
-			ArchivalLocation: common.StringPtr("S3:foo/bar"),
+			DestinationUUID:          common.StringPtr(extent.GetDestinationUUID()),
+			ExtentUUID:               common.StringPtr(extent.GetExtentUUID()),
+			Status:                   common.MetadataExtentStatusPtr(shared.ExtentStatus_ARCHIVED),
+			ArchivalLocation:         common.StringPtr("S3:foo/bar"),
+			RemoteExtentPrimaryStore: common.StringPtr(uuid.New()),
 		}
 
 		t0 := time.Now().UnixNano() / int64(time.Millisecond)
@@ -634,14 +636,17 @@ func (s *CassandraSuite) TestExtentCRU() {
 
 		s.Nil(err)
 		s.Equal(updateExtent.Status, updateResult.ExtentStats.Status)
-		s.Equal(updateExtent.ArchivalLocation, updateResult.ExtentStats.ArchivalLocation)
+		s.Equal(updateExtent.GetArchivalLocation(), updateResult.GetExtentStats().GetArchivalLocation())
+		s.Equal(updateExtent.GetRemoteExtentPrimaryStore(), updateResult.GetExtentStats().GetExtent().GetRemoteExtentPrimaryStore())
 
 		readExtentStats = &m.ReadExtentStatsRequest{DestinationUUID: extent.DestinationUUID, ExtentUUID: extent.ExtentUUID}
 		time.Sleep(1 * time.Second)
 		extentStats, err = s.client.ReadExtentStats(nil, readExtentStats)
 		s.Nil(err)
 		s.NotNil(extentStats)
-		s.Equal(shared.ExtentStatus_ARCHIVED, extentStats.ExtentStats.GetStatus())
+		s.Equal(shared.ExtentStatus_ARCHIVED, extentStats.GetExtentStats().GetStatus())
+		s.Equal(updateExtent.GetArchivalLocation(), extentStats.GetExtentStats().GetArchivalLocation())
+		s.Equal(updateExtent.GetRemoteExtentPrimaryStore(), extentStats.GetExtentStats().GetExtent().GetRemoteExtentPrimaryStore())
 		s.True(extentStats.GetExtentStats().GetStatusUpdatedTimeMillis() >= t0)
 		s.True(extentStats.GetExtentStats().GetStatusUpdatedTimeMillis() <= tX)
 	}

diff --git a/clients/metadata/schema/metadata_test.cql b/clients/metadata/schema/metadata_test.cql
@@ -115,7 +115,8 @@ CREATE TYPE extent (
   input_host_uuid uuid,
   status int, -- ExtentStatus enum
   archival_location text,
-  origin_zone text
+  origin_zone text,
+  remote_extent_primary_store text, // the primary store for remote extent
 );
 
 CREATE TYPE extent_replica_stats (

diff --git a/clients/metadata/schema/v13/201612180000_xdc_add_remote_extent_primary_store.cql b/clients/metadata/schema/v13/201612180000_xdc_add_remote_extent_primary_store.cql
@@ -0,0 +1 @@
+ALTER TYPE extent ADD remote_extent_primary_store text;
diff --git a/clients/metadata/schema/v13/manifest.json b/clients/metadata/schema/v13/manifest.json
@@ -0,0 +1,8 @@
+{
+	"CurrVersion": 13,
+	"MinCompatibleVersion": 8,
+	"Description": "add remote_extent_primary_store to extent type",
+	"SchemaUpdateCqlFiles": [
+		"201612180000_xdc_add_remote_extent_primary_store.cql"
+	]
+}
diff --git a/cmd/tools/cli/main.go b/cmd/tools/cli/main.go
@@ -53,7 +53,7 @@ func main() {
 	})
 	app.Name = "cherami"
 	app.Usage = "A command-line tool for cherami users"
-	app.Version = "1.1.4"
+	app.Version = "1.1.5"
 	app.Flags = []cli.Flag{
 		cli.BoolTFlag{
 			Name:  "hyperbahn",
@@ -66,8 +66,8 @@ func main() {
 		},
 		cli.StringFlag{
 			Name:  "env",
-			Value: "",
-			Usage: "Deployment to connect. By default connects to production. Use \"staging\" to connect to staging",
+			Value: "staging",
+			Usage: "Deployment to connect to. By default connects to staging. Use \"prod\" to connect to production",
 		},
 		cli.StringFlag{
 			Name:   "hyperbahn_bootstrap_file, hbfile",