Skip to content

Commit

Permalink
Add a WithFxOptionsForService option for our integration tests (#4895)
Browse files Browse the repository at this point in the history
<!-- Describe what has changed in this PR -->
**What changed?**
I added an option that allows user to add fx graph modification to
individual services when starting them in the SetupSuite lifecycle stage
of their integration tests.

<!-- Tell your future self why have you made these changes -->
**Why?**
This provides a super simple way to change test set ups or extract
dependencies from the graph. It should make it a lot easier to write
integration tests with more complex setups or assertions. I also need it
for a subsequent PR that adds integration tests to the new DLQ.
Specifically, I need a way to override the `ExecutableWrapper`, and I
think this is simpler than adding it to the long and growing list of
dependencies in the `TestClusterConfig`.

<!-- How have you verified this change? Tested locally? Added a unit
test? Checked in staging env? -->
**How did you test it?**
I added an integration test which sets this up and provides an
`fx.Populate` option for the `primitives.ServiceName` variable in the
current graph. I then verified that this service name matched the name
of the service these options were supplied to, to ensure that the
options were provided to the correct graph (because we have different
graphs for each service).

<!-- Assuming the worst case, what can be broken when deploying this
change to production? -->
**Potential risks**


<!-- Is this PR a hotfix candidate or require that a notification be
sent to the broader community? (Yes/No) -->
**Is hotfix candidate?**
  • Loading branch information
MichaelSnowden committed Oct 6, 2023
1 parent 5ab5265 commit 2f62c28
Show file tree
Hide file tree
Showing 4 changed files with 135 additions and 4 deletions.
36 changes: 35 additions & 1 deletion tests/functionalbase.go → tests/functional_test_base.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,11 @@ import (
"github.com/stretchr/testify/suite"
"go.temporal.io/api/operatorservice/v1"
"go.temporal.io/api/workflowservice/v1"
"go.uber.org/fx"
"gopkg.in/yaml.v3"

"go.temporal.io/server/common/primitives"

commonpb "go.temporal.io/api/common/v1"
enumspb "go.temporal.io/api/enums/v1"
historypb "go.temporal.io/api/history/v1"
Expand Down Expand Up @@ -76,14 +79,45 @@ type (
archivalNamespace string
dynamicConfigOverrides map[dynamicconfig.Key]interface{}
}
// suiteParams contains the variables which are used to configure the test suite via the Option argument to
// setupSuite.
suiteParams struct {
fxOptions map[primitives.ServiceName][]fx.Option
}
Option func(params *suiteParams)
)

func (s *FunctionalTestBase) setupSuite(defaultClusterConfigFile string) {
// WithFxOptionsForService returns an Option which, when passed as an argument to setupSuite, will append the given list
// of fx options to the end of the arguments to the fx.New call for the given service. For example, if you want to
// obtain the shard controller for the history service, you can do this:
//
// var shardController shard.Controller
// s.setupSuite(t, tests.WithFxOptionsForService(primitives.HistoryService, fx.Populate(&shardController)))
// // now you can use shardController during your test
//
// This is similar to the pattern of plumbing dependencies through the TestClusterConfig, but it's much more convenient,
// scalable and flexible. The reason we need to do this on a per-service basis is that there are separate fx apps for
// each one.
func WithFxOptionsForService(serviceName primitives.ServiceName, options ...fx.Option) Option {
return func(params *suiteParams) {
params.fxOptions[serviceName] = append(params.fxOptions[serviceName], options...)
}
}

func (s *FunctionalTestBase) setupSuite(defaultClusterConfigFile string, options ...Option) {
params := suiteParams{
fxOptions: make(map[primitives.ServiceName][]fx.Option),
}
for _, opt := range options {
opt(&params)
}

s.setupLogger()

clusterConfig, err := GetTestClusterConfig(defaultClusterConfigFile)
s.Require().NoError(err)
clusterConfig.DynamicConfigOverrides = s.dynamicConfigOverrides
clusterConfig.ServiceFxOptions = params.fxOptions
s.testClusterConfig = clusterConfig

if clusterConfig.FrontendAddress != "" {
Expand Down
79 changes: 79 additions & 0 deletions tests/functional_test_base_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// The MIT License
//
// Copyright (c) 2020 Temporal Technologies Inc. All rights reserved.
//
// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package tests

import (
"testing"

"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"go.uber.org/fx"

"go.temporal.io/server/common/primitives"
)

type functionalTestBaseSuite struct {
*require.Assertions
FunctionalTestBase
frontendServiceName primitives.ServiceName
matchingServiceName primitives.ServiceName
historyServiceName primitives.ServiceName
workerServiceName primitives.ServiceName
}

func (s *functionalTestBaseSuite) SetupSuite() {
s.setupSuite("testdata/cluster.yaml",
WithFxOptionsForService(primitives.FrontendService, fx.Populate(&s.frontendServiceName)),
WithFxOptionsForService(primitives.MatchingService, fx.Populate(&s.matchingServiceName)),
WithFxOptionsForService(primitives.HistoryService, fx.Populate(&s.historyServiceName)),
WithFxOptionsForService(primitives.WorkerService, fx.Populate(&s.workerServiceName)),
)

}

func (s *functionalTestBaseSuite) TearDownSuite() {
s.tearDownSuite()
}

func (s *functionalTestBaseSuite) TestWithFxOptionsForService() {
// This test works by using the WithFxOptionsForService option to obtain the ServiceName from the graph, and then
// it verifies that the ServiceName is correct. It does this because we are targeting the fx.App for a particular
// service, so we'll know our fx options were provided to the right service if, when we use them to get the current
// service name, it matches the target service. A more realistic example would use the option to obtain an actual
// useful object like a history shard controller, or do some graph modifications with fx.Decorate.

s.Equal(primitives.FrontendService, s.frontendServiceName)
s.Equal(primitives.MatchingService, s.matchingServiceName)
s.Equal(primitives.HistoryService, s.historyServiceName)
s.Equal(primitives.WorkerService, s.workerServiceName)
}

func (s *functionalTestBaseSuite) SetupTest() {
s.Assertions = require.New(s.T())
}

func TestFunctionalTestBaseSuite(t *testing.T) {
suite.Run(t, new(functionalTestBaseSuite))
}
18 changes: 15 additions & 3 deletions tests/onebox.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,10 @@ type (
tlsConfigProvider *encryption.FixedTLSConfigProvider
captureMetricsHandler *metricstest.CaptureHandler

onGetClaims func(*authorization.AuthInfo) (*authorization.Claims, error)
onAuthorize func(context.Context, *authorization.Claims, *authorization.CallTarget) (authorization.Result, error)
callbackLock sync.RWMutex // Must be used for above callbacks
onGetClaims func(*authorization.AuthInfo) (*authorization.Claims, error)
onAuthorize func(context.Context, *authorization.Claims, *authorization.CallTarget) (authorization.Result, error)
callbackLock sync.RWMutex // Must be used for above callbacks
serviceFxOptions map[primitives.ServiceName][]fx.Option
}

// HistoryConfig contains configs for history service
Expand Down Expand Up @@ -172,6 +173,8 @@ type (
DynamicConfigOverrides map[dynamicconfig.Key]interface{}
TLSConfigProvider *encryption.FixedTLSConfigProvider
CaptureMetricsHandler *metricstest.CaptureHandler
// ServiceFxOptions is populated by WithFxOptionsForService.
ServiceFxOptions map[primitives.ServiceName][]fx.Option
}

listenHostPort string
Expand Down Expand Up @@ -207,6 +210,7 @@ func newTemporal(params *TemporalParams) *temporalImpl {
tlsConfigProvider: params.TLSConfigProvider,
captureMetricsHandler: params.CaptureMetricsHandler,
dcClient: testDCClient,
serviceFxOptions: params.ServiceFxOptions,
}
impl.overrideHistoryDynamicConfig(testDCClient)
return impl
Expand Down Expand Up @@ -446,6 +450,7 @@ func (c *temporalImpl) startFrontend(hosts map[primitives.ServiceName][]string,
frontend.Module,
fx.Populate(&frontendService, &clientBean, &namespaceRegistry, &rpcFactory),
temporal.FxLogAdapter,
c.getFxOptionsForService(primitives.FrontendService),
)
err = feApp.Err()
if err != nil {
Expand Down Expand Up @@ -539,6 +544,7 @@ func (c *temporalImpl) startHistory(
replication.Module,
fx.Populate(&historyService, &clientBean, &namespaceRegistry),
temporal.FxLogAdapter,
c.getFxOptionsForService(primitives.HistoryService),
)
err = app.Err()
if err != nil {
Expand Down Expand Up @@ -629,6 +635,7 @@ func (c *temporalImpl) startMatching(hosts map[primitives.ServiceName][]string,
matching.Module,
fx.Populate(&matchingService, &clientBean, &namespaceRegistry),
temporal.FxLogAdapter,
c.getFxOptionsForService(primitives.MatchingService),
)
err = app.Err()
if err != nil {
Expand Down Expand Up @@ -724,6 +731,7 @@ func (c *temporalImpl) startWorker(hosts map[primitives.ServiceName][]string, st
worker.Module,
fx.Populate(&workerService, &clientBean, &namespaceRegistry),
temporal.FxLogAdapter,
c.getFxOptionsForService(primitives.WorkerService),
)
err = app.Err()
if err != nil {
Expand All @@ -742,6 +750,10 @@ func (c *temporalImpl) startWorker(hosts map[primitives.ServiceName][]string, st
c.shutdownWG.Done()
}

func (c *temporalImpl) getFxOptionsForService(serviceName primitives.ServiceName) fx.Option {
return fx.Options(c.serviceFxOptions[serviceName]...)
}

func (c *temporalImpl) createSystemNamespace() error {
err := c.metadataMgr.InitializeSystemNamespaces(context.Background(), c.clusterMetadataConfig.CurrentClusterName)
if err != nil {
Expand Down
6 changes: 6 additions & 0 deletions tests/test_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,11 @@ import (

"github.com/pborman/uuid"
"go.temporal.io/api/operatorservice/v1"
"go.uber.org/fx"
"go.uber.org/multierr"

"go.temporal.io/server/common/primitives"

"go.temporal.io/server/api/adminservice/v1"
"go.temporal.io/server/api/matchingservice/v1"
persistencespb "go.temporal.io/server/api/persistence/v1"
Expand Down Expand Up @@ -96,6 +99,8 @@ type (
DynamicConfigOverrides map[dynamicconfig.Key]interface{}
GenerateMTLS bool
EnableMetricsCapture bool
// ServiceFxOptions can be populated using WithFxOptionsForService.
ServiceFxOptions map[primitives.ServiceName][]fx.Option
}

// WorkerConfig is the config for enabling/disabling Temporal worker
Expand Down Expand Up @@ -259,6 +264,7 @@ func NewCluster(options *TestClusterConfig, logger log.Logger) (*TestCluster, er
NamespaceReplicationTaskExecutor: namespace.NewReplicationTaskExecutor(options.ClusterMetadata.CurrentClusterName, testBase.MetadataManager, logger),
DynamicConfigOverrides: options.DynamicConfigOverrides,
TLSConfigProvider: tlsConfigProvider,
ServiceFxOptions: options.ServiceFxOptions,
}

if options.EnableMetricsCapture {
Expand Down

0 comments on commit 2f62c28

Please sign in to comment.