Adds actor reminder scheduler perf tests (dapr#35)

* Adds actor reminder scheduler perf tests

Signed-off-by: joshvanl <me@joshvanl.dev>

* Run daprd in debug mode for test relying on new debug line

Signed-off-by: joshvanl <me@joshvanl.dev>

---------

Signed-off-by: joshvanl <me@joshvanl.dev>

pkg/actors/actors.go

@@ -239,7 +239,7 @@ func newActorsWithClock(opts ActorsOpts, clock clock.WithTicker) (ActorRuntime,
 		if opts.Config.SchedulerClients == nil {
 			return nil, fmt.Errorf("scheduler reminders are enabled, but no Scheduler clients are available")
 		}
-		log.Info("Using Scheduler service for reminders.")
+		log.Debug("Using Scheduler service for reminders.")
 		// We want to delete "a.actorsReminders" once we move to Scheduler service.
 		a.actorsReminders = reminders.NoOpReminders() // disable old reminder system if using Scheduler for reminders
 	} else {
@@ -1210,7 +1210,7 @@ func (a *actorsRuntime) CreateReminder(ctx context.Context, req *CreateReminderR
 	}
 
 	if a.schedulerClients != nil && a.schedulerReminders {
-		log.Info("Using Scheduler service for reminders")
+		log.Debug("Using Scheduler service for reminders")
 		var dueTime *string
 		if len(req.DueTime) > 0 {
 			dueTime = ptr.Of(req.DueTime)

tests/config/kubernetes_actor_reminder_scheduler_config.yaml

@@ -0,0 +1,8 @@
+apiVersion: dapr.io/v1alpha1
+kind: Configuration
+metadata:
+  name: featureactorreminderscheduler
+spec:
+  features:
+  - name: SchedulerReminders
+    enabled: true

tests/dapr_tests.mk

@@ -551,6 +551,7 @@ setup-test-components: setup-app-configurations
 	$(KUBECTL) apply -f ./tests/config/kubernetes_secret_config.yaml --namespace $(DAPR_TEST_NAMESPACE)
 	$(KUBECTL) apply -f ./tests/config/kubernetes_redis_secret.yaml --namespace $(DAPR_TEST_NAMESPACE)
 	$(KUBECTL) apply -f ./tests/config/kubernetes_redis_host_config.yaml --namespace $(DAPR_TEST_NAMESPACE)
+	$(KUBECTL) apply -f ./tests/config/kubernetes_actor_reminder_scheduler_config.yaml --namespace $(DAPR_TEST_NAMESPACE)
 	$(KUBECTL) apply -f ./tests/config/dapr_$(DAPR_TEST_STATE_STORE)_state.yaml --namespace $(DAPR_TEST_NAMESPACE)
 	$(KUBECTL) apply -f ./tests/config/dapr_$(DAPR_TEST_STATE_STORE)_state_actorstore.yaml --namespace $(DAPR_TEST_NAMESPACE)
 	$(KUBECTL) apply -f ./tests/config/dapr_$(DAPR_TEST_QUERY_STATE_STORE)_query_state.yaml --namespace $(DAPR_TEST_NAMESPACE)

tests/integration/suite/actors/reminders/scheduler/reminder.go

@@ -92,6 +92,7 @@ spec:
 		daprd.WithSchedulerAddresses(r.scheduler.Address()),
 		daprd.WithAppPort(srv.Port()),
 		daprd.WithAppProtocol("http"),
+		daprd.WithLogLevel("debug"),
 		daprd.WithExecOptions(
 			exec.WithStdout(r.loglineSchedulerReminders.Stdout()),
 		))

tests/perf/actor_reminder/actor_reminder_test.go

@@ -24,22 +24,25 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/dapr/dapr/tests/perf"
 	"github.com/dapr/dapr/tests/perf/utils"
 	kube "github.com/dapr/dapr/tests/platforms/kubernetes"
 	"github.com/dapr/dapr/tests/runner"
 	"github.com/dapr/dapr/tests/runner/summary"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 const (
-	numHealthChecks = 60 // Number of times to check for endpoint health per app.
-	actorType       = "PerfTestActorReminder"
-	appName         = "perf-actor-reminder-service"
+	numHealthChecks  = 60 // Number of times to check for endpoint health per app.
+	actorType        = "PerfTestActorReminder"
+	appName          = "perf-actor-reminder-service"
+	appNameScheduler = "perf-actor-reminder-scheduler-service"
 
 	// Target for the QPS
-	targetQPS = 56
+	targetQPS                  = 56
+	targetSchedulerQPS float64 = 2850
 )
 
 var tr *runner.TestRunner
@@ -67,6 +70,26 @@ func TestMain(m *testing.M) {
 				"TEST_APP_ACTOR_TYPE": actorType,
 			},
 		},
+		{
+			AppName:           appNameScheduler,
+			DaprEnabled:       true,
+			ImageName:         "perf-actorfeatures",
+			Replicas:          1,
+			IngressEnabled:    true,
+			AppPort:           3000,
+			DaprCPULimit:      "4.0",
+			DaprCPURequest:    "0.1",
+			DaprMemoryLimit:   "512Mi",
+			DaprMemoryRequest: "250Mi",
+			AppCPULimit:       "4.0",
+			AppCPURequest:     "0.1",
+			AppMemoryLimit:    "800Mi",
+			AppMemoryRequest:  "2500Mi",
+			AppEnv: map[string]string{
+				"TEST_APP_ACTOR_TYPE": actorType,
+			},
+			Config: "featureactorreminderscheduler",
+		},
 	}
 
 	tr = runner.NewTestRunner("actorreminder", testApps, nil, nil)
@@ -164,3 +187,98 @@ func TestActorReminderRegistrationPerformance(t *testing.T) {
 	assert.Equal(t, 0, restarts)
 	assert.True(t, daprResult.ActualQPS > targetQPS)
 }
+
+func TestActorReminderSchedulerTriggerPerformance(t *testing.T) {
+	p := perf.Params(
+		perf.WithQPS(5000),
+		perf.WithConnections(8),
+		perf.WithDuration("1m"),
+		perf.WithPayload("{}"),
+	)
+
+	// Get the ingress external url of test app
+	testAppURL := tr.Platform.AcquireAppExternalURL(appNameScheduler)
+	require.NotEmpty(t, testAppURL, "test app external URL must not be empty")
+
+	// Check if test app endpoint is available
+	t.Logf("test app url: %s", testAppURL+"/health")
+	_, err := utils.HTTPGetNTimes(testAppURL+"/health", numHealthChecks)
+	require.NoError(t, err)
+
+	// Perform dapr test
+	endpoint := fmt.Sprintf("http://127.0.0.1:3500/v1.0/actors/%v/{uuid}/reminders/myreminder", actorType)
+	p.TargetEndpoint = endpoint
+	p.Payload = `{"dueTime":"24h","period":"24h"}`
+	body, err := json.Marshal(&p)
+	require.NoError(t, err)
+
+	t.Logf("running dapr test with params: %s", body)
+	daprResp, err := utils.HTTPPost(fmt.Sprintf("%s/test", testAppURL), body)
+	t.Log("checking err...")
+	require.NoError(t, err)
+	require.NotEmpty(t, daprResp)
+	// fast fail if daprResp starts with error
+	require.False(t, strings.HasPrefix(string(daprResp), "error"))
+
+	// Let test run for 90s triggering the timers and collect metrics.
+	time.Sleep(90 * time.Second)
+
+	appUsage, err := tr.Platform.GetAppUsage(appNameScheduler)
+	require.NoError(t, err)
+
+	sidecarUsage, err := tr.Platform.GetSidecarUsage(appNameScheduler)
+	require.NoError(t, err)
+
+	restarts, err := tr.Platform.GetTotalRestarts(appNameScheduler)
+	require.NoError(t, err)
+
+	t.Logf("dapr test results: %s", string(daprResp))
+	t.Logf("target dapr app consumed %vm CPU and %vMb of Memory", appUsage.CPUm, appUsage.MemoryMb)
+	t.Logf("target dapr sidecar consumed %vm CPU and %vMb of Memory", sidecarUsage.CPUm, sidecarUsage.MemoryMb)
+	t.Logf("target dapr app or sidecar restarted %v times", restarts)
+
+	var daprResult perf.TestResult
+	err = json.Unmarshal(daprResp, &daprResult)
+	require.NoErrorf(t, err, "Failed to unmarshal: %s", string(daprResp))
+
+	percentiles := map[int]string{2: "90th", 3: "99th"}
+
+	for k, v := range percentiles {
+		daprValue := daprResult.DurationHistogram.Percentiles[k].Value
+		t.Logf("%s percentile: %sms", v, fmt.Sprintf("%.2f", daprValue*1000))
+	}
+
+	t.Logf("Actual QPS: %.2f, expected QPS: %.0f", daprResult.ActualQPS, targetSchedulerQPS)
+
+	report := perf.NewTestReport(
+
+		[]perf.TestResult{daprResult},
+		"Actor Reminder",
+		sidecarUsage,
+		appUsage)
+
+	report.SetTotalRestartCount(restarts)
+	err = utils.UploadAzureBlob(report)
+
+	if err != nil {
+		t.Error(err)
+	}
+
+	summary.ForTest(t).
+		Service(appNameScheduler).
+		Client(appNameScheduler).
+		CPU(appUsage.CPUm).
+		Memory(appUsage.MemoryMb).
+		SidecarCPU(sidecarUsage.CPUm).
+		SidecarMemory(sidecarUsage.MemoryMb).
+		Restarts(restarts).
+		ActualQPS(daprResult.ActualQPS).
+		Params(p).
+		OutputFortio(daprResult).
+		Flush()
+
+	assert.Equal(t, 0, daprResult.RetCodes.Num400)
+	assert.Equal(t, 0, daprResult.RetCodes.Num500)
+	assert.Equal(t, 0, restarts)
+	assert.GreaterOrEqual(t, daprResult.ActualQPS, targetSchedulerQPS)
+}