Update web API to return the real data of application counts (#1990)

**What this PR does / why we need it**:

**Which issue(s) this PR fixes**:

Fixes #

**Does this PR introduce a user-facing change?**:
<!--
If no, just write "NONE" in the release-note block below.
-->
```release-note
NONE
```

This PR was merged by Kapetanios.

cmd/pipecd/server.go

@@ -254,7 +254,7 @@ func (s *server) run(ctx context.Context, t cli.Telemetry) error {
 			return err
 		}
 
-		service := grpcapi.NewWebAPI(ctx, ds, sls, alss, cmds, is, rd, cfg.ProjectMap(), encryptDecrypter, t.Logger)
+		service := grpcapi.NewWebAPI(ctx, ds, fs, sls, alss, cmds, is, rd, cfg.ProjectMap(), encryptDecrypter, t.Logger)
 		opts := []rpc.Option{
 			rpc.WithPort(s.webAPIPort),
 			rpc.WithGracePeriod(s.gracePeriod),

pkg/app/api/grpcapi/BUILD.bazel

@@ -25,6 +25,7 @@ go_library(
         "//pkg/config:go_default_library",
         "//pkg/crypto:go_default_library",
         "//pkg/datastore:go_default_library",
+        "//pkg/filestore:go_default_library",
         "//pkg/git:go_default_library",
         "//pkg/insight/insightstore:go_default_library",
         "//pkg/model:go_default_library",

pkg/app/api/grpcapi/web_api.go

@@ -37,6 +37,7 @@ import (
 	"github.com/pipe-cd/pipe/pkg/config"
 	"github.com/pipe-cd/pipe/pkg/crypto"
 	"github.com/pipe-cd/pipe/pkg/datastore"
+	"github.com/pipe-cd/pipe/pkg/filestore"
 	"github.com/pipe-cd/pipe/pkg/git"
 	"github.com/pipe-cd/pipe/pkg/insight/insightstore"
 	"github.com/pipe-cd/pipe/pkg/model"
@@ -60,6 +61,7 @@ type WebAPI struct {
 	applicationLiveStateStore applicationlivestatestore.Store
 	insightstore              insightstore.Store
 	commandStore              commandstore.Store
+	insightStore              insightstore.Store
 	encrypter                 encrypter
 
 	appProjectCache        cache.Cache
@@ -75,6 +77,7 @@ type WebAPI struct {
 func NewWebAPI(
 	ctx context.Context,
 	ds datastore.DataStore,
+	fs filestore.Store,
 	sls stagelogstore.Store,
 	alss applicationlivestatestore.Store,
 	cmds commandstore.Store,
@@ -94,6 +97,7 @@ func NewWebAPI(
 		insightstore:              is,
 		applicationLiveStateStore: alss,
 		commandStore:              cmds,
+		insightStore:              insightstore.NewStore(fs),
 		projectsInConfig:          projs,
 		encrypter:                 encrypter,
 		appProjectCache:           memorycache.NewTTLCache(ctx, 24*time.Hour, 3*time.Hour),
@@ -1452,53 +1456,29 @@ func (a *WebAPI) GetInsightData(ctx context.Context, req *webservice.GetInsightD
 }
 
 func (a *WebAPI) GetInsightApplicationCount(ctx context.Context, req *webservice.GetInsightApplicationCountRequest) (*webservice.GetInsightApplicationCountResponse, error) {
-	_, err := rpcauth.ExtractClaims(ctx)
+	claims, err := rpcauth.ExtractClaims(ctx)
 	if err != nil {
 		a.logger.Error("failed to authenticate the current user", zap.Error(err))
 		return nil, err
 	}
 
-	// TODO: Fetch application count data from insight store.
-	counts := []*model.InsightApplicationCount{
-		{
-			Labels: map[string]string{
-				model.InsightApplicationCountLabelKey_KIND.String():          model.ApplicationKind_KUBERNETES.String(),
-				model.InsightApplicationCountLabelKey_ACTIVE_STATUS.String(): model.ApplicationActiveStatus_ENABLED.String(),
-			},
-			Count: 123,
-		},
-		{
-			Labels: map[string]string{
-				model.InsightApplicationCountLabelKey_KIND.String():          model.ApplicationKind_KUBERNETES.String(),
-				model.InsightApplicationCountLabelKey_ACTIVE_STATUS.String(): model.ApplicationActiveStatus_DISABLED.String(),
-			},
-			Count: 8,
-		},
-		{
-			Labels: map[string]string{
-				model.InsightApplicationCountLabelKey_KIND.String():          model.ApplicationKind_TERRAFORM.String(),
-				model.InsightApplicationCountLabelKey_ACTIVE_STATUS.String(): model.ApplicationActiveStatus_ENABLED.String(),
-			},
-			Count: 75,
-		},
-		{
-			Labels: map[string]string{
-				model.InsightApplicationCountLabelKey_KIND.String():          model.ApplicationKind_LAMBDA.String(),
-				model.InsightApplicationCountLabelKey_ACTIVE_STATUS.String(): model.ApplicationActiveStatus_DISABLED.String(),
-			},
-			Count: 2,
-		},
-		{
-			Labels: map[string]string{
-				model.InsightApplicationCountLabelKey_KIND.String():          model.ApplicationKind_CLOUDRUN.String(),
-				model.InsightApplicationCountLabelKey_ACTIVE_STATUS.String(): model.ApplicationActiveStatus_ENABLED.String(),
-			},
-			Count: 15,
-		},
+	// TODO: Cache application counts in the cache service.
+	c, err := a.insightStore.LoadApplicationCounts(ctx, claims.Role.ProjectId)
+	if err != nil {
+		if err == filestore.ErrNotFound {
+			return nil, status.Error(codes.NotFound, "Not found")
+		}
+		a.logger.Error("failed to load application counts", zap.Error(err))
+		return nil, status.Error(codes.Internal, "failed to load application counts")
+	}
+
+	counts := make([]*model.InsightApplicationCount, 0, len(c.Counts))
+	for i := range c.Counts {
+		counts = append(counts, &c.Counts[i])
 	}
 
 	return &webservice.GetInsightApplicationCountResponse{
-		UpdatedAt: time.Now().Unix(),
 		Counts:    counts,
+		UpdatedAt: c.UpdatedAt,
 	}, nil
 }

pkg/app/ops/insightcollector/applications.go

@@ -44,10 +44,7 @@ func (c *Collector) collectApplicationCount(ctx context.Context, apps []*model.A
 }
 
 func (c *Collector) updateApplicationCounts(ctx context.Context, projectID string, apps []*model.Application, target time.Time) error {
-	counts, err := insight.MakeApplicationCounts(apps, target)
-	if err != nil {
-		return fmt.Errorf("failed to make application counts: %w", err)
-	}
+	counts := insight.MakeApplicationCounts(apps, target)
 
 	if err := c.insightstore.PutApplicationCounts(ctx, projectID, counts); err != nil {
 		return fmt.Errorf("failed to put application counts: %w", err)
@@ -73,7 +70,7 @@ func (c *Collector) listApplications(ctx context.Context, to time.Time) ([]*mode
 			Orders: []datastore.Order{
 				{
 					Field:     "CreatedAt",
-					Direction: datastore.Desc,
+					Direction: datastore.Asc,
 				},
 				{
 					Field:     "Id",

pkg/insight/applicationcount.go

@@ -21,12 +21,48 @@ import (
 )
 
 type ApplicationCounts struct {
-	Counts        []model.InsightApplicationCount `json:"counts"`
-	AccumulatedTo int64                           `json:"accumulated_to"`
+	Counts    []model.InsightApplicationCount `json:"counts"`
+	UpdatedAt int64                           `json:"updated_at"`
 }
 
-func MakeApplicationCounts(apps []*model.Application, now time.Time) (*ApplicationCounts, error) {
-	return nil, nil
+func MakeApplicationCounts(apps []*model.Application, now time.Time) ApplicationCounts {
+	if len(apps) == 0 {
+		return ApplicationCounts{
+			UpdatedAt: now.Unix(),
+		}
+	}
+
+	type key struct {
+		status string
+		kind   string
+	}
+	m := make(map[key]int)
+	for _, app := range apps {
+		k := key{
+			status: model.ApplicationActiveStatus_ENABLED.String(),
+			kind:   app.Kind.String(),
+		}
+		if app.Disabled {
+			k.status = model.ApplicationActiveStatus_DISABLED.String()
+		}
+		m[k] = m[k] + 1
+	}
+
+	counts := make([]model.InsightApplicationCount, 0, len(m))
+	for k, c := range m {
+		counts = append(counts, model.InsightApplicationCount{
+			Labels: map[string]string{
+				model.InsightApplicationCountLabelKey_KIND.String():          k.kind,
+				model.InsightApplicationCountLabelKey_ACTIVE_STATUS.String(): k.status,
+			},
+			Count: int32(c),
+		})
+	}
+
+	return ApplicationCounts{
+		Counts:    counts,
+		UpdatedAt: now.Unix(),
+	}
 }
 
 func determineApplicationStatus(app *model.Application) model.ApplicationActiveStatus {

pkg/insight/applicationcount_test.go

@@ -15,8 +15,93 @@
 package insight
 
 import (
+	"fmt"
+	"sort"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/pipe-cd/pipe/pkg/model"
 )
 
 func TestMakeApplicationCounts(t *testing.T) {
+	now := time.Now()
+
+	testcases := []struct {
+		name     string
+		apps     []*model.Application
+		expected ApplicationCounts
+	}{
+		{
+			name: "empty",
+			apps: nil,
+			expected: ApplicationCounts{
+				UpdatedAt: now.Unix(),
+			},
+		},
+		{
+			name: "multiple applications",
+			apps: []*model.Application{
+				{
+					Id:       "1",
+					Kind:     model.ApplicationKind_KUBERNETES,
+					Disabled: false,
+				},
+				{
+					Id:       "2",
+					Kind:     model.ApplicationKind_KUBERNETES,
+					Disabled: false,
+				},
+				{
+					Id:       "3",
+					Kind:     model.ApplicationKind_KUBERNETES,
+					Disabled: true,
+				},
+				{
+					Id:       "4",
+					Kind:     model.ApplicationKind_CLOUDRUN,
+					Disabled: true,
+				},
+			},
+			expected: ApplicationCounts{
+				Counts: []model.InsightApplicationCount{
+					{
+						Labels: map[string]string{
+							"KIND":          "KUBERNETES",
+							"ACTIVE_STATUS": "ENABLED",
+						},
+						Count: 2,
+					},
+					{
+						Labels: map[string]string{
+							"KIND":          "KUBERNETES",
+							"ACTIVE_STATUS": "DISABLED",
+						},
+						Count: 1,
+					},
+					{
+						Labels: map[string]string{
+							"KIND":          "CLOUDRUN",
+							"ACTIVE_STATUS": "DISABLED",
+						},
+						Count: 1,
+					},
+				},
+				UpdatedAt: now.Unix(),
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		c := MakeApplicationCounts(tc.apps, now)
+		// We can use fmt to sort by Labels because maps are printed in key-sorted order.
+		sort.Slice(c.Counts, func(i, j int) bool {
+			return fmt.Sprint(c.Counts[i].Labels) > fmt.Sprint(c.Counts[j].Labels)
+		})
+		sort.Slice(tc.expected.Counts, func(i, j int) bool {
+			return fmt.Sprint(tc.expected.Counts[i].Labels) > fmt.Sprint(tc.expected.Counts[j].Labels)
+		})
+		assert.Equal(t, tc.expected, c)
+	}
 }

pkg/insight/insightstore/applicationcountstore.go

@@ -38,7 +38,7 @@ func (s *store) LoadApplicationCounts(ctx context.Context, projectID string) (*i
 }
 
 // PutApplicationCounts saves the ApplicationCounts data for a specific project into file store.
-func (s *store) PutApplicationCounts(ctx context.Context, projectID string, counts *insight.ApplicationCounts) error {
+func (s *store) PutApplicationCounts(ctx context.Context, projectID string, counts insight.ApplicationCounts) error {
 	data, err := json.Marshal(counts)
 	if err != nil {
 		return err

pkg/insight/insightstore/applicationcountstore_test.go

@@ -55,7 +55,7 @@ func TestLoadApplicationCounts(t *testing.T) {
 			name:      "successfully loaded from filestore",
 			projectID: "pid1",
 			content: `{
-				"accumulated_to": 1609459200,
+				"updated_at": 1609459200,
 				"counts": [
 					{
 						"labels": {
@@ -88,7 +88,7 @@ func TestLoadApplicationCounts(t *testing.T) {
 						Count: 1,
 					},
 				},
-				AccumulatedTo: 1609459200,
+				UpdatedAt: 1609459200,
 			},
 		},
 	}

pkg/insight/insightstore/insightstore.go

@@ -22,5 +22,5 @@ type Store interface {
 	PutMilestone(ctx context.Context, m *insight.Milestone) error
 
 	LoadApplicationCounts(ctx context.Context, projectID string) (*insight.ApplicationCounts, error)
-	PutApplicationCounts(ctx context.Context, projectID string, counts *insight.ApplicationCounts) error
+	PutApplicationCounts(ctx context.Context, projectID string, counts insight.ApplicationCounts) error
 }