Merge main into release 0.6

CHANGELOG.md

@@ -1,3 +1,11 @@
+## v0.6.0-rc.2
+
+-   [Bugfix] Dereference slice when finding source subscriptions #808
+-   [Bugfix] Use redis client in scheduler #807
+-   [Bugfix] Fixed an issue where the source type would not be updated when updating a Github source
+-   [Bugfix] Fixed an issue where the application details would not be loaded when creating a subscription
+-   [Bugfix] Fixed an issue where an organization created by a user would not show on the top bar to be selected
+
 ## v0.6.0-rc.1
 
 -   [Change] Introduce organisations to partition different sets of projects.

VERSION

@@ -1 +1 @@
-v0.6.0-rc.1
+v0.6.0-rc.2

analytics/active_group_analytics.go

@@ -9,48 +9,70 @@ import (
 )
 
 type ActiveGroupAnalytics struct {
-	groupRepo datastore.GroupRepository
-	eventRepo datastore.EventRepository
-	client    AnalyticsClient
-	host      string
+	groupRepo  datastore.GroupRepository
+	eventRepo  datastore.EventRepository
+	orgRepo    datastore.OrganisationRepository
+	client     AnalyticsClient
+	instanceID string
 }
 
-func newActiveGroupAnalytics(groupRepo datastore.GroupRepository, eventRepo datastore.EventRepository, client AnalyticsClient, host string) *ActiveGroupAnalytics {
+func newActiveGroupAnalytics(groupRepo datastore.GroupRepository, eventRepo datastore.EventRepository, orgRepo datastore.OrganisationRepository, client AnalyticsClient, instanceID string) *ActiveGroupAnalytics {
 	return &ActiveGroupAnalytics{
-		groupRepo: groupRepo,
-		eventRepo: eventRepo,
-		client:    client,
-		host:      host,
+		groupRepo:  groupRepo,
+		eventRepo:  eventRepo,
+		orgRepo:    orgRepo,
+		client:     client,
+		instanceID: instanceID,
 	}
 
 }
 
 func (a *ActiveGroupAnalytics) Track() error {
-	groups, err := a.groupRepo.LoadGroups(context.Background(), &datastore.GroupFilter{})
+	return a.track(PerPage, Page, 0)
+}
+
+func (a *ActiveGroupAnalytics) track(perPage, page, count int) error {
+	ctx := context.Background()
+	orgs, _, err := a.orgRepo.LoadOrganisationsPaged(ctx, datastore.Pageable{PerPage: perPage, Page: page, Sort: -1})
+
 	if err != nil {
 		return err
 	}
 
-	count := 0
-	now := time.Now()
-	for _, group := range groups {
-		filter := datastore.SearchParams{
-			CreatedAtStart: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).Unix(),
-			CreatedAtEnd:   time.Date(now.Year(), now.Month(), now.Day(), 23, 59, 59, 999999999, time.UTC).Unix(),
-		}
+	if len(orgs) == 0 {
+		return a.client.Export(a.Name(), Event{"Count": count, "instanceID": a.instanceID})
+	}
 
-		events, _, err := a.eventRepo.LoadEventsPaged(context.Background(), group.UID, "", filter, datastore.Pageable{Sort: -1})
+	now := time.Now()
+	for _, org := range orgs {
+		groups, err := a.groupRepo.LoadGroups(ctx, &datastore.GroupFilter{OrgID: org.UID})
 		if err != nil {
-			log.WithError(err).Error("failed to load events paged")
+			log.WithError(err).Error("failed to load organisation groups")
 			continue
 		}
 
-		if len(events) > 0 {
-			count += 1
+		for _, group := range groups {
+			filter := datastore.SearchParams{
+				CreatedAtStart: time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC).Unix(),
+				CreatedAtEnd:   time.Date(now.Year(), now.Month(), now.Day(), 23, 59, 59, 999999999, time.UTC).Unix(),
+			}
+
+			events, _, err := a.eventRepo.LoadEventsPaged(ctx, group.UID, "", filter, datastore.Pageable{Sort: -1})
+			if err != nil {
+				log.WithError(err).Error("failed to load events paged")
+				continue
+			}
+
+			if len(events) > 0 {
+				count += 1
+			}
 		}
 	}
 
-	return a.client.Export(a.Name(), Event{"Count": count, "Host": a.host})
+	perPage += 20
+	page += 1
+
+	return a.track(perPage, page, count)
 
 }
 

analytics/active_group_analytics_test.go

@@ -13,9 +13,10 @@ import (
 func provideActiveGroupAnalytics(ctrl *gomock.Controller) *ActiveGroupAnalytics {
 	groupRepo := mocks.NewMockGroupRepository(ctrl)
 	eventRepo := mocks.NewMockEventRepository(ctrl)
+	orgRepo := mocks.NewMockOrganisationRepository(ctrl)
 	client := NewNoopAnalyticsClient()
 
-	return newActiveGroupAnalytics(groupRepo, eventRepo, client, TestHost)
+	return newActiveGroupAnalytics(groupRepo, eventRepo, orgRepo, client, TestInstanceID)
 }
 
 func Test_TrackActiveGroupAnalytics(t *testing.T) {
@@ -30,17 +31,21 @@ func Test_TrackActiveGroupAnalytics(t *testing.T) {
 			dbFn: func(ga *ActiveGroupAnalytics) {
 				groupRepo := ga.groupRepo.(*mocks.MockGroupRepository)
 				eventRepo := ga.eventRepo.(*mocks.MockEventRepository)
-
-				groupRepo.EXPECT().LoadGroups(gomock.Any(), gomock.Any()).Return([]*datastore.Group{{UID: "123456", Name: "test"}}, nil)
-				eventRepo.EXPECT().LoadEventsPaged(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, datastore.PaginationData{}, nil)
+				orgRepo := ga.orgRepo.(*mocks.MockOrganisationRepository)
+				gomock.InOrder(
+					orgRepo.EXPECT().LoadOrganisationsPaged(gomock.Any(), gomock.Any()).Return([]datastore.Organisation{{UID: "123"}}, datastore.PaginationData{}, nil),
+					groupRepo.EXPECT().LoadGroups(gomock.Any(), gomock.Any()).Return([]*datastore.Group{{UID: "123456", Name: "test"}}, nil),
+					eventRepo.EXPECT().LoadEventsPaged(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, datastore.PaginationData{}, nil),
+					orgRepo.EXPECT().LoadOrganisationsPaged(gomock.Any(), gomock.Any()).Return(nil, datastore.PaginationData{}, nil),
+				)
 			},
 		},
 
 		{
 			name: "should_fail_to_track_active_group_analytics",
 			dbFn: func(ga *ActiveGroupAnalytics) {
-				groupRepo := ga.groupRepo.(*mocks.MockGroupRepository)
-				groupRepo.EXPECT().LoadGroups(gomock.Any(), gomock.Any()).Return(nil, errors.New("failed"))
+				orgRepo := ga.orgRepo.(*mocks.MockOrganisationRepository)
+				orgRepo.EXPECT().LoadOrganisationsPaged(gomock.Any(), gomock.Any()).Return(nil, datastore.PaginationData{}, errors.New("failed"))
 			},
 			wantErr: true,
 		},

analytics/analytics.go

@@ -21,6 +21,8 @@ const (
 	DailyUserCount         string = "Daily User Count"
 	MixPanelDevToken       string = "YTAwYWI1ZWE3OTE2MzQwOWEwMjk4ZTA1NTNkNDQ0M2M="
 	MixPanelProdToken      string = "YWViNzUwYWRmYjM0YTZmZjJkMzg2YTYyYWVhY2M2NWI="
+	PerPage                int    = 20
+	Page                   int    = 1
 )
 
 type Tracker interface {
@@ -45,10 +47,10 @@ type Repo struct {
 }
 
 type Analytics struct {
-	Repo     *Repo
-	trackers analyticsMap
-	client   AnalyticsClient
-	host     string
+	Repo       *Repo
+	trackers   analyticsMap
+	client     AnalyticsClient
+	instanceID string
 }
 
 func newAnalytics(Repo *Repo, cfg config.Configuration) (*Analytics, error) {
@@ -58,7 +60,22 @@ func newAnalytics(Repo *Repo, cfg config.Configuration) (*Analytics, error) {
 	}
 
 	a := &Analytics{Repo: Repo, client: client}
-	a.host = cfg.Host
+
+	config, err := a.Repo.ConfigRepo.LoadConfiguration(context.Background())
+	if err != nil {
+		if errors.Is(err, datastore.ErrConfigNotFound) {
+			return nil, err
+		}
+
+		log.WithError(err).Error("failed to track metrics")
+		return nil, err
+	}
+
+	if !config.IsAnalyticsEnabled {
+		return nil, nil
+	}
+
+	a.instanceID = config.UID
 
 	a.RegisterTrackers()
 	return a, nil
@@ -68,7 +85,8 @@ func TrackDailyAnalytics(Repo *Repo, cfg config.Configuration) func(context.Cont
 	return func(ctx context.Context, t *asynq.Task) error {
 		a, err := newAnalytics(Repo, cfg)
 		if err != nil {
-			log.Fatal(err)
+			log.WithError(err).Error("Failed to initialize analytics")
+			return nil
 		}
 
 		a.trackDailyAnalytics()
@@ -78,19 +96,6 @@ func TrackDailyAnalytics(Repo *Repo, cfg config.Configuration) func(context.Cont
 }
 
 func (a *Analytics) trackDailyAnalytics() {
-	config, err := a.Repo.ConfigRepo.LoadConfiguration(context.Background())
-	if err != nil {
-		if errors.Is(err, datastore.ErrConfigNotFound) {
-			return
-		}
-
-		log.WithError(err).Error("failed to track metrics")
-	}
-
-	if !config.IsAnalyticsEnabled {
-		return
-	}
-
 	for _, tracker := range a.trackers {
 		go func(tracker Tracker) {
 			err := tracker.Track()
@@ -103,11 +108,11 @@ func (a *Analytics) trackDailyAnalytics() {
 
 func (a *Analytics) RegisterTrackers() {
 	a.trackers = analyticsMap{
-		DailyEventCount:        newEventAnalytics(a.Repo.EventRepo, a.Repo.GroupRepo, a.Repo.OrgRepo, a.client, a.host),
-		DailyOrganisationCount: newOrganisationAnalytics(a.Repo.OrgRepo, a.client, a.host),
-		DailyGroupCount:        newGroupAnalytics(a.Repo.GroupRepo, a.client, a.host),
-		DailyActiveGroupCount:  newActiveGroupAnalytics(a.Repo.GroupRepo, a.Repo.EventRepo, a.client, a.host),
-		DailyUserCount:         newUserAnalytics(a.Repo.UserRepo, a.client, a.host),
+		DailyEventCount:        newEventAnalytics(a.Repo.EventRepo, a.Repo.GroupRepo, a.Repo.OrgRepo, a.client, a.instanceID),
+		DailyOrganisationCount: newOrganisationAnalytics(a.Repo.OrgRepo, a.client, a.instanceID),
+		DailyGroupCount:        newGroupAnalytics(a.Repo.GroupRepo, a.client, a.instanceID),
+		DailyActiveGroupCount: newActiveGroupAnalytics(a.Repo.GroupRepo, a.Repo.EventRepo, a.Repo.OrgRepo, a.client, a.instanceID),
+		DailyUserCount:         newUserAnalytics(a.Repo.UserRepo, a.client, a.instanceID),
 	}
 
 }

analytics/event_analytics.go

@@ -2,57 +2,78 @@ package analytics
 
 import (
 	"context"
+	"time"
 
 	"github.com/frain-dev/convoy/datastore"
 	log "github.com/sirupsen/logrus"
 )
 
 type EventAnalytics struct {
-	eventRepo datastore.EventRepository
-	groupRepo datastore.GroupRepository
-	orgRepo   datastore.OrganisationRepository
-	client    AnalyticsClient
-	host      string
+	eventRepo  datastore.EventRepository
+	groupRepo  datastore.GroupRepository
+	orgRepo    datastore.OrganisationRepository
+	client     AnalyticsClient
+	instanceID string
 }
 
-func newEventAnalytics(eventRepo datastore.EventRepository, groupRepo datastore.GroupRepository, orgRepo datastore.OrganisationRepository, client AnalyticsClient, host string) *EventAnalytics {
+func newEventAnalytics(eventRepo datastore.EventRepository, groupRepo datastore.GroupRepository, orgRepo datastore.OrganisationRepository, client AnalyticsClient, instanceID string) *EventAnalytics {
 	return &EventAnalytics{
-		eventRepo: eventRepo,
-		groupRepo: groupRepo,
-		orgRepo:   orgRepo,
-		client:    client,
-		host:      host,
+		eventRepo:  eventRepo,
+		groupRepo:  groupRepo,
+		orgRepo:    orgRepo,
+		client:     client,
+		instanceID: instanceID,
 	}
 }
 
 func (ea *EventAnalytics) Track() error {
+	return ea.track(PerPage, Page)
+}
+
+func (ea *EventAnalytics) track(perPage, page int) error {
 	ctx := context.Background()
-	groups, err := ea.groupRepo.LoadGroups(ctx, &datastore.GroupFilter{})
+	orgs, _, err := ea.orgRepo.LoadOrganisationsPaged(ctx, datastore.Pageable{PerPage: perPage, Page: page, Sort: -1})
+
 	if err != nil {
 		return err
 	}
 
-	for _, group := range groups {
-		_, pagination, err := ea.eventRepo.LoadEventsPaged(ctx, group.UID, "", datastore.SearchParams{}, datastore.Pageable{Sort: -1})
-		if err != nil {
-			log.WithError(err).Error("failed to load events paged")
-			continue
-		}
+	if len(orgs) == 0 {
+		return nil
+	}
 
-		org, err := ea.orgRepo.FetchOrganisationByID(ctx, group.OrganisationID)
+	now := time.Now()
+	for _, org := range orgs {
+		groups, err := ea.groupRepo.LoadGroups(ctx, &datastore.GroupFilter{OrgID: org.UID})
 		if err != nil {
-			log.WithError(err).Error("failed to load fetch organisation")
+			log.WithError(err).Error("failed to load organisation groups")
 			continue
 		}
 
-		err = ea.client.Export(ea.Name(), Event{"Count": pagination.Total, "Project": group.Name, "Organization": org.Name, "Host": ea.host})
-		if err != nil {
-			log.WithError(err).Error("failed to load export metrics")
-			continue
+		for _, group := range groups {
+			filter := datastore.SearchParams{
+				CreatedAtStart: time.Unix(0, 0).Unix(),
+				CreatedAtEnd:   time.Date(now.Year(), now.Month(), now.Day(), 23, 59, 59, 999999999, time.UTC).Unix(),
+			}
+
+			_, pagination, err := ea.eventRepo.LoadEventsPaged(ctx, group.UID, "", filter, datastore.Pageable{PerPage: 20, Page: 1, Sort: -1})
+			if err != nil {
+				log.WithError(err).Error("failed to load events paged")
+				continue
+			}
+
+			err = ea.client.Export(ea.Name(), Event{"Count": pagination.Total, "Project": group.Name, "Organization": org.Name, "instanceID": ea.instanceID})
+			if err != nil {
+				log.WithError(err).Error("failed to load export metrics")
+				continue
+			}
 		}
 	}
 
-	return nil
+	perPage += 20
+	page += 1
+
+	return ea.track(perPage, page)
 }
 
 func (ea *EventAnalytics) Name() string {

analytics/event_analytics_test.go

@@ -10,15 +10,15 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-var TestHost = "http://test-host.com"
+var TestInstanceID = "3d9c49ce-8367-43ec-8884-568c8d43faec"
 
 func provideEventAnalytics(ctrl *gomock.Controller) *EventAnalytics {
 	eventRepo := mocks.NewMockEventRepository(ctrl)
 	groupRepo := mocks.NewMockGroupRepository(ctrl)
 	orgRepo := mocks.NewMockOrganisationRepository(ctrl)
 	client := NewNoopAnalyticsClient()
 
-	return newEventAnalytics(eventRepo, groupRepo, orgRepo, client, TestHost)
+	return newEventAnalytics(eventRepo, groupRepo, orgRepo, client, TestInstanceID)
 }
 
 func Test_TrackEventAnalytics(t *testing.T) {
@@ -34,17 +34,20 @@ func Test_TrackEventAnalytics(t *testing.T) {
 				groupRepo := ea.groupRepo.(*mocks.MockGroupRepository)
 				eventRepo := ea.eventRepo.(*mocks.MockEventRepository)
 				orgRepo := ea.orgRepo.(*mocks.MockOrganisationRepository)
-				groupRepo.EXPECT().LoadGroups(gomock.Any(), gomock.Any()).Return([]*datastore.Group{{UID: "123456", Name: "test-group"}}, nil)
-				eventRepo.EXPECT().LoadEventsPaged(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, datastore.PaginationData{}, nil)
-				orgRepo.EXPECT().FetchOrganisationByID(gomock.Any(), gomock.Any()).Return(&datastore.Organisation{UID: "123456", Name: "test-org"}, nil)
+				gomock.InOrder(
+					orgRepo.EXPECT().LoadOrganisationsPaged(gomock.Any(), gomock.Any()).Return([]datastore.Organisation{{UID: "123"}}, datastore.PaginationData{}, nil),
+					groupRepo.EXPECT().LoadGroups(gomock.Any(), gomock.Any()).Return([]*datastore.Group{{UID: "123456", Name: "test-group"}}, nil),
+					eventRepo.EXPECT().LoadEventsPaged(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, datastore.PaginationData{}, nil),
+					orgRepo.EXPECT().LoadOrganisationsPaged(gomock.Any(), gomock.Any()).Return(nil, datastore.PaginationData{}, nil),
+				)
 			},
 		},
 
 		{
 			name: "should_fail_to_track_event_analytics",
 			dbFn: func(ea *EventAnalytics) {
-				groupRepo := ea.groupRepo.(*mocks.MockGroupRepository)
-				groupRepo.EXPECT().LoadGroups(gomock.Any(), gomock.Any()).Return(nil, errors.New("failed"))
+				orgRepo := ea.orgRepo.(*mocks.MockOrganisationRepository)
+				orgRepo.EXPECT().LoadOrganisationsPaged(gomock.Any(), gomock.Any()).Return(nil, datastore.PaginationData{}, errors.New("failed"))
 			},
 			wantErr: true,
 		},