Sammyteillet/dat 50 aau i see a drift only for past data (#25)

* refactor: rename folder notion_database

* feature: filter data after first day only for drift

* refactor: remove some comments

charts/charts.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"sort"
 	"strings"
 	"time"
 
@@ -19,6 +18,13 @@ type ChartResponse struct {
 	URL     string `json:"url"`
 }
 
+type CommitData struct {
+	Lines           int
+	KPI             decimal.Decimal
+	CommitTimestamp int64
+	CommitUrl       string
+}
+
 func ProcessCharts(historyFilepath string, metric common.Metric) []common.KPIReport {
 
 	data, err := getKeysFromJSON(historyFilepath)
@@ -47,31 +53,23 @@ func OrderDataAndCreateChart(KPIName string, periodId string, unsortedResults ma
 	CommitUrl       string
 }) common.KPIReport {
 	// Extract the values from the map into a slice of struct objects
-	var dataSortableArray []struct {
-		Lines           int
-		KPI             decimal.Decimal
-		CommitTimestamp int64
-		CommitUrl       string
-	}
+	var dataSortableArray []CommitData
+
 	for _, stats := range unsortedResults {
 		KPI, _ := decimal.NewFromString(stats.KPI)
-		dataSortableArray = append(dataSortableArray, struct {
-			Lines           int
-			KPI             decimal.Decimal
-			CommitTimestamp int64
-			CommitUrl       string
-		}{
+		dataSortableArray = append(dataSortableArray, CommitData{
 			Lines:           stats.Lines,
 			KPI:             KPI,
 			CommitTimestamp: stats.CommitTimestamp,
 			CommitUrl:       stats.CommitUrl,
 		})
 	}
 
-	// Sort the slice by CommitTimestamp
-	sort.Slice(dataSortableArray, func(i, j int) bool {
-		return dataSortableArray[i].CommitTimestamp < dataSortableArray[j].CommitTimestamp
-	})
+	sortedAndFilteredArray := FilterAndSortByCommitTimestamp(dataSortableArray, getFirstDateOfPeriod(periodId))
+
+	if len(sortedAndFilteredArray) == 0 {
+		return common.KPIReport{}
+	}
 
 	var diff []interface{}
 	var labels []interface{}
@@ -80,12 +78,12 @@ func OrderDataAndCreateChart(KPIName string, periodId string, unsortedResults ma
 	upcolor := "rgb(82 156 202)"
 	downcolor := "rgb(255 163 68)"
 	var prevKPI decimal.Decimal
-	initialValue := dataSortableArray[0].KPI
-	latestValue := dataSortableArray[len(dataSortableArray)-1].KPI
+	initialValue := sortedAndFilteredArray[0].KPI
+	latestValue := sortedAndFilteredArray[len(sortedAndFilteredArray)-1].KPI
 	var events []common.EventObject
 	minOfChart := 0
 
-	for _, v := range dataSortableArray {
+	for _, v := range sortedAndFilteredArray {
 		roundedKPI := v.KPI
 		// TODO
 		roundedMin := 32000

charts/sort_filter_events.go

@@ -0,0 +1,53 @@
+package charts
+
+import (
+	"log"
+	"sort"
+	"time"
+
+	"github.com/data-drift/kpi-git-history/common"
+	"github.com/data-drift/kpi-git-history/reports"
+)
+
+func FilterAndSortByCommitTimestamp(dataSortableArray []CommitData, driftDay time.Time) []CommitData {
+	filteredArray := make([]CommitData, 0, len(dataSortableArray))
+	for i := range dataSortableArray {
+		timestamp := time.Unix(dataSortableArray[i].CommitTimestamp, 0)
+		if timestamp.After(driftDay) {
+			filteredArray = append(filteredArray, dataSortableArray[i])
+		}
+	}
+
+	sort.Slice(filteredArray, func(i, j int) bool {
+		return filteredArray[i].CommitTimestamp < filteredArray[j].CommitTimestamp
+	})
+
+	return filteredArray
+}
+
+func getFirstDateOfPeriod(periodKey string) time.Time {
+	timegrain, _ := reports.GetTimeGrain(periodKey)
+	var lastDay time.Time
+	switch timegrain {
+	case common.Day:
+		lastDay, _ = time.Parse("2006-01-02", periodKey)
+	case common.Week:
+		periodTime, _ := time.Parse("2006-W01", periodKey)
+		lastDay = periodTime.AddDate(0, 0, (7 - int(periodTime.Weekday()))).Add(time.Duration(23)*time.Hour + time.Duration(59)*time.Minute + time.Duration(59)*time.Second)
+	case common.Month:
+		periodTime, _ := time.Parse("2006-01", periodKey)
+
+		lastDay = periodTime.AddDate(0, 1, -1).Add(time.Duration(23)*time.Hour + time.Duration(59)*time.Minute + time.Duration(59)*time.Second)
+	case common.Quarter:
+		periodTime, _ := reports.ParseQuarterDate(periodKey)
+
+		lastDay = periodTime
+	case common.Year:
+		periodTime, _ := time.Parse("2006", periodKey)
+		lastDay = time.Date(periodTime.Year(), 12, 31, 23, 59, 59, 0, time.UTC)
+	default:
+		log.Fatalf("Invalid time grain: %s", timegrain)
+	}
+	return lastDay
+
+}

debug/debug.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/data-drift/kpi-git-history/charts"
 	"github.com/data-drift/kpi-git-history/common"
+	"github.com/data-drift/kpi-git-history/database/notion_database"
 	"github.com/data-drift/kpi-git-history/github"
 	"github.com/data-drift/kpi-git-history/history"
 	"github.com/data-drift/kpi-git-history/reports"
@@ -29,6 +30,8 @@ func DebugFunction() {
 	filepath := os.Getenv("DEFAULT_FILE_PATH")
 	githubApplicationId, _ := strconv.ParseInt(githubApplicationIdStr, 10, 64)
 
+	_ = notion_database.AssertDatabaseHasDatadriftProperties(notionDatabaseID, notionAPIKey)
+
 	client, _ := github.CreateClientFromGithubApp(int64(githubApplicationId))
 	ctx := context.Background()
 
@@ -45,7 +48,7 @@ func DebugFunction() {
 			KPIColumnName:  kpiColumn,
 			DateColumnName: dateColumn,
 			Filepath:       githubRepoFilePath,
-			TimeGrains:     []common.TimeGrain{common.Month, common.Year},
+			TimeGrains:     []common.TimeGrain{common.Quarter, common.Year},
 			Dimensions:     []string{},
 		})
 
@@ -62,7 +65,7 @@ func DebugFunction() {
 	// 	return
 	// }
 
-	for _, chartResult := range chartResults[:1] {
+	for _, chartResult := range chartResults {
 		err := reports.CreateReport(common.SyncConfig{NotionAPIKey: notionAPIKey, NotionDatabaseID: notionDatabaseID}, chartResult)
 		if err != nil {
 			println(err)

github/webhook.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/data-drift/kpi-git-history/charts"
 	"github.com/data-drift/kpi-git-history/common"
-	notion_database "github.com/data-drift/kpi-git-history/database/notion"
+	"github.com/data-drift/kpi-git-history/database/notion_database"
 	"github.com/data-drift/kpi-git-history/history"
 	"github.com/data-drift/kpi-git-history/reports"
 	"github.com/gin-gonic/gin"

reports/reports.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/data-drift/kpi-git-history/common"
-	notion_database "github.com/data-drift/kpi-git-history/database/notion"
+	"github.com/data-drift/kpi-git-history/database/notion_database"
 	"github.com/dstotijn/go-notion"
 	"github.com/shopspring/decimal"
 )