Fix stuff here and there (#158)

* Fix stuff here and there * Fix eslint issue * Delete unused report * Fix responsiveness of report cards * Fix eslint issues
th0th · Feb 22, 2023 · 2394334 · 2394334
1 parent 6470ef1
commit 2394334
Show file tree

Hide file tree

Showing 45 changed files with 249 additions and 481 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -146,7 +146,7 @@ jobs:
           path: ~/.npm
 
       - name: Pull docker images
-        run: docker compose pull clickhouse nginx-proxy postgres redis
+        run: docker compose pull --ignore-buildable
 
       - env:
           DEBUG: ${{ vars.DEBUG }}

diff --git a/backend/migrations/clickhouse/000001_create_tables.up.sql b/backend/migrations/clickhouse/000001_create_tables.up.sql
@@ -12,7 +12,6 @@ CREATE TABLE "events" (
   "locale" Nullable(String),
   "operating_system_name" Nullable(String),
   "operating_system_version" Nullable(String),
-  "path" String,
   "referrer" Nullable(String),
   "site_id" UInt64,
   "time_zone" Nullable(String),

diff --git a/backend/pkg/model/event.go b/backend/pkg/model/event.go
@@ -29,7 +29,6 @@ type Event struct {
 	Locale                 *string
 	OperatingSystemName    *string
 	OperatingSystemVersion *string
-	Path                   string
 	Referrer               *string
 	SiteId                 uint64
 	TimeZone               *string
@@ -55,11 +54,6 @@ const (
 	EventKindPageView EventKind = "PAGE_VIEW"
 )
 
-const (
-	EventUserAgentMaxLength = 2000
-	EventUserAgentMinLength = 2
-)
-
 func (e *Event) TableName() string {
 	return "events_buffer"
 }
@@ -104,8 +98,6 @@ func (e *Event) FillFromUrl(url string, safeQueryParameters []string) {
 
 	u.RawQuery = q.Encode()
 	e.Url = u.String()
-
-	e.Path = u.Path
 }
 
 func (e *Event) FillFromUserAgent(userAgent string) {

diff --git a/backend/pkg/restapi/sitereport/device_type.go b/backend/pkg/restapi/sitereport/device_type.go
@@ -3,13 +3,13 @@ package sitereport
 import (
 	"github.com/gofiber/fiber/v2"
 	dm "github.com/th0th/poeticmetric/backend/pkg/restapi/middleware/depot"
-	"github.com/th0th/poeticmetric/backend/pkg/service/sitedevicetypereport"
+	"github.com/th0th/poeticmetric/backend/pkg/service/sitereport/devicetype"
 )
 
 func deviceType(c *fiber.Ctx) error {
 	dp := dm.Get(c)
 
-	report, err := sitedevicetypereport.Get(dp, getFilters(c))
+	report, err := devicetype.Get(dp, getFilters(c))
 	if err != nil {
 		return err
 	}

diff --git a/backend/pkg/restapi/sitereport/main.go b/backend/pkg/restapi/sitereport/main.go
@@ -43,7 +43,6 @@ func Add(app *fiber.App) {
 	group.Get("/overview", overview)
 	group.Get("/page-view", pageView)
 	group.Get("/page-view-trends", pageViewTrends)
-	group.Get("/path-duration", pathDuration)
 	group.Get("/path", paginationCursorMiddleware[path2.PaginationCursor], path)
 	group.Get("/referrer-path", paginationCursorMiddleware[referrerpath.PaginationCursor], referrerPath)
 	group.Get("/referrer-site", paginationCursorMiddleware[referrersite.PaginationCursor], referrerSite)
@@ -53,7 +52,6 @@ func Add(app *fiber.App) {
 	group.Get("/utm-source", paginationCursorMiddleware[utmsource.PaginationCursor], utmSource)
 	group.Get("/utm-term", paginationCursorMiddleware[utmterm.PaginationCursor], utmTerm)
 	group.Get("/visitor", visitor)
-	group.Get("/visitor-page-view", visitorPageView)
 	group.Get("/visitor-trends", visitorTrends)
 }
 

diff --git a/backend/pkg/restapi/sitereport/overview.go b/backend/pkg/restapi/sitereport/overview.go
@@ -3,13 +3,13 @@ package sitereport
 import (
 	"github.com/gofiber/fiber/v2"
 	dm "github.com/th0th/poeticmetric/backend/pkg/restapi/middleware/depot"
-	"github.com/th0th/poeticmetric/backend/pkg/service/siteoverviewreport"
+	overview2 "github.com/th0th/poeticmetric/backend/pkg/service/sitereport/overview"
 )
 
 func overview(c *fiber.Ctx) error {
 	dp := dm.Get(c)
 
-	report, err := siteoverviewreport.Get(dp, getFilters(c))
+	report, err := overview2.Get(dp, getFilters(c))
 	if err != nil {
 		return err
 	}

diff --git a/backend/pkg/restapi/sitereport/page_view_trends.go b/backend/pkg/restapi/sitereport/page_view_trends.go
@@ -3,13 +3,13 @@ package sitereport
 import (
 	"github.com/gofiber/fiber/v2"
 	dm "github.com/th0th/poeticmetric/backend/pkg/restapi/middleware/depot"
-	"github.com/th0th/poeticmetric/backend/pkg/service/sitepageviewtrends"
+	"github.com/th0th/poeticmetric/backend/pkg/service/sitereport/pageviewtrends"
 )
 
 func pageViewTrends(c *fiber.Ctx) error {
 	dp := dm.Get(c)
 
-	report, err := sitepageviewtrends.Get(dp, getFilters(c))
+	report, err := pageviewtrends.Get(dp, getFilters(c))
 	if err != nil {
 		return err
 	}

diff --git a/backend/pkg/restapi/sitereport/path_duration.go b/backend/pkg/restapi/sitereport/path_duration.go
diff --git a/backend/pkg/restapi/sitereport/visitor_page_view.go b/backend/pkg/restapi/sitereport/visitor_page_view.go
diff --git a/backend/pkg/service/bootstrap/create_site.go b/backend/pkg/service/bootstrap/create_site.go
@@ -81,6 +81,11 @@ func createSite(dp *depot.Depot) error {
 				event.Referrer = pointer.Get(fmt.Sprintf("%s%s", gofakeit.RandomString(referrerSites), gofakeit.RandomString(referrerPaths)))
 			}
 
+			event.FillFromUrl(gofakeit.RandomString(urls), nil)
+			event.FillFromUserAgent(gofakeit.RandomString(userAgents))
+
+			event.VisitorId = gofakeit.RandomString(visitorIds)
+
 			//lint:ignore SA4000 we use double Bool() here to reduce the possibility
 			if gofakeit.Bool() && gofakeit.Bool() {
 				event.UtmSource = pointer.Get(gofakeit.RandomString(utmSources))
@@ -90,11 +95,6 @@ func createSite(dp *depot.Depot) error {
 				event.UtmTerm = pointer.Get(gofakeit.RandomString(utmTerms))
 			}
 
-			event.FillFromUrl(gofakeit.RandomString(urls), nil)
-			event.FillFromUserAgent(gofakeit.RandomString(userAgents))
-
-			event.VisitorId = gofakeit.RandomString(visitorIds)
-
 			events = append(events, event)
 		}
 

diff --git a/backend/pkg/service/sitepathdurationreport/main.go b/backend/pkg/service/sitepathdurationreport/main.go
diff --git a/backend/pkg/service/sitepathdurationreport/main_test.go b/backend/pkg/service/sitepathdurationreport/main_test.go
diff --git a/.../pkg/service/sitedevicetypereport/main.go → ...pkg/service/sitereport/devicetype/main.go b/.../pkg/service/sitedevicetypereport/main.go → ...pkg/service/sitereport/devicetype/main.go
@@ -1,4 +1,4 @@
-package sitedevicetypereport
+package devicetype
 
 import (
 	"strings"

diff --git a/backend/pkg/service/sitereport/filter/main.go b/backend/pkg/service/sitereport/filter/main.go
@@ -71,7 +71,7 @@ func Apply(dp *depot.Depot, filters *Filters) *gorm.DB {
 	}
 
 	if filters.Path != nil {
-		clickHouseSession.Where("path = ?", *filters.Path)
+		clickHouseSession.Where("pathFull(url) = ?", *filters.Path)
 	}
 
 	if filters.Referrer != nil {

diff --git a/...nd/pkg/service/siteoverviewreport/main.go → ...d/pkg/service/sitereport/overview/main.go b/...nd/pkg/service/siteoverviewreport/main.go → ...d/pkg/service/sitereport/overview/main.go
@@ -1,4 +1,4 @@
-package siteoverviewreport
+package overview
 
 import (
 	"context"

diff --git a/backend/pkg/service/sitereport/pageview/main.go b/backend/pkg/service/sitereport/pageview/main.go
@@ -13,7 +13,7 @@ import (
 
 type Datum struct {
 	DateTime      time.Time `json:"dateTime"`
-	PageViewCount uint64    `json:"pageViewCount"`
+	PageViewCount *uint64   `json:"pageViewCount"`
 }
 
 type Report struct {
@@ -23,9 +23,9 @@ type Report struct {
 }
 
 func Get(dp *depot.Depot, filters *filter.Filters) (*Report, error) {
-	interval := interval.GetVisitorPageViewInterval(filters)
+	interval2 := interval.GetVisitorPageViewInterval(filters)
 	report := &Report{
-		Interval: interval,
+		Interval: interval2,
 	}
 
 	q := filter.Apply(dp, filters)
@@ -37,7 +37,7 @@ func Get(dp *depot.Depot, filters *filter.Filters) (*Report, error) {
 				"count(*) as page_view_count",
 			}, ","),
 			map[string]interface{}{
-				"timeWindowInterval": gorm.Expr(interval.ToQuery()),
+				"timeWindowInterval": gorm.Expr(interval2.ToQuery()),
 				"timeZone":           filters.GetTimeZone(),
 			},
 		).
@@ -49,12 +49,12 @@ func Get(dp *depot.Depot, filters *filter.Filters) (*Report, error) {
 				"select",
 				strings.Join([]string{
 					"@start + interval arrayJoin(range(0, toUInt64(dateDiff('second', @start, @end)), @intervalSeconds)) second as date_time",
-					"0 as page_view_count",
+					"if(date_time > now(), null, 0) as page_view_count",
 				}, ","),
 			}, " "),
 			map[string]any{
 				"end":             filters.End,
-				"intervalSeconds": interval.ToDuration().Seconds(),
+				"intervalSeconds": interval2.ToDuration().Seconds(),
 				"start":           filters.Start,
 			},
 		)
@@ -78,8 +78,10 @@ func Get(dp *depot.Depot, filters *filter.Filters) (*Report, error) {
 	var pageViewCountsLength float64 = 0
 
 	for _, d := range report.Data {
-		pageViewCountsSum += float64(d.PageViewCount)
-		pageViewCountsLength += 1
+		if d.PageViewCount != nil {
+			pageViewCountsSum += float64(*d.PageViewCount)
+			pageViewCountsLength += 1
+		}
 	}
 
 	if pageViewCountsLength != 0 {

diff --git a/...nd/pkg/service/sitepageviewtrends/main.go → ...service/sitereport/pageviewtrends/main.go b/...nd/pkg/service/sitepageviewtrends/main.go → ...service/sitereport/pageviewtrends/main.go
@@ -1,4 +1,4 @@
-package sitepageviewtrends
+package pageviewtrends
 
 import (
 	"strings"

diff --git a/...g/service/sitepageviewtrends/main_test.go → ...ce/sitereport/pageviewtrends/main_test.go b/...g/service/sitepageviewtrends/main_test.go → ...ce/sitereport/pageviewtrends/main_test.go
@@ -1,4 +1,4 @@
-package sitepageviewtrends
+package pageviewtrends
 
 import (
 	"fmt"