-
Notifications
You must be signed in to change notification settings - Fork 0
/
gcf_analytics.go
104 lines (85 loc) · 3.7 KB
/
gcf_analytics.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
// Package gcf_analytics google cloud function is triggered by upload to illuminating_upload_json_bucket_output by gcf_interestcal cloud
// function and starts appending the calculations to BigQuery `delta_calculations` table in `gcfdeltaanalytics` table.
// Scope: Analytics triggered by a Submit of Interest Request only supported currently.
// Currently,
// * This appends a row in BigQuery for Interest Request Submitted. There is also env variable for query only mode to bypass this fpr development mode.
// * Based on the interest request submitted, this does query on BigQuery for delta calculations overall for last 5 requests to
// compare the delta calculations. This result is placed in Analytics struct
// * Then it also Publishes as JSON the analytics message to PubSub.
//
// Depends on gcf_interestcal cloud function deployed and having run successfully. See System Diagram for more details.
//nolint:revive,stylecheck // using underscore in package name for clarity
package gcf_analytics
import (
"context"
"fmt"
"log"
"os"
"strconv"
"time"
"github.com/GoogleCloudPlatform/functions-framework-go/functions"
"github.com/cloudevents/sdk-go/v2/event"
)
// required as per google cloud functions version 2
// https://cloud.google.com/functions/docs/writing/write-http-functions
//
//nolint:gochecknoinits
func init() {
functions.CloudEvent("DeltaCalAnalytics", deltaCalAnalytics)
}
// StorageObjectData contains metadata of the Cloud Storage object.
type StorageObjectData struct {
Bucket string `json:"bucket,omitempty"`
Name string `json:"name,omitempty"`
Metageneration int64 `json:"metageneration,string,omitempty"`
TimeCreated time.Time `json:"timeCreated,omitempty"`
Updated time.Time `json:"updated,omitempty"`
}
// analyzeDeltaStorage consumes a CloudEvent message for Interest calculation with Delta uploaded to output bucket.
func deltaCalAnalytics(ctx context.Context, evt event.Event) error {
log.Println("deltaCalAnalytics Triggered: event triggered by output storage bucket file")
log.Printf("Event ID: %s", evt.ID())
log.Printf("Event Type: %s", evt.Type())
var data StorageObjectData
if err := evt.DataAs(&data); err != nil {
return fmt.Errorf("event.DataAs: %w", err)
}
log.Printf("Bucket: %s", data.Bucket)
log.Printf("File: %s", data.Name)
// log.Printf("Metageneration: %d", data.Metageneration)
// log.Printf("Created: %s", data.TimeCreated)
// log.Printf("Updated: %s", data.Updated)
log.Println("ctx is", ctx)
log.Println("QUERY_MODE_ONLY_DEVELOPMENT is", os.Getenv("QUERY_MODE_ONLY_DEVELOPMENT"))
// if queryOnlyMode is true, then no data will be imported to big query
// only query will be executed and published to pubsub
queryOnlyMode, err := strconv.ParseBool(os.Getenv("QUERY_MODE_ONLY_DEVELOPMENT"))
if err != nil {
return fmt.Errorf("strconv.ParseBool for QUERY_MODE_ONLY_DEVELOPMENT : %w", err)
}
log.Println("queryOnlyMode bool got is", queryOnlyMode)
// read the file from the bucket
if err := readAndAppendToBigQuery(ctx, queryOnlyMode, data); err != nil {
return fmt.Errorf("readAndAppendToBigQuery: %w", err)
}
if err := queryAndPublishAnalytics(ctx); err != nil {
return fmt.Errorf("queryAndPublishAnalytics: %w", err)
}
return nil
}
func readAndAppendToBigQuery(ctx context.Context, queryOnlyMode bool, data StorageObjectData) error {
if queryOnlyMode {
return nil
}
respData, err := readObjectFromBucket(ctx, data.Bucket, data.Name)
if err != nil {
return fmt.Errorf("readObjectFromBucket: %w", err)
}
log.Printf("File Data read is: %v", string(respData))
err = appendCal(ctx, respData)
if err != nil {
log.Println("import data to big query error is", err)
return fmt.Errorf("appendCal err : %w", err)
}
return nil
}