From 34c345b1b930283ed85d36e11096da125fcaec73 Mon Sep 17 00:00:00 2001
From: Yonas Habteab <yonas.habteab@icinga.com>
Date: Fri, 16 Jun 2023 14:17:45 +0200
Subject: [PATCH] Flatten empty custom vars of type `array` & `map` correctly

---
 pkg/flatten/flatten.go          | 23 ++++++++++++++++++++---
 pkg/icingadb/v1/customvar.go    | 15 ++++++---------
 schema/mysql/schema.sql         |  2 +-
 schema/mysql/upgrades/1.2.0.sql |  1 +
 schema/pgsql/schema.sql         |  2 +-
 schema/pgsql/upgrades/1.2.0.sql |  1 +
 6 files changed, 30 insertions(+), 14 deletions(-)
 create mode 100644 schema/mysql/upgrades/1.2.0.sql
 create mode 100644 schema/pgsql/upgrades/1.2.0.sql

diff --git a/pkg/flatten/flatten.go b/pkg/flatten/flatten.go
index a31705be6..688a3691d 100644
--- a/pkg/flatten/flatten.go
+++ b/pkg/flatten/flatten.go
@@ -1,26 +1,43 @@
 package flatten
 
 import (
+	"database/sql"
+	"fmt"
+	"github.com/icinga/icingadb/pkg/types"
 	"strconv"
 )
 
 // Flatten creates flat, one-dimensional maps from arbitrarily nested values, e.g. JSON.
-func Flatten(value interface{}, prefix string) map[string]interface{} {
+func Flatten(value interface{}, prefix string) map[string]types.String {
 	var flatten func(string, interface{})
-	flattened := make(map[string]interface{})
+	flattened := make(map[string]types.String)
 
 	flatten = func(key string, value interface{}) {
 		switch value := value.(type) {
 		case map[string]interface{}:
+			if len(value) == 0 {
+				flattened[key+"."] = types.String{}
+				break
+			}
+
 			for k, v := range value {
 				flatten(key+"."+k, v)
 			}
 		case []interface{}:
+			if len(value) == 0 {
+				flattened[key+"[]"] = types.String{}
+				break
+			}
+
 			for i, v := range value {
 				flatten(key+"["+strconv.Itoa(i)+"]", v)
 			}
 		default:
-			flattened[key] = value
+			val := fmt.Sprintf("%v", value)
+			if value == nil {
+				val = "null"
+			}
+			flattened[key] = types.String{NullString: sql.NullString{String: val, Valid: true}}
 		}
 	}
 
diff --git a/pkg/icingadb/v1/customvar.go b/pkg/icingadb/v1/customvar.go
index 0e85cc07e..462b87c4e 100644
--- a/pkg/icingadb/v1/customvar.go
+++ b/pkg/icingadb/v1/customvar.go
@@ -2,7 +2,6 @@ package v1
 
 import (
 	"context"
-	"fmt"
 	"github.com/icinga/icingadb/internal"
 	"github.com/icinga/icingadb/pkg/com"
 	"github.com/icinga/icingadb/pkg/contracts"
@@ -25,7 +24,7 @@ type CustomvarFlat struct {
 	CustomvarMeta    `json:",inline"`
 	Flatname         string       `json:"flatname"`
 	FlatnameChecksum types.Binary `json:"flatname_checksum"`
-	Flatvalue        string       `json:"flatvalue"`
+	Flatvalue        types.String `json:"flatvalue"`
 }
 
 func NewCustomvar() contracts.Entity {
@@ -117,11 +116,9 @@ func flattenCustomvars(ctx context.Context, g *errgroup.Group, cvs <-chan contra
 					flattened := flatten.Flatten(value, customvar.Name)
 
 					for flatname, flatvalue := range flattened {
-						var fv string
-						if flatvalue == nil {
-							fv = "null"
-						} else {
-							fv = fmt.Sprintf("%v", flatvalue)
+						var fv interface{}
+						if flatvalue.Valid {
+							fv = flatvalue.String
 						}
 
 						select {
@@ -131,7 +128,7 @@ func flattenCustomvars(ctx context.Context, g *errgroup.Group, cvs <-chan contra
 									IdMeta: IdMeta{
 										// TODO(el): Schema comment is wrong.
 										// Without customvar.Id we would produce duplicate keys here.
-										Id: utils.Checksum(objectpacker.MustPackSlice(customvar.EnvironmentId, customvar.Id, flatname, flatvalue)),
+										Id: utils.Checksum(objectpacker.MustPackSlice(customvar.EnvironmentId, customvar.Id, flatname, fv)),
 									},
 								},
 								EnvironmentMeta: EnvironmentMeta{
@@ -141,7 +138,7 @@ func flattenCustomvars(ctx context.Context, g *errgroup.Group, cvs <-chan contra
 							},
 							Flatname:         flatname,
 							FlatnameChecksum: utils.Checksum(flatname),
-							Flatvalue:        fv,
+							Flatvalue:        flatvalue,
 						}:
 						case <-ctx.Done():
 							return ctx.Err()
diff --git a/schema/mysql/schema.sql b/schema/mysql/schema.sql
index 36135c29f..467b810de 100644
--- a/schema/mysql/schema.sql
+++ b/schema/mysql/schema.sql
@@ -983,7 +983,7 @@ CREATE TABLE customvar_flat (
   flatname_checksum binary(20) NOT NULL COMMENT 'sha1(flatname after conversion)',
 
   flatname varchar(512) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Path converted with `.` and `[ ]`',
-  flatvalue text NOT NULL,
+  flatvalue text DEFAULT NULL,
 
   PRIMARY KEY (id),
 
diff --git a/schema/mysql/upgrades/1.2.0.sql b/schema/mysql/upgrades/1.2.0.sql
new file mode 100644
index 000000000..b91f9adec
--- /dev/null
+++ b/schema/mysql/upgrades/1.2.0.sql
@@ -0,0 +1 @@
+ALTER TABLE customvar_flat MODIFY COLUMN flatvalue text DEFAULT NULL;
diff --git a/schema/pgsql/schema.sql b/schema/pgsql/schema.sql
index af103a547..8253f4103 100644
--- a/schema/pgsql/schema.sql
+++ b/schema/pgsql/schema.sql
@@ -1553,7 +1553,7 @@ CREATE TABLE customvar_flat (
   flatname_checksum bytea20 NOT NULL,
 
   flatname citext NOT NULL,
-  flatvalue text NOT NULL,
+  flatvalue text NULL,
 
   CONSTRAINT pk_customvar_flat PRIMARY KEY (id)
 );
diff --git a/schema/pgsql/upgrades/1.2.0.sql b/schema/pgsql/upgrades/1.2.0.sql
new file mode 100644
index 000000000..abda26117
--- /dev/null
+++ b/schema/pgsql/upgrades/1.2.0.sql
@@ -0,0 +1 @@
+ALTER TABLE customvar_flat ALTER COLUMN flatvalue DROP NOT NULL;