310 added pre-built docker images

removed setting all api keys into destinations (if it hasn't)
added telemetry mappings type usage
swapped blacklist symbols replacer -> whitelist

configurator-builder.Dockerfile

@@ -0,0 +1,23 @@
+### JS/GO BUILDER
+FROM golang:1.16.3-alpine3.13
+
+# Install dependencies
+RUN apk add git make bash npm yarn
+
+# JS
+RUN mkdir /frontend
+WORKDIR /frontend
+
+# Install yarn dependencies
+ADD configurator/frontend/package.json configurator/frontend/yarn.lock /frontend/
+RUN yarn install --prefer-offline --frozen-lockfile --network-timeout 1000000
+
+# GO
+RUN mkdir -p /go/src/github.com/jitsucom/jitsu/configurator/backend && \
+    mkdir -p /go/src/github.com/jitsucom/jitsu/server
+
+WORKDIR /go/src/github.com/jitsucom/jitsu/configurator/backend
+
+ADD configurator/backend/go.mod configurator/backend/go.sum ./
+ADD server/go.mod server/go.sum /go/src/github.com/jitsucom/jitsu/server/
+RUN go mod download

configurator.Dockerfile

@@ -1,5 +1,5 @@
 # BASE STAGE
-FROM alpine:3.12 as main
+FROM alpine:3.13 as main
 
 ENV CONFIGURATOR_USER=configurator
 
@@ -14,41 +14,38 @@ RUN addgroup -S $CONFIGURATOR_USER \
 RUN ln -s /home/$CONFIGURATOR_USER/data/config /home/$CONFIGURATOR_USER/app/res && \
     ln -s /home/$CONFIGURATOR_USER/data/logs /home/$CONFIGURATOR_USER/logs && \
     chown -R $CONFIGURATOR_USER:$CONFIGURATOR_USER /home/$CONFIGURATOR_USER/logs
-
-#######################################
-# BUILD BASE STAGE
-FROM golang:1.14.6-alpine3.12 as builder
-
-# Install dependencies
-RUN apk add git make bash npm yarn
-
 #######################################
 # BUILD JS STAGE
-FROM builder as jsbuilder
-
+FROM jitsucom/configurator-builder as jsbuilder
 
 # Install yarn dependencies
-ADD configurator/frontend/package.json /app/package.json
+ADD configurator/frontend/package.json /frontend/package.json
 
 ARG SKIP_UI_BUILD
 ENV SKIP_UI=$SKIP_UI_BUILD
 
-WORKDIR /app
+WORKDIR /frontend
 
 # We need to make sure empty 'build' directory exists if SKIP_UI_BUILD==true and yarn won't make it
 RUN mkdir build
 
-RUN if [ "$SKIP_UI" != "true" ]; then yarn install --network-timeout 1000000; fi
+RUN if [ "$SKIP_UI" != "true" ]; then yarn install --prefer-offline --frozen-lockfile --network-timeout 1000000; fi
 
 # Copy project
 ADD configurator/frontend/. ./
 
+# write the output of the date command into a file called tmp_variable
+RUN free | awk 'FNR == 2 {print $2}' > ./build/mem
+
+# Check RAM > 4gb else error (JS build requires >4gb RAM)
+RUN if [ $(cat ./build/mem) < "4000000" ]; then echo echo Docker build requires 4gb of RAM. Configure it in the machine Docker configuration && exit 1; else rm ./build/mem; fi
+
 # Build
-RUN if [ "$SKIP_UI" != "true" ]; then yarn build --network-timeout 1000000; fi
+RUN if [ "$SKIP_UI" != "true" ]; then yarn docker_build; fi
 
 #######################################
 # BUILD BACKEND STAGE
-FROM builder as builder
+FROM jitsucom/configurator-builder as builder
 
 ENV CONFIGURATOR_USER=configurator
 
@@ -79,7 +76,7 @@ ENV TZ=UTC
 # copy static files from build-image
 COPY --from=builder /go/src/github.com/jitsucom/jitsu/$CONFIGURATOR_USER/backend/build/dist /home/$CONFIGURATOR_USER/app
 
-COPY --from=jsbuilder /app/build /home/$CONFIGURATOR_USER/app/web
+COPY --from=jsbuilder /frontend/build /home/$CONFIGURATOR_USER/app/web
 
 RUN chown -R $CONFIGURATOR_USER:$CONFIGURATOR_USER /home/$CONFIGURATOR_USER/app
 

configurator/backend/destinations/config_mapper.go

@@ -35,6 +35,7 @@ func MapConfig(destinationID string, destination *entities.Destination, defaultS
 	if err != nil {
 		return nil, err
 	}
+
 	enrichMappingRules(destination, config)
 	setEnrichmentRules(destination, config)
 
@@ -53,10 +54,11 @@ func MapConfig(destinationID string, destination *entities.Destination, defaultS
 			AnonymousIDNode: destination.UsersRecognition.AnonymousIDNode,
 			UserIDNode:      destination.UsersRecognition.UserIDJSONNode,
 		}
-	} else {
-		config.UsersRecognition = &enstorages.UsersRecognition{Enabled: false}
 	}
 
+	//only keys
+	config.OnlyTokens = destination.OnlyKeys
+
 	return config, nil
 }
 
@@ -280,34 +282,26 @@ func mapFacebook(fbDestination *entities.Destination) (*enstorages.DestinationCo
 
 func enrichMappingRules(destination *entities.Destination, enDestinationConfig *enstorages.DestinationConfig) {
 	if !destination.Mappings.IsEmpty() {
-		var rules []string
+		var mappingFields []schema.MappingField
 		for _, rule := range destination.Mappings.Rules {
-			var cast string
-			switch rule.Action {
-			case "move", "erase":
-				cast = ""
-			case "cast/int":
-				cast = "(integer) "
-			case "cast/double":
-				cast = "(double) "
-			case "cast/date":
-				cast = "(timestamp) "
-			case "cast/string":
-				cast = "(string) "
-			}
-			rules = append(rules, rule.SourceField+" -> "+cast+rule.DestinationField)
+			mappingFields = append(mappingFields, schema.MappingField{
+				Src:        rule.SourceField,
+				Dst:        rule.DestinationField,
+				Action:     rule.Action,
+				Type:       rule.Type,
+				ColumnType: rule.ColumnType,
+				Value:      rule.Value,
+			})
 		}
 
 		if enDestinationConfig.DataLayout == nil {
 			enDestinationConfig.DataLayout = &enstorages.DataLayout{}
 		}
 
-		enDestinationConfig.DataLayout.Mapping = rules
-		mappingType := schema.Default
-		if !destination.Mappings.KeepFields {
-			mappingType = schema.Strict
+		enDestinationConfig.DataLayout.Mappings = &schema.Mapping{
+			KeepUnmapped: &destination.Mappings.KeepFields,
+			Fields:       mappingFields,
 		}
-		enDestinationConfig.DataLayout.MappingType = mappingType
 	}
 }
 

configurator/backend/entities/destination.go

@@ -40,7 +40,10 @@ func (m *Mappings) IsEmpty() bool {
 
 //MapRule entity is stored in main storage (Firebase or Redis)
 type MapRule struct {
-	Action           string `firestore:"_action" json:"_action"`
-	SourceField      string `firestore:"_srcField" json:"_srcField"`
-	DestinationField string `firestore:"_dstField" json:"_dstField"`
+	Action           string      `firestore:"_action" json:"_action"`
+	SourceField      string      `firestore:"_srcField" json:"_srcField"`
+	DestinationField string      `firestore:"_dstField" json:"_dstField"`
+	Type             string      `firestore:"_type" json:"_type"`
+	ColumnType       string      `firestore:"_columnType" json:"_columnType"`
+	Value            interface{} `firestore:"_value" json:"_value"`
 }

configurator/backend/go.mod

@@ -1,6 +1,6 @@
 module github.com/jitsucom/jitsu/configurator
 
-go 1.14
+go 1.16
 
 require (
 	cloud.google.com/go/firestore v1.3.0

configurator/backend/handlers/destinations.go

@@ -43,25 +43,11 @@ func (dh *DestinationsHandler) GetHandler(c *gin.Context) {
 	}
 
 	idConfig := map[string]enstorages.DestinationConfig{}
-	keysByProject, err := dh.configurationsService.GetAPIKeysGroupByProjectID()
-	if err != nil {
-		logging.Errorf("Error getting api keys grouped by project id. All destinations will be skipped: %v", err)
-		c.JSON(http.StatusInternalServerError, enmiddleware.ErrorResponse{Error: err.Error(), Message: "Failed to get API keys"})
-		return
-	}
 	for projectID, destinationsEntity := range destinationsMap {
 		if len(destinationsEntity.Destinations) == 0 {
 			continue
 		}
 
-		//if only tokens empty - put all tokens by project
-		keys, _ := keysByProject[projectID]
-
-		projectTokenIDs := []string{}
-		for _, k := range keys {
-			projectTokenIDs = append(projectTokenIDs, k.ID)
-		}
-
 		for _, destination := range destinationsEntity.Destinations {
 			destinationID := projectID + "." + destination.UID
 			enDestinationConfig, err := destinations.MapConfig(destinationID, destination, dh.defaultS3)
@@ -70,11 +56,6 @@ func (dh *DestinationsHandler) GetHandler(c *gin.Context) {
 				continue
 			}
 
-			if len(destination.OnlyKeys) > 0 {
-				enDestinationConfig.OnlyTokens = destination.OnlyKeys
-			} else {
-				enDestinationConfig.OnlyTokens = projectTokenIDs
-			}
 			idConfig[destinationID] = *enDestinationConfig
 		}
 	}

configurator/frontend/package.json

@@ -39,6 +39,7 @@
     "clean": "rm -rf ./node_modules/ build dist",
     "start": "NODE_ENV=development craco start",
     "build": "CI=false NODE_ENV=production craco build",
+    "docker_build": "CI=false NODE_ENV=production ANALYTICS_KEYS='{\"eventnative\": \"js.gpon6lmpwquappfl07tuq.ka5sxhsm08cmblny72tevi\"}' craco build",
     "test": "craco test",
     "eslint:check": "eslint . --ext .tx,.tsx --ignore-pattern node_modules/",
     "eslint:fix": "eslint . --ext .tx,.tsx --ignore-pattern node_modules/ --fix"

configurator/frontend/src/lib/components/ApiKeys/ApiKeys.tsx

@@ -85,7 +85,7 @@ export default class ApiKeys extends LoadableComponent<{}, State> {
             };
             let newTokens = [...this.state.tokens, newToken];
             await this.saveTokens(newTokens, 'NEW');
-            message.info('New token has been saved!');
+            message.info('New API key has been saved!');
           }}
         >
           Generate New Key

go.mod

@@ -1,6 +1,6 @@
 module github.com/jitsucom/jitsu
 
-go 1.14
+go 1.16
 
 require (
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect

server-builder.Dockerfile

@@ -0,0 +1,21 @@
+### JS/GO BUILDER
+FROM golang:1.16.3-alpine3.13
+
+# Install dependencies
+RUN apk add git make bash npm yarn
+
+# JS
+RUN mkdir /javascript-sdk
+WORKDIR /javascript-sdk
+
+# Install yarn dependencies
+ADD javascript-sdk/package.json javascript-sdk/yarn.lock ./
+RUN yarn install --prefer-offline --frozen-lockfile --network-timeout 1000000
+
+# GO
+RUN mkdir -p /go/src/github.com/jitsucom/jitsu/server
+
+WORKDIR /go/src/github.com/jitsucom/jitsu/server
+
+ADD server/go.mod server/go.sum /go/src/github.com/jitsucom/jitsu/server/
+RUN go mod download

server.Dockerfile

@@ -1,5 +1,5 @@
 # BASE STAGE
-FROM alpine:3.12 as main
+FROM alpine:3.13 as main
 
 RUN apk add --no-cache build-base python3 py3-pip python3-dev tzdata
 
@@ -21,16 +21,9 @@ RUN ln -s /home/$EVENTNATIVE_USER/data/config /home/$EVENTNATIVE_USER/app/res &&
     ln -s /home/$EVENTNATIVE_USER/data/logs /home/$EVENTNATIVE_USER/logs && \
     chown -R $EVENTNATIVE_USER:$EVENTNATIVE_USER /home/$EVENTNATIVE_USER/logs
 
-#######################################
-# BUILD BASE STAGE
-FROM golang:1.14.6-alpine3.12 as builder
-
-# Install dependencies
-RUN apk add git make bash npm yarn build-base
-
 #######################################
 # BUILD JS STAGE
-FROM builder as jsbuilder
+FROM jitsucom/server-builder as jsbuilder
 
 RUN mkdir /app
 
@@ -45,16 +38,15 @@ RUN make clean_js js assemble_js &&\
 
 #######################################
 # BUILD JS SDK STAGE
-FROM builder as jsSdkbuilder
+FROM jitsucom/server-builder as jsSdkbuilder
 
-RUN mkdir /javascript-sdk && \
-    mkdir /app
+RUN mkdir /app
 
 WORKDIR /javascript-sdk
 
 # Install npm dependencies
-ADD javascript-sdk/package.json ./package.json
-RUN yarn install --network-timeout 1000000
+ADD javascript-sdk/package.json javascript-sdk/yarn.lock ./
+RUN yarn install --prefer-offline --frozen-lockfile --network-timeout 1000000
 
 # Copy project
 ADD javascript-sdk/. .
@@ -66,20 +58,16 @@ RUN yarn build && \
 
 #######################################
 # BUILD BACKEND STAGE
-FROM builder as builder
+FROM jitsucom/server-builder as builder
 
 RUN mkdir /app
 
-RUN mkdir -p /go/src/github.com/jitsucom/jitsu/jitsu/server
-
-WORKDIR /go/src/github.com/jitsucom/jitsu/jitsu/server
+WORKDIR /go/src/github.com/jitsucom/jitsu/server
 
 #Caching dependencies
 ADD server/go.mod server/go.sum ./
 RUN go mod download
 
-#######
-
 #Copy backend
 ADD server/. ./.
 ADD .git ./.git
@@ -108,4 +96,4 @@ USER $EVENTNATIVE_USER
 VOLUME ["/home/$EVENTNATIVE_USER/data"]
 EXPOSE 8001
 
-ENTRYPOINT ["./eventnative", "-cfg=../data/config/eventnative.yaml", "-cr=true"]
+ENTRYPOINT ["./eventnative", "-cfg=../data/config/eventnative.yaml", "-cr=true"]