Main

cmd/logConsumer/main.go

@@ -152,7 +152,7 @@ func indexExists(client *logharbour.ElasticsearchClient, indexName string) (bool
 }
 
 func createIndexWithMapping(client *logharbour.ElasticsearchClient, indexName string) error {
-	err := client.CreateIndex(indexName, esLogsMapping)
+	err := client.CreateIndex(indexName, logharbour.ESLogsMapping)
 	if err != nil {
 		return fmt.Errorf("failed to create index: %v", err)
 	}

logharbour/clientFn.go

@@ -36,7 +36,7 @@ const (
 	DIALTIMEOUT = 500 * time.Second
 	ACTIVITY    = "A"
 	DEBUG       = "D"
-	field       = "data.changes.field"
+	field       = "data.change_data.changes.field"
 )
 
 var (
@@ -84,7 +84,6 @@ type GetSetParam struct {
 	Ndays    *int         `json:"ndays" validate:"omitempty,number,lt=100"`
 	RemoteIP *string      `json:"remoteIP" validate:"omitempty"`
 	Pri      *LogPriority `json:"pri" validate:"omitempty,oneof=1 2 3 4 5 6 7 8"`
-	setAttr  string       `json:"setAttr"`
 }
 
 // GetLogs retrieves an slice of logEntry from Elasticsearch based on the fields provided in logParam.
@@ -111,7 +110,9 @@ func GetLogs(querytoken string, client *elasticsearch.TypedClient, logParam GetL
 			queries = append(queries, logType)
 		}
 	}
-
+	if ok, module := termQueryForField(module, logParam.Module); ok {
+		queries = append(queries, module)
+	}
 	if ok, who := termQueryForField(who, logParam.Who); ok {
 
 		queries = append(queries, who)
@@ -152,7 +153,7 @@ func GetLogs(querytoken string, client *elasticsearch.TypedClient, logParam GetL
 	}
 
 	if len(queries) == 0 {
-		return nil, 0, fmt.Errorf("No Filter param")
+		return nil, 0, fmt.Errorf("no Filter param")
 	}
 
 	// sorting record on base of when
@@ -198,15 +199,16 @@ func GetLogs(querytoken string, client *elasticsearch.TypedClient, logParam GetL
 	if err != nil {
 		return nil, 0, fmt.Errorf("Error while searching document in es:%v", err)
 	}
-	var logEnter LogEntry
+
 
 	// Unmarshalling hit.source into LogEntry
 	if res != nil {
 		for _, hit := range res.Hits.Hits {
-			if err := json.Unmarshal([]byte(hit.Source_), &logEnter); err != nil {
+			var logEntery LogEntry
+			if err := json.Unmarshal([]byte(hit.Source_), &logEntery); err != nil {
 				return nil, 0, fmt.Errorf("error while unmarshalling response:%v", err)
 			}
-			logEntries = append(logEntries, logEnter)
+			logEntries = append(logEntries, logEntery)
 		}
 	}
 	return logEntries, int(res.Hits.Total.Value), nil
@@ -282,9 +284,10 @@ func GetLocalIPAddress() (string, error) {
 func GetSet(queryToken string, client *elasticsearch.TypedClient, setAttr string, setParam GetSetParam) (map[string]int64, error) {
 
 	var (
-		query   *types.Query
-		zero    = 0
-		dataMap = make(map[string]int64)
+		query           *types.Query
+		dataMap         = make(map[string]int64)
+		aggResponseSize = 1000
+		logSize         = 0
 	)
 
 	// Validate setAttr
@@ -308,11 +311,12 @@ func GetSet(queryToken string, client *elasticsearch.TypedClient, setAttr string
 	// This will return a set of unique values for an attribute based on method parameters
 	res, err := client.Search().Index(Index).Request(&search.Request{
 		Query: query,
-		Size:  &zero,
+		Size:  &logSize,
 		Aggregations: map[string]types.Aggregations{
 			logSet: {
 				Terms: &types.TermsAggregation{
 					Field: some.String(setAttr),
+					Size:  &aggResponseSize,
 					Order: map[string]sortorder.SortOrder{"_count": sortorder.SortOrder{"asc"}},
 				},
 			},
@@ -620,7 +624,7 @@ func GetChanges(querytoken string, client *elasticsearch.TypedClient, logParam G
 
 	if logParam.Field != nil {
 
-		if ok, field := termQueryForField("data.changes.field", logParam.Field); ok {
+		if ok, field := termQueryForField("data.change_data.changes.field", logParam.Field); ok {
 			queries = append(queries, field)
 		}
 	}
@@ -665,7 +669,7 @@ func GetChanges(querytoken string, client *elasticsearch.TypedClient, logParam G
 	}
 
 	if len(queries) == 0 {
-		return nil, 0, fmt.Errorf("No Filter param")
+		return nil, 0, fmt.Errorf("no Filter param")
 	}
 
 	// sorting record on base of when

cmd/logConsumer/es_logs_mapping.go → logharbour/es_logs_mapping.go

@@ -1,6 +1,6 @@
-package main
+package logharbour
 
-const esLogsMapping = `{
+const ESLogsMapping = `{
   "mappings": {
     "properties": {
       "id": {
@@ -33,11 +33,11 @@ const esLogsMapping = `{
       "class": {
         "type": "keyword"
       },
-      "instanceId": {
+      "instance": {
         "type": "keyword"
       },
       "status": {
-        "type": "boolean"
+        "type": "integer"
       },
       "error": {
         "type": "text"

logharbour/test/elastic_test.go

@@ -7,7 +7,8 @@ import (
 	"time"
 
 	"github.com/remiges-tech/logharbour/logharbour"
-	elasticsearchctl "github.com/remiges-tech/logharbour/server/elasticSearchCtl/elasticSearch"
+	estestutils "github.com/remiges-tech/logharbour/logharbour/test"
+
 	"github.com/stretchr/testify/require"
 )
 
@@ -55,7 +56,7 @@ func TestGetLogs(t *testing.T) {
 		t.Run(tc.Name, func(t *testing.T) {
 
 			if tc.TestJsonFile != "" {
-				tc.ExpectedLogEntries, err = elasticsearchctl.ReadLogFromFile(tc.TestJsonFile)
+				tc.ExpectedLogEntries, err = estestutils.ReadLogFromFile(tc.TestJsonFile)
 				if err != nil {
 					fmt.Printf("error converting data from log file:%v\n", err)
 				}

server/elasticSearchCtl/elasticSearch/elasticSearchCtl.go → logharbour/test/estestutils.go

@@ -1,4 +1,4 @@
-package elasticsearchctl
+package logharbour
 
 import (
 	"context"

logharbour/test/main_test.go

@@ -9,67 +9,14 @@ import (
 	"time"
 
 	es "github.com/elastic/go-elasticsearch/v8"
-	elasticsearchctl "github.com/remiges-tech/logharbour/server/elasticSearchCtl/elasticSearch"
+	"github.com/remiges-tech/logharbour/logharbour"
+	estestutils "github.com/remiges-tech/logharbour/logharbour/test"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/modules/elasticsearch"
 )
 
 var (
-	indexBody = `{
-		"settings": {
-		  "number_of_shards": 1,
-		  "number_of_replicas": 0
-		},
-		"mappings": {
-		  "properties": {
-			"app": {
-			  "type": "keyword"
-			},
-			"system": {
-			  "type": "keyword"
-			},
-			"module": {
-			  "type": "keyword"
-			},
-			"type": {
-			  "type": "keyword"
-			},
-			"pri": {
-			  "type": "keyword"
-			},
-			"when": {
-			  "type": "date"
-			},
-			"who": {
-			  "type": "keyword"
-			},
-			"op": {
-			  "type": "keyword"
-			},
-			"class": {
-			  "type": "keyword"
-			},
-			"instance": {
-			  "type": "keyword"
-			},
-			"status": {
-			  "type": "integer"
-			},
-			"error": {
-			  "type": "keyword"
-			},
-			"remote_ip": {
-			  "type": "ip"
-			},
-			"msg": {
-			  "type": "keyword"
-			},
-			"data": {
-			  "type": "text"
-			}
-		  }
-		}
-	  }`
+	indexBody   = logharbour.ESLogsMapping
 	typedClient *es.TypedClient
 	filepath    = "../test/testData/testData.json"
 	indexName   = "logharbour"
@@ -129,16 +76,16 @@ func TestMain(m *testing.M) {
 
 func fillElasticWithData(esClient *es.Client, indexName, indexBody, filepath string) error {
 
-	if err := elasticsearchctl.CreateElasticIndex(esClient, indexName, indexBody); err != nil {
+	if err := estestutils.CreateElasticIndex(esClient, indexName, indexBody); err != nil {
 		return fmt.Errorf("error while creating elastic search index: %v", err)
 	}
 
-	logEntries, err := elasticsearchctl.ReadLogFromFile(filepath)
+	logEntries, err := estestutils.ReadLogFromFile(filepath)
 	if err != nil {
 		return fmt.Errorf("error converting data from log file:%v", err)
 	}
 
-	if err := elasticsearchctl.InsertLog(esClient, logEntries, indexName); err != nil {
+	if err := estestutils.InsertLog(esClient, logEntries, indexName); err != nil {
 		return fmt.Errorf("error while inserting data in elastic search: %v", err.Error())
 	}
 

server/config_dev.json

@@ -4,6 +4,6 @@
     "app_server_port": "8090",
     "db_user": "elastic",
     "index_name": "logharbour",
-    "db_password": "XSTwxC*giO71PGZm5urS",
-    "certificate_fingerprint": "4b41377142441840d1099bcde5d294d25b8e7b39daf0a879343e5b552bc17f2c"
+    "db_password": "9jjWQryjHca-9flzDcKU",
+    "certificate_fingerprint": "3395adb7832f24e043ea7101b7f821bd97786fc808c335ba439a3681585119fc"
 }