Limit local status and result logs and queries

admin/main.go

@@ -72,12 +72,12 @@ const (
 const (
 	// Static files folder
 	staticFilesFolder string = "./static"
-	// Carved files folder
-	carvedFilesFolder string = "carved_files/"
 	// Default refreshing interval in seconds
 	defaultRefresh int = 300
 	// Default hours to classify nodes as inactive
 	defaultInactive int = -72
+	// Hourly interval to cleanup logs
+	hourlyInterval int = 60
 )
 
 // osquery
@@ -182,6 +182,7 @@ func loadConfiguration(file, service string) (types.JSONConfigurationService, er
 
 // Initialization code
 func init() {
+	log.Printf("==================== Initializing %s v%s", serviceName, serviceVersion)
 	// Command line flags
 	flag.Usage = adminUsage
 	// Define flags
@@ -231,6 +232,7 @@ func init() {
 
 // Go go!
 func main() {
+	log.Printf("==================== Starting %s v%s", serviceName, serviceVersion)
 	// Database handler
 	dbConfig, err := backend.LoadConfiguration(*dbFlag, backend.DBKey)
 	if err != nil {
@@ -328,6 +330,43 @@ func main() {
 		}
 	}()
 
+	// Cleaning up status/result/query logs
+	go func() {
+		for {
+			_e, err := envs.All()
+			if err != nil {
+				log.Printf("error getting environments when cleaning up logs - %v", err)
+			}
+			for _, e := range _e {
+				if settingsmgr.CleanStatusLogs() {
+					if settingsmgr.DebugService(settings.ServiceAdmin) {
+						log.Println("DebugService: Cleaning up status logs")
+					}
+					if err := loggerDB.CleanStatusLogs(e.Name, settingsmgr.CleanStatusInterval()); err != nil {
+						log.Printf("error cleaning up status logs - %v", err)
+					}
+				}
+				if settingsmgr.CleanResultLogs() {
+					if settingsmgr.DebugService(settings.ServiceAdmin) {
+						log.Println("DebugService: Cleaning up result logs")
+					}
+					if err := loggerDB.CleanResultLogs(e.Name, settingsmgr.CleanResultInterval()); err != nil {
+						log.Printf("error cleaning up result logs - %v", err)
+					}
+				}
+			}
+			if settingsmgr.CleanQueryLogs() {
+				if settingsmgr.DebugService(settings.ServiceAdmin) {
+					log.Println("DebugService: Cleaning up query logs")
+				}
+				if err := loggerDB.CleanQueryLogs(settingsmgr.CleanQueryEntries()); err != nil {
+					log.Printf("error cleaning up query logs - %v", err)
+				}
+			}
+			time.Sleep(time.Duration(hourlyInterval) * time.Second)
+		}
+	}()
+
 	// Initialize Admin handlers before router
 	handlersAdmin = ahandlers.CreateHandlersAdmin(
 		ahandlers.WithDB(db),
@@ -454,15 +493,8 @@ func main() {
 		routerAdmin.PathPrefix("/saml/").Handler(samlMiddleware)
 	}
 
-	// multiple listeners channel
-	finish := make(chan bool)
-
 	// Launch HTTP server for admin
-	go func() {
-		serviceAdmin := adminConfig.Listener + ":" + adminConfig.Port
-		log.Printf("%s v%s - HTTP listening %s", serviceName, serviceVersion, serviceAdmin)
-		log.Fatal(http.ListenAndServe(serviceAdmin, routerAdmin))
-	}()
-
-	<-finish
+	serviceAdmin := adminConfig.Listener + ":" + adminConfig.Port
+	log.Printf("%s v%s - HTTP listening %s", serviceName, serviceVersion, serviceAdmin)
+	log.Fatal(http.ListenAndServe(serviceAdmin, routerAdmin))
 }

api/main.go

@@ -135,6 +135,7 @@ func loadConfiguration(file string) (types.JSONConfigurationService, error) {
 
 // Initialization code
 func init() {
+	log.Printf("==================== Initializing %s v%s", serviceName, serviceVersion)
 	var err error
 	// Command line flags
 	flag.Usage = apiUsage
@@ -168,6 +169,7 @@ func init() {
 
 // Go go!
 func main() {
+	log.Printf("==================== Starting %s v%s", serviceName, serviceVersion)
 	// Database handler
 	dbConfig, err := backend.LoadConfiguration(*dbFlag, backend.DBKey)
 	if err != nil {
@@ -285,15 +287,8 @@ func main() {
 	routerAPI.Handle(_apiPath(apiEnvironmentsPath), handlerAuthCheck(http.HandlerFunc(apiEnvironmentsHandler))).Methods("GET")
 	routerAPI.Handle(_apiPath(apiEnvironmentsPath)+"/", handlerAuthCheck(http.HandlerFunc(apiEnvironmentsHandler))).Methods("GET")
 
-	// multiple listeners channel
-	finish := make(chan bool)
-
 	// Launch HTTP server for TLS endpoint
-	go func() {
-		serviceListener := apiConfig.Listener + ":" + apiConfig.Port
-		log.Printf("%s v%s - HTTP listening %s", serviceName, serviceVersion, serviceListener)
-		log.Fatal(http.ListenAndServe(serviceListener, routerAPI))
-	}()
-
-	<-finish
+	serviceListener := apiConfig.Listener + ":" + apiConfig.Port
+	log.Printf("%s v%s - HTTP listening %s", serviceName, serviceVersion, serviceListener)
+	log.Fatal(http.ListenAndServe(serviceListener, routerAPI))
 }

environments/environments.go

@@ -322,7 +322,7 @@ func (environment *Environment) ExpireEnroll(name string) error {
 	return nil
 }
 
-// RotateRemove to replace Secret and SecrtPath for enrolling in an environment
+// RotateRemove to replace Secret and SecretPath for enrolling in an environment
 func (environment *Environment) RotateRemove(name string) error {
 	env, err := environment.Get(name)
 	if err != nil {

logging/db.go

@@ -2,16 +2,23 @@ package logging
 
 import (
 	"encoding/json"
+	"fmt"
 	"log"
 	"time"
 
 	"github.com/jinzhu/gorm"
 
 	"github.com/jmpsec/osctrl/backend"
+	"github.com/jmpsec/osctrl/queries"
 	"github.com/jmpsec/osctrl/settings"
 	"github.com/jmpsec/osctrl/types"
 )
 
+const (
+	// Default interval in seconds for cleanup old logs
+	defaultCleanupInterval = 86400
+)
+
 // OsqueryResultData to log result data to database
 type OsqueryResultData struct {
 	gorm.Model
@@ -92,24 +99,51 @@ func (logDB *LoggerDB) Settings(mgr *settings.Settings) {
 		if err := mgr.NewStringValue(settings.ServiceAdmin, settings.QueryResultLink, settings.QueryLink); err != nil {
 			log.Fatalf("Failed to add %s to settings: %v", settings.QueryResultLink, err)
 		}
-	} else if err := mgr.SetString(settings.QueryLink, settings.ServiceAdmin, settings.QueryResultLink, false); err != nil {
-		log.Printf("Error setting %s with %s - %v", settings.QueryResultLink, settings.QueryLink, err)
 	}
 	// Setting link for status logs
 	if !mgr.IsValue(settings.ServiceAdmin, settings.StatusLogsLink) {
 		if err := mgr.NewStringValue(settings.ServiceAdmin, settings.StatusLogsLink, settings.StatusLink); err != nil {
-			log.Fatalf("Failed to add %s to settings: %v", settings.DebugHTTP, err)
+			log.Fatalf("Failed to add %s to settings: %v", settings.StatusLogsLink, err)
 		}
-	} else if err := mgr.SetString(settings.StatusLink, settings.ServiceAdmin, settings.StatusLogsLink, false); err != nil {
-		log.Printf("Error setting %s with %s - %v", settings.StatusLogsLink, settings.StatusLink, err)
 	}
 	// Setting link for result logs
 	if !mgr.IsValue(settings.ServiceAdmin, settings.ResultLogsLink) {
 		if err := mgr.NewStringValue(settings.ServiceAdmin, settings.ResultLogsLink, settings.ResultsLink); err != nil {
-			log.Fatalf("Failed to add %s to settings: %v", settings.DebugHTTP, err)
+			log.Fatalf("Failed to add %s to settings: %v", settings.ResultLogsLink, err)
+		}
+	}
+	// Setting values to enable log cleanup for status logs
+	if !mgr.IsValue(settings.ServiceAdmin, settings.CleanStatusLogs) {
+		if err := mgr.NewBooleanValue(settings.ServiceAdmin, settings.CleanStatusLogs, false); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.CleanStatusLogs, err)
+		}
+	}
+	if !mgr.IsValue(settings.ServiceAdmin, settings.CleanStatusInterval) {
+		if err := mgr.NewIntegerValue(settings.ServiceAdmin, settings.CleanStatusInterval, defaultCleanupInterval); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.CleanStatusInterval, err)
+		}
+	}
+	// Setting values to enable log cleanup for result logs
+	if !mgr.IsValue(settings.ServiceAdmin, settings.CleanResultLogs) {
+		if err := mgr.NewBooleanValue(settings.ServiceAdmin, settings.CleanResultLogs, false); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.CleanResultLogs, err)
+		}
+	}
+	if !mgr.IsValue(settings.ServiceAdmin, settings.CleanResultInterval) {
+		if err := mgr.NewIntegerValue(settings.ServiceAdmin, settings.CleanResultInterval, defaultCleanupInterval); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.CleanResultInterval, err)
+		}
+	}
+	// Setting values to enable log cleanup for query logs
+	if !mgr.IsValue(settings.ServiceAdmin, settings.CleanQueryLogs) {
+		if err := mgr.NewBooleanValue(settings.ServiceAdmin, settings.CleanQueryLogs, false); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.CleanQueryLogs, err)
+		}
+	}
+	if !mgr.IsValue(settings.ServiceAdmin, settings.CleanQueryEntries) {
+		if err := mgr.NewIntegerValue(settings.ServiceAdmin, settings.CleanQueryEntries, 100); err != nil {
+			log.Fatalf("Failed to add %s to settings: %v", settings.CleanQueryEntries, err)
 		}
-	} else if err := mgr.SetString(settings.ResultsLink, settings.ServiceAdmin, settings.ResultLogsLink, false); err != nil {
-		log.Printf("Error setting %s with %s - %v", settings.ResultLogsLink, settings.ResultsLink, err)
 	}
 }
 
@@ -230,3 +264,62 @@ func (logDB *LoggerDB) ResultLogs(uuid, environment string, seconds int64) ([]Os
 	}
 	return logs, nil
 }
+
+// CleanStatusLogs will delete old status logs
+func (logDB *LoggerDB) CleanStatusLogs(environment string, seconds int64) error {
+	minusSeconds := time.Now().Add(time.Duration(-seconds) * time.Second)
+	if err := logDB.Database.Unscoped().Where("environment = ?", environment).Where("created_at < ?", minusSeconds).Delete(&OsqueryStatusData{}).Error; err != nil {
+		return fmt.Errorf("CleanStatusLogs %v", err)
+	}
+	return nil
+}
+
+// CleanResultLogs will delete old status logs
+func (logDB *LoggerDB) CleanResultLogs(environment string, seconds int64) error {
+	minusSeconds := time.Now().Add(time.Duration(-seconds) * time.Second)
+	if err := logDB.Database.Unscoped().Where("environment = ?", environment).Where("created_at < ?", minusSeconds).Delete(&OsqueryResultData{}).Error; err != nil {
+		return fmt.Errorf("CleanResultLogs %v", err)
+	}
+	return nil
+}
+
+// CleanQueryLogs will delete old query logs
+func (logDB *LoggerDB) CleanQueryLogs(entries int64) error {
+	// TODO this would be better and simpler with foreign keys and delete cascade
+	// Find queries to delete with OFFSET
+	var oldQueries []queries.DistributedQuery
+	logDB.Database.Offset(entries).Find(&oldQueries)
+	for _, q := range oldQueries {
+		if q.Completed {
+			// Get query results
+			var queriesData []OsqueryQueryData
+			if err := logDB.Database.Where("name = ?", q.Name).Find(&queriesData).Error; err != nil {
+				return err
+			}
+			if err := logDB.Database.Unscoped().Delete(&queriesData).Error; err != nil {
+				return err
+			}
+			// Get query targets
+			var queriesTargets []queries.DistributedQueryTarget
+			if err := logDB.Database.Where("name = ?", q.Name).Find(&queriesTargets).Error; err != nil {
+				return err
+			}
+			if err := logDB.Database.Unscoped().Delete(&queriesTargets).Error; err != nil {
+				return err
+			}
+			// Get query executions
+			var queriesExecutions []queries.DistributedQueryExecution
+			if err := logDB.Database.Where("name = ?", q.Name).Find(&queriesExecutions).Error; err != nil {
+				return err
+			}
+			if err := logDB.Database.Unscoped().Delete(&queriesExecutions).Error; err != nil {
+				return err
+			}
+			// Delete query
+			if err := logDB.Database.Unscoped().Delete(&q).Error; err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

settings/settings.go

@@ -60,9 +60,15 @@ const (
 
 // Names for setting values for logging
 const (
-	QueryResultLink string = "query_result_link"
-	StatusLogsLink  string = "status_logs_link"
-	ResultLogsLink  string = "result_logs_link"
+	QueryResultLink     string = "query_result_link"
+	StatusLogsLink      string = "status_logs_link"
+	ResultLogsLink      string = "result_logs_link"
+	CleanStatusLogs     string = "clean_status_logs"
+	CleanStatusInterval string = "clean_status_interval"
+	CleanResultLogs     string = "clean_result_logs"
+	CleanResultInterval string = "clean_result_interval"
+	CleanQueryLogs      string = "clean_query_logs"
+	CleanQueryEntries   string = "clean_query_entries"
 )
 
 // Default values for the setting values for logging
@@ -104,9 +110,9 @@ type Settings struct {
 
 // ValidTypes to check validity of settings type
 var ValidTypes = map[string]struct{}{
-  TypeString: struct{}{},
-  TypeBoolean: struct{}{},
-  TypeInteger: struct{}{},
+	TypeString:  struct{}{},
+	TypeBoolean: struct{}{},
+	TypeInteger: struct{}{},
 }
 
 // NewSettings to initialize the access to settings and table
@@ -192,7 +198,7 @@ func (conf *Settings) NewIntegerValue(service, name string, value int64) error {
 // VerifyType to make sure type is valid
 func (conf *Settings) VerifyType(sType string) bool {
 	_, ok := ValidTypes[sType]
-  return ok
+	return ok
 }
 
 // DeleteValue deletes an existing settings value
@@ -517,6 +523,60 @@ func (conf *Settings) ResultLogsLink() string {
 	return value.String
 }
 
+// CleanStatusLogs checks if status logs cleanup is enabled
+func (conf *Settings) CleanStatusLogs() bool {
+	value, err := conf.RetrieveValue(ServiceAdmin, CleanStatusLogs)
+	if err != nil {
+		return false
+	}
+	return value.Boolean
+}
+
+// CleanStatusInterval gets the interval in seconds to cleanup status logs
+func (conf *Settings) CleanStatusInterval() int64 {
+	value, err := conf.RetrieveValue(ServiceAdmin, CleanStatusInterval)
+	if err != nil {
+		return 0
+	}
+	return value.Integer
+}
+
+// CleanResultLogs checks if result logs cleanup is enabled
+func (conf *Settings) CleanResultLogs() bool {
+	value, err := conf.RetrieveValue(ServiceAdmin, CleanResultLogs)
+	if err != nil {
+		return false
+	}
+	return value.Boolean
+}
+
+// CleanResultInterval gets the interval in seconds to cleanup result logs
+func (conf *Settings) CleanResultInterval() int64 {
+	value, err := conf.RetrieveValue(ServiceAdmin, CleanResultInterval)
+	if err != nil {
+		return 0
+	}
+	return value.Integer
+}
+
+// CleanQueryLogs checks if query logs cleanup is enabled
+func (conf *Settings) CleanQueryLogs() bool {
+	value, err := conf.RetrieveValue(ServiceAdmin, CleanQueryLogs)
+	if err != nil {
+		return false
+	}
+	return value.Boolean
+}
+
+// CleanQueryEntries gets the number of entries to cleanup in query logs
+func (conf *Settings) CleanQueryEntries() int64 {
+	value, err := conf.RetrieveValue(ServiceAdmin, CleanQueryEntries)
+	if err != nil {
+		return 0
+	}
+	return value.Integer
+}
+
 // DefaultEnv gets the default environment
 // FIXME customize the fallover one
 func (conf *Settings) DefaultEnv(service string) string {