Skip to content

Fixing issue with logs #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 11, 2020
Merged

Fixing issue with logs #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions admin/handlers/handlers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"github.com/jmpsec/osctrl/admin/sessions"
"github.com/jmpsec/osctrl/carves"
"github.com/jmpsec/osctrl/environments"
"github.com/jmpsec/osctrl/logging"
"github.com/jmpsec/osctrl/metrics"
"github.com/jmpsec/osctrl/nodes"
"github.com/jmpsec/osctrl/queries"
Expand Down Expand Up @@ -43,6 +44,7 @@ type HandlersAdmin struct {
Carves *carves.Carves
Settings *settings.Settings
Metrics *metrics.Metrics
LoggerDB *logging.LoggerDB
Sessions *sessions.SessionManager
ServiceVersion string
OsqueryTables []types.OsqueryTable
Expand Down Expand Up @@ -99,6 +101,12 @@ func WithMetrics(metrics *metrics.Metrics) HandlersOption {
}
}

func WithLoggerDB(logger *logging.LoggerDB) HandlersOption {
return func(h *HandlersAdmin) {
h.LoggerDB = logger
}
}

func WithSessions(sessions *sessions.SessionManager) HandlersOption {
return func(h *HandlersAdmin) {
h.Sessions = sessions
Expand Down
6 changes: 3 additions & 3 deletions admin/handlers/json-logs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request)
// Get logs
logJSON := []LogJSON{}
if logType == "status" {
statusLogs, err := h.postgresStatusLogs(UUID, env, secondsBack)
statusLogs, err := h.LoggerDB.StatusLogs(UUID, env, secondsBack)
if err != nil {
log.Printf("error getting logs %v", err)
h.Inc(metricJSONErr)
Expand All @@ -129,7 +129,7 @@ func (h *HandlersAdmin) JSONLogsHandler(w http.ResponseWriter, r *http.Request)
logJSON = append(logJSON, _l)
}
} else if logType == "result" {
resultLogs, err := h.postgresResultLogs(UUID, env, secondsBack)
resultLogs, err := h.LoggerDB.ResultLogs(UUID, env, secondsBack)
if err != nil {
log.Printf("error getting logs %v", err)
h.Inc(metricJSONErr)
Expand Down Expand Up @@ -178,7 +178,7 @@ func (h *HandlersAdmin) JSONQueryLogsHandler(w http.ResponseWriter, r *http.Requ
return
}
// Get logs
queryLogs, err := h.postgresQueryLogs(name)
queryLogs, err := h.LoggerDB.QueryLogs(name)
if err != nil {
log.Printf("error getting logs %v", err)
h.Inc(metricJSONErr)
Expand Down
72 changes: 0 additions & 72 deletions admin/handlers/postgres.go
View file
Open in desktop

This file was deleted.

9 changes: 8 additions & 1 deletion admin/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/jmpsec/osctrl/backend"
"github.com/jmpsec/osctrl/carves"
"github.com/jmpsec/osctrl/environments"
"github.com/jmpsec/osctrl/logging"
"github.com/jmpsec/osctrl/metrics"
"github.com/jmpsec/osctrl/nodes"
"github.com/jmpsec/osctrl/queries"
Expand Down Expand Up @@ -108,6 +109,7 @@ var (
osqueryTables []types.OsqueryTable
adminMetrics *metrics.Metrics
handlersAdmin *ahandlers.HandlersAdmin
loggerDB *logging.LoggerDB
)

// Variables for flags
Expand Down Expand Up @@ -280,7 +282,11 @@ func main() {
if err != nil {
log.Fatalf("Error loading metrics - %v", err)
}

// Initialize DB logger
loggerDB, err = logging.CreateLoggerDB(*dbFlag, backend.DBKey)
if err != nil {
log.Fatalf("Error loading logger - %v", err)
}
// Start SAML Middleware if we are using SAML
if adminConfig.Auth == settings.AuthSAML {
if settingsmgr.DebugService(settings.ServiceAdmin) {
Expand Down Expand Up @@ -330,6 +336,7 @@ func main() {
ahandlers.WithCarves(carvesmgr),
ahandlers.WithSettings(settingsmgr),
ahandlers.WithMetrics(adminMetrics),
ahandlers.WithLoggerDB(loggerDB),
ahandlers.WithSessions(sessionsmgr),
ahandlers.WithVersion(serviceVersion),
ahandlers.WithOsqueryTables(osqueryTables),
Expand Down
7 changes: 7 additions & 0 deletions admin/static/js/nodeactions.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,3 +85,10 @@ function showCarveFiles(_uuids) {
});
$("#carveModal").modal();
}

function changeBackValue(table_id, range_input, range_output) {
range_output.value = range_input.value;
var table = $('#' + table_id).DataTable();
var _url = table.ajax.url();
table.ajax.url(_url.split('seconds=')[0] + 'seconds=' + (range_output.value*3600));
}
6 changes: 3 additions & 3 deletions admin/templates/conf.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
<label for="conf_range">Configuration Interval: <b><output id="conf_output">{{ .Environment.ConfigInterval }}</output></b> seconds</label>
<input type="range" class="form-control-range" id="conf_range"
value="{{ .Environment.ConfigInterval }}" min="10" max="600" step="10"
oninput="changeIntervalValue('conf_range', 'conf_output');">
oninput="changeIntervalValue(conf_range, conf_output);">
<input id="conf_range_save" type="hidden" value="{{ .Environment.ConfigInterval }}">
</div>
</div>
Expand All @@ -52,7 +52,7 @@
<label for="logging_range">Logging Interval: <b><output id="logging_output">{{ .Environment.LogInterval }}</output></b> seconds</label>
<input type="range" class="form-control-range" id="logging_range"
value="{{ .Environment.LogInterval }}" min="10" max="600" step="10"
oninput="changeIntervalValue('logging_range', 'logging_output');">
oninput="changeIntervalValue(logging_range, logging_output);">
<input id="logging_range_save" type="hidden" value="{{ .Environment.LogInterval }}">
</div>
</div>
Expand All @@ -61,7 +61,7 @@
<label for="query_range">Query Interval: <b><output id="query_output">{{ .Environment.QueryInterval }}</output></b> seconds</label>
<input type="range" class="form-control-range" id="query_range"
value="{{ .Environment.QueryInterval }}" min="10" max="300" step="1"
oninput="changeIntervalValue('query_range', 'query_output');">
oninput="changeIntervalValue(query_range, query_output);">
<input id="query_range_save" type="hidden" value="{{ .Environment.QueryInterval }}">
</div>
</div>
Expand Down
20 changes: 10 additions & 10 deletions admin/templates/node.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,10 @@
<div class="tab-pane fade" id="status-logs" role="tabpanel">
<div class="card mt-2">
<div id="status-card-header" class="card-header">
<i class="fas fa-stream"></i> Last 6 hours of status logs for node {{ .UUID }}
<i class="fas fa-stream"></i>
<label for="back_hours_status">Last <b><output id="back_output_status">6</output></b> hours of status logs for node {{ .UUID }}</label>
<input type="range" class="form-control-range" id="back_hours_status"
value="6" min="1" max="24" step="1" oninput="changeBackValue('tableStatusLogs', back_hours_status, back_output_status);">
<div class="card-header-actions">
<small>Refresh in <span id="status_refresh_seconds">60</span> seconds</small>
<button id="status_refresh_pause" class="btn btn-sm btn-outline-dark" data-tooltip="true"
Expand Down Expand Up @@ -337,7 +340,10 @@
<div class="tab-pane fade" id="result-logs" role="tabpanel">
<div class="card mt-2">
<div id="result-card-header" class="card-header">
<i class="fas fa-stream"></i> Last 6 hours of result logs for node {{ .UUID }}
<i class="fas fa-stream"></i>
<label for="back_hours_result">Last <b><output id="back_output_result">6</output></b> hours of status logs for node {{ .UUID }}</label>
<input type="range" class="form-control-range" id="back_hours_result"
value="6" min="1" max="24" step="1" oninput="changeBackValue('tableResultLogs', back_hours_result, back_output_result);">
<div class="card-header-actions">
<small>Refresh in <span id="result_refresh_seconds">60</span> seconds</small>
<button id="result_refresh_pause" class="btn btn-sm btn-outline-dark" data-tooltip="true"
Expand Down Expand Up @@ -442,7 +448,7 @@
searching : true,
processing : true,
ajax : {
url: "/json/logs/status/{{ .Environment }}/{{ .UUID }}",
url: "/json/logs/status/{{ .Environment }}/{{ .UUID }}?seconds=" + ($("#back_hours_status").val() * 3600),
dataSrc: function(json) {
$('#status-card-header').removeClass("bg-danger");
return json.data;
Expand All @@ -466,8 +472,6 @@
});
// Display the number of seconds left and refresh for status logs
var refreshSecondsStatus = 60;
// Time to go back in seconds, default is 6 hours
var refreshBackSeconds = 21600;
var timeLeftStatus = refreshSecondsStatus;
var tableTimerStatus = setInterval(function(){
if (document.getElementById("status_refresh_value").value === 'yes') {
Expand All @@ -476,7 +480,6 @@
document.getElementById("status_refresh_seconds").textContent = timeLeftStatus;
if (timeLeftStatus <= 0) {
timeLeftStatus = refreshSecondsStatus;
tableStatusLogs.ajax.url("/json/logs/status/{{ .Environment }}/{{ .UUID }}?seconds=" + refreshBackSeconds);
tableStatusLogs.ajax.reload();
}
},1000);
Expand All @@ -489,7 +492,7 @@
searching : true,
processing : true,
ajax : {
url: "/json/logs/result/{{ .Environment }}/{{ .UUID }}",
url: "/json/logs/result/{{ .Environment }}/{{ .UUID }}?seconds=" + (back_output_result.value * 3600),
dataSrc: function(json) {
$('#result-card-header').removeClass("bg-danger");
return json.data;
Expand All @@ -514,8 +517,6 @@

// Display the number of seconds left and refresh for result logs
var refreshSecondsResult = 60;
// Time to go back in seconds, default is 6 hours
var refreshBackSeconds = 21600;
var timeLeftResult = refreshSecondsResult;
var tableTimerResult = setInterval(function(){
if (document.getElementById("result_refresh_value").value === 'yes') {
Expand All @@ -524,7 +525,6 @@
document.getElementById("result_refresh_seconds").textContent = timeLeftResult;
if (timeLeftResult <= 0) {
timeLeftResult = refreshSecondsResult;
tableResultLogs.ajax.url("/json/logs/result/{{ .Environment }}/{{ .UUID }}?seconds=" + refreshBackSeconds);
tableResultLogs.ajax.reload();
}
},1000);
Expand Down
53 changes: 44 additions & 9 deletions logging/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package logging
import (
"encoding/json"
"log"
"time"

"github.com/jinzhu/gorm"

Expand All @@ -11,13 +12,6 @@ import (
"github.com/jmpsec/osctrl/types"
)

const (
// DBName as JSON key for configuration
DBName string = "db"
// DBFile as default file for configuration
DBFile string = "config/" + DBName + ".json"
)

// OsqueryResultData to log result data to database
type OsqueryResultData struct {
gorm.Model
Expand Down Expand Up @@ -59,9 +53,9 @@ type LoggerDB struct {
Enabled bool
}

func CreateLoggerDB() (*LoggerDB, error) {
func CreateLoggerDB(dbfile, dbname string) (*LoggerDB, error) {
// Load DB configuration
config, err := backend.LoadConfiguration(DBFile, DBName)
config, err := backend.LoadConfiguration(dbfile, dbname)
if err != nil {
return nil, err
}
Expand All @@ -75,6 +69,18 @@ func CreateLoggerDB() (*LoggerDB, error) {
Configuration: config,
Enabled: true,
}
// table osquery_status_data
if err := database.AutoMigrate(OsqueryStatusData{}).Error; err != nil {
log.Fatalf("Failed to AutoMigrate table (osquery_status_data): %v", err)
}
// table osquery_result_data
if err := database.AutoMigrate(OsqueryResultData{}).Error; err != nil {
log.Fatalf("Failed to AutoMigrate table (osquery_result_data): %v", err)
}
// table osquery_query_data
if err := database.AutoMigrate(OsqueryQueryData{}).Error; err != nil {
log.Fatalf("Failed to AutoMigrate table (osquery_query_data): %v", err)
}
return l, nil
}

Expand Down Expand Up @@ -195,3 +201,32 @@ func (logDB *LoggerDB) Query(data []byte, environment, uuid, name string, status
log.Printf("NewRecord did not return true")
}
}

// QueryLogs will retrieve all query logs
func (logDB *LoggerDB) QueryLogs(name string) ([]OsqueryQueryData, error) {
var logs []OsqueryQueryData
if err := logDB.Database.Where("name = ?", name).Find(&logs).Error; err != nil {
return logs, err
}
return logs, nil
}

// StatusLogs will retrieve all status logs
func (logDB *LoggerDB) StatusLogs(uuid, environment string, seconds int64) ([]OsqueryStatusData, error) {
var logs []OsqueryStatusData
minusSeconds := time.Now().Add(time.Duration(-seconds) * time.Second)
if err := logDB.Database.Where("uuid = ? AND environment = ?", uuid, environment).Where("created_at > ?", minusSeconds).Find(&logs).Error; err != nil {
return logs, err
}
return logs, nil
}

// ResultLogs will retrieve all result logs
func (logDB *LoggerDB) ResultLogs(uuid, environment string, seconds int64) ([]OsqueryResultData, error) {
var logs []OsqueryResultData
minusSeconds := time.Now().Add(time.Duration(-seconds) * time.Second)
if err := logDB.Database.Where("uuid = ? AND environment = ?", uuid, environment).Where("created_at > ?", minusSeconds).Find(&logs).Error; err != nil {
return logs, err
}
return logs, nil
}
Loading