diff --git a/.idea/dataSources.local.xml b/.idea/dataSources.local.xml index e57e31bd..f4465d2c 100644 --- a/.idea/dataSources.local.xml +++ b/.idea/dataSources.local.xml @@ -13,7 +13,7 @@ - + " diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml index 38543815..152b0114 100644 --- a/.idea/dataSources.xml +++ b/.idea/dataSources.xml @@ -15,11 +15,11 @@ - + sqlite.xerial true org.sqlite.JDBC - jdbc:sqlite:$USER_HOME$/ij-perf-report-db/db.sqlite + jdbc:sqlite:$USER_HOME$/ij-perf-db/db.sqlite file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.25.1/sqlite-jdbc-3.25.1.jar diff --git a/.idea/runConfigurations/collect.xml b/.idea/runConfigurations/collect.xml new file mode 100644 index 00000000..b450c1a6 --- /dev/null +++ b/.idea/runConfigurations/collect.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/serve.xml b/.idea/runConfigurations/serve.xml new file mode 100644 index 00000000..e3fcf830 --- /dev/null +++ b/.idea/runConfigurations/serve.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/Makefile b/Makefile index e013c431..52bf2bb6 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,11 @@ # https://github.com/bvinc/go-sqlite-lite/issues/10#issuecomment-498539630 +# go get -u github.com/go-bindata/go-bindata/... + +assets: + go-bindata -o ./pkg/analyzer/sqlScript.go -pkg analyzer -prefix ./pkg/analyzer/sql ./pkg/analyzer/sql + build: lint go mod tidy make build-mac @@ -13,6 +18,7 @@ build-mac: build-linux: env GOOS=linux GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++ go build -ldflags='-s -w' -o dist/linux/report-aggregator ./ + XZ_OPT=-9 tar -cJf dist/linux-report-aggregator.tar.xz dist/linux/report-aggregator build-windows: env GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=/usr/local/bin/x86_64-w64-mingw32-gcc CXX=/usr/local/bin/x86_64-w64-mingw32-g++ go build -ldflags='-s -w' -o dist/windows/report-aggregator.exe ./ diff --git a/go.mod b/go.mod index a1294bd3..a28e6c12 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,14 @@ require ( github.com/didip/tollbooth v4.0.2+incompatible github.com/json-iterator/go v1.1.7 github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pkg/errors v0.8.1 // indirect github.com/rs/cors v1.7.0 github.com/tdewolff/minify/v2 v2.5.2 go.uber.org/atomic v1.4.0 // indirect - go.uber.org/multierr v1.1.0 // indirect + go.uber.org/multierr v1.2.0 // indirect go.uber.org/zap v1.10.0 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 // indirect ) diff --git a/go.sum b/go.sum index ac1a8b6c..e60c87fb 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -8,7 +9,9 @@ github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ij github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/develar/errors v0.9.0 h1:ftXOTwkajtgkUwLTw1iKG+mJwrUTvCp9Zr/Z6Y+rvMY= github.com/develar/errors v0.9.0/go.mod h1:zNbO3fZHcBjapJKbvUnvyaNrKGKkxgaL6C8Z7uNzQMc= github.com/didip/tollbooth v4.0.2+incompatible h1:fVSa33JzSz0hoh2NxpwZtksAzAgd7zjmGO20HCZtF4M= github.com/didip/tollbooth v4.0.2+incompatible/go.mod h1:A9b0665CE6l1KmzpDws2++elm/CsuWBMa5Jv4WY0PEY= @@ -22,10 +25,17 @@ github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2 h1:YocNLcTBdEd github.com/mcuadros/go-version v0.0.0-20190830083331-035f6764e8d2/go.mod h1:76rfSfYPWj01Z85hUf/ituArm797mNKcvINh1OlsZKo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= @@ -41,12 +51,18 @@ github.com/tdewolff/parse/v2 v2.3.9/go.mod h1:HansaqmN4I/U7L6/tUp0NcwT2tFO0F4EAW github.com/tdewolff/test v1.0.0/go.mod h1:DiQUlutnqlEvdvhSn2LPGy4TFwRauAaYDsL+683RNX4= github.com/tdewolff/test v1.0.4 h1:ih38SXuQJ32Hng5EtSW32xqEsVeMnPp6nNNRPhBBDE8= github.com/tdewolff/test v1.0.4/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4= +go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/analyzer/analyzer.go b/pkg/analyzer/analyzer.go index 72e21933..a170692c 100644 --- a/pkg/analyzer/analyzer.go +++ b/pkg/analyzer/analyzer.go @@ -27,10 +27,10 @@ type ReportAnalyzer struct { waitGroup sync.WaitGroup closeOnce sync.Once - minifier *minify.M - db *sqlite3.Conn - insertStatement *sqlite3.Stmt - hash hash.Hash + minifier *minify.M + db *sqlite3.Conn + putStatement *sqlite3.Stmt + hash hash.Hash machine string @@ -66,12 +66,6 @@ func CreateReportAnalyzer(dbPath string, machine string, analyzeContext context. machine: machine, } - analyzer.insertStatement, err = db.Prepare(`INSERT INTO report (id, machine, generated_time, metrics_version, metrics, raw_report) VALUES (?, ?, ?, ?, ?, ?)`) - if err != nil { - util.Close(db, logger) - return nil, err - } - go func() { for { select { @@ -156,12 +150,18 @@ func parseTime(report *model.Report) (*time.Time, error) { parsedTime, err := time.Parse(time.RFC1123Z, report.Generated) if err != nil { parsedTime, err = time.Parse(time.RFC1123, report.Generated) - if err != nil { - parsedTime, err = time.Parse("Jan 2, 2006, 3:04:05 PM MST", report.Generated) - if err != nil { - return nil, errors.WithStack(err) - } - } + } + + if err != nil { + parsedTime, err = time.Parse("Jan 2, 2006, 3:04:05 PM MST", report.Generated) + } + + if err != nil { + parsedTime, err = time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", report.Generated) + } + + if err != nil { + return nil, errors.WithStack(err) } return &parsedTime, nil } @@ -171,10 +171,10 @@ func (t *ReportAnalyzer) Close() error { close(t.input) }) - insertStatement := t.insertStatement - if insertStatement != nil { - util.Close(insertStatement, t.logger) - t.insertStatement = nil + putStatement := t.putStatement + if putStatement != nil { + util.Close(putStatement, t.logger) + t.putStatement = nil } db := t.db @@ -193,7 +193,7 @@ func (t *ReportAnalyzer) Done() <-chan struct{} { return t.waitChannel } -const metricsVersion = 1 +const metricsVersion = 2 func (t *ReportAnalyzer) doAnalyze(report *model.Report) error { t.waitGroup.Add(1) @@ -207,13 +207,13 @@ func (t *ReportAnalyzer) doAnalyze(report *model.Report) error { id := base64.RawURLEncoding.EncodeToString(t.hash.Sum(nil)) - isAlreadyProcessed, err := t.isReportAlreadyProcessed(id) + currentMetricsVersion, err := t.getMetricsVersion(id) if err != nil { return errors.WithStack(err) } logger := t.logger.With(zap.String("id", id), zap.String("generatedTime", time.Unix(report.GeneratedTime, 0).Format(time.RFC1123))) - if isAlreadyProcessed { + if currentMetricsVersion == metricsVersion { logger.Info("report already processed") return nil } @@ -228,26 +228,46 @@ func (t *ReportAnalyzer) doAnalyze(report *model.Report) error { return errors.WithStack(err) } - err = t.insertStatement.Exec(id, t.machine, report.GeneratedTime, metricsVersion, serializedMetrics, report.RawData) + statement := t.putStatement + if statement == nil { + statement, err = t.db.Prepare(`REPLACE INTO report (id, machine, generated_time, metrics_version, metrics, raw_report) VALUES (?, ?, ?, ?, ?, ?)`) + if err != nil { + return err + } + + t.putStatement = statement + } + + err = statement.Exec(id, t.machine, report.GeneratedTime, metricsVersion, serializedMetrics, report.RawData) if err != nil { return errors.WithStack(err) } - logger.Info("new report added") + if currentMetricsVersion >= 0 && currentMetricsVersion == metricsVersion { + logger.Info("report metrics updated", zap.Int("oldMetricsVersion", currentMetricsVersion), zap.Int("newMetricsVersion", metricsVersion)) + } else { + logger.Info("new report added") + } return nil } -func (t *ReportAnalyzer) isReportAlreadyProcessed(id string) (bool, error) { +func (t *ReportAnalyzer) getMetricsVersion(id string) (int, error) { stmt, err := t.db.Prepare(`SELECT metrics_version FROM report WHERE id = ?`, id) if err != nil { - return false, errors.WithStack(err) + return 1, errors.WithStack(err) } defer util.Close(stmt, t.logger) hasRow, err := stmt.Step() if err != nil { - return false, errors.WithStack(err) + return -1, errors.WithStack(err) } - return hasRow, nil + + if hasRow { + result, _, err := stmt.ColumnInt(0) + return result, errors.WithStack(err) + } + + return -1, nil } diff --git a/pkg/analyzer/db.go b/pkg/analyzer/db.go index 66932420..8abdfc03 100644 --- a/pkg/analyzer/db.go +++ b/pkg/analyzer/db.go @@ -20,12 +20,9 @@ func prepareDatabaseFile(filePath string, logger *zap.Logger) error { dirStat, err := os.Stat(dir) if err == nil && dirStat.IsDir() { // dir exists - check file and copy if needed (for backup purposes) - _, err := os.Stat(filePath) - if err == nil { - err = createBackup(filePath, dir, logger) - if err != nil { - return err - } + err = createBackup(filePath, dir, logger) + if err != nil { + return errors.WithStack(err) } } else { err := os.MkdirAll(dir, 0777) @@ -43,55 +40,92 @@ func prepareDatabaseFile(filePath string, logger *zap.Logger) error { } func createBackup(filePath string, dirPath string, logger *zap.Logger) error { - fileBaseName := filepath.Base(filePath) - backupFile, err := os.Create(filepath.Join(dirPath, strings.TrimSuffix(fileBaseName, filepath.Ext(fileBaseName))+"-backup-"+time.Now().Format(time.RFC822)+".sqlite")) + oldConnection, err := sqlite3.Open(filePath, sqlite3.OPEN_READWRITE) if err != nil { + sqlErr, ok := err.(*sqlite3.Error) + if ok && sqlErr.Code() == sqlite3.CANTOPEN { + // file is new, no need to backup + return nil + } return errors.WithStack(err) } - defer util.Close(backupFile, logger) + defer util.Close(oldConnection, logger) - file, err := os.Open(filePath) + fileBaseName := filepath.Base(filePath) + newFilePath := filepath.Join(dirPath, strings.TrimSuffix(fileBaseName, filepath.Ext(fileBaseName))+"-backup-"+time.Now().Format("Jan-_2_15-04-05")+".sqlite") + + newConnection, err := sqlite3.Open(newFilePath) if err != nil { return errors.WithStack(err) } - defer util.Close(file, logger) + defer util.Close(newConnection, logger) - _, err = io.Copy(file, backupFile) + backup, err := oldConnection.Backup("main", newConnection, "main") if err != nil { return errors.WithStack(err) } + + defer util.Close(backup, logger) + + err = backup.Step(-1) + if err != nil && err != io.EOF { + return errors.WithStack(err) + } + return nil } +const toolDbVersion = 1 + func prepareDatabase(dbPath string, logger *zap.Logger) (*sqlite3.Conn, error) { db, err := sqlite3.Open(dbPath) if err != nil { return nil, errors.WithStack(err) } - db.BusyTimeout(5 * time.Second) - - err = db.Exec(` -create table report -( - id string not null primary key, - machine string not null, - generated_time int not null, - metrics_version int not null, - metrics string not null, - raw_report string not null -); + isPrepared := false -create index machine_index on report(machine) + defer func() { + if !isPrepared { + util.Close(db, logger) + } + }() -`) + db.BusyTimeout(5 * time.Second) + dbVersion, err := readDbVersion(db, logger) if err != nil { - util.Close(db, logger) return nil, err } + if dbVersion == 0 { + err = db.Exec(string(MustAsset("create-db.sql"))) + if err != nil { + return nil, err + } + } else if dbVersion > toolDbVersion { + return nil, errors.Errorf("Database version %d is not supported (tool is outdated)", dbVersion) + } + + isPrepared = true return db, nil } + +func readDbVersion(db *sqlite3.Conn, logger *zap.Logger) (int, error) { + statement, err := db.Prepare("PRAGMA user_version") + if err != nil { + return -1, errors.WithStack(err) + } + + defer util.Close(statement, logger) + + _, err = statement.Step() + if err != nil { + return -1, errors.WithStack(err) + } + + dbVersion, _, err := statement.ColumnInt(0) + return dbVersion, errors.WithStack(err) +} diff --git a/pkg/analyzer/metrics.go b/pkg/analyzer/metrics.go index 188b3007..66b4d25f 100644 --- a/pkg/analyzer/metrics.go +++ b/pkg/analyzer/metrics.go @@ -11,9 +11,9 @@ func (t *ReportAnalyzer) computeMetrics(report *model.Report, logger *zap.Logger Bootstrap: -1, Splash: -1, - AppInitPreparation: -1, - AppInit: -1, - PluginDescriptorsLoading: -1, + AppInitPreparation: -1, + AppInit: -1, + PluginDescriptorLoading: -1, AppComponentCreation: -1, ProjectComponentCreation: -1, @@ -25,7 +25,7 @@ func (t *ReportAnalyzer) computeMetrics(report *model.Report, logger *zap.Logger return nil } - // v < 12: PluginDescriptorsLoading can be or in MainActivities, or in PrepareAppInitActivities + // v < 12: PluginDescriptorLoading can be or in MainActivities, or in PrepareAppInitActivities for _, activity := range report.MainActivities { switch activity.Name { @@ -36,13 +36,22 @@ func (t *ReportAnalyzer) computeMetrics(report *model.Report, logger *zap.Logger metrics.AppInitPreparation = activity.Duration case "app initialization": metrics.AppInit = activity.Duration + + case "plugin descriptor loading": + metrics.PluginDescriptorLoading = activity.Duration case "plugin descriptors loading": - metrics.PluginDescriptorsLoading = activity.Duration + metrics.PluginDescriptorLoading = activity.Duration case "app component creation": metrics.AppComponentCreation = activity.Duration + case "app components creation": + metrics.AppComponentCreation = activity.Duration + case "project component creation": metrics.ProjectComponentCreation = activity.Duration + case "project components creation": + metrics.ProjectComponentCreation = activity.Duration + case "module loading": metrics.ModuleLoading = activity.Duration } @@ -52,7 +61,7 @@ func (t *ReportAnalyzer) computeMetrics(report *model.Report, logger *zap.Logger for _, activity := range report.PrepareAppInitActivities { switch activity.Name { case "plugin descriptors loading": - metrics.PluginDescriptorsLoading = activity.Start + metrics.PluginDescriptorLoading = activity.Start case "splash initialization": metrics.Splash = activity.Start } @@ -65,11 +74,16 @@ func (t *ReportAnalyzer) computeMetrics(report *model.Report, logger *zap.Logger } } + if metrics.Splash == -1 && version.Compare(report.Version, "6", ">=") { + logger.Info("metric 'splash' not found") + } + if metrics.Bootstrap == -1 { - logRequiredMetricNotFound(logger, "bootstrap") - return nil + if version.Compare(report.Version, "6", ">=") { + logRequiredMetricNotFound(logger, "bootstrap") + } } - if metrics.PluginDescriptorsLoading == -1 { + if metrics.PluginDescriptorLoading == -1 { logRequiredMetricNotFound(logger, "pluginDescriptorsLoading") return nil } diff --git a/pkg/analyzer/sql/create-db.sql b/pkg/analyzer/sql/create-db.sql new file mode 100644 index 00000000..aad03679 --- /dev/null +++ b/pkg/analyzer/sql/create-db.sql @@ -0,0 +1,13 @@ +create table report +( + id string not null primary key, + machine string not null, + generated_time int not null, + metrics_version int not null, + metrics string not null, + raw_report string not null +); + +create index machine_index on report (machine); + +pragma user_version=1 \ No newline at end of file diff --git a/pkg/analyzer/sqlScript.go b/pkg/analyzer/sqlScript.go new file mode 100644 index 00000000..417ce756 --- /dev/null +++ b/pkg/analyzer/sqlScript.go @@ -0,0 +1,244 @@ +// Code generated for package analyzer by go-bindata DO NOT EDIT. (@generated) +// sources: +// pkg/analyzer/sql/create-db.sql +package analyzer + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +func bindataRead(data []byte, name string) ([]byte, error) { + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + + var buf bytes.Buffer + _, err = io.Copy(&buf, gz) + clErr := gz.Close() + + if err != nil { + return nil, fmt.Errorf("Read %q: %v", name, err) + } + if clErr != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +type asset struct { + bytes []byte + info os.FileInfo +} + +type bindataFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time +} + +// Name return file name +func (fi bindataFileInfo) Name() string { + return fi.name +} + +// Size return file size +func (fi bindataFileInfo) Size() int64 { + return fi.size +} + +// Mode return file mode +func (fi bindataFileInfo) Mode() os.FileMode { + return fi.mode +} + +// Mode return file modify time +func (fi bindataFileInfo) ModTime() time.Time { + return fi.modTime +} + +// IsDir return file whether a directory +func (fi bindataFileInfo) IsDir() bool { + return fi.mode&os.ModeDir != 0 +} + +// Sys return file is sys mode +func (fi bindataFileInfo) Sys() interface{} { + return nil +} + +var _createDbSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x7c\x90\xc1\xce\x82\x40\x0c\x84\xef\xfb\x14\x73\xe4\x4f\xfe\x8b\x67\xe3\xb3\x6c\x2a\x34\xd8\xc8\x76\x49\xb7\xa8\xbc\xbd\x41\x96\x03\x89\xd2\x5b\xdb\x6f\xd2\xce\xb4\xc6\xe4\x0c\xa7\xeb\xc0\x30\x1e\xb3\x79\x68\x02\x00\x48\x87\x5d\x15\x37\xd1\x1e\x9a\x1d\x3a\x0d\x03\x46\x93\x44\x36\xe3\xce\xf3\xff\x47\x90\xa8\xbd\x89\xf2\x2f\xc1\x0a\xf5\xac\x6c\xe4\xdc\x45\x97\xc4\x80\xa8\x2f\xe3\x3d\x94\xd8\x4d\xda\x12\x1f\x6c\x45\xb2\x1e\x42\xc7\xe7\x8c\x9e\x71\x75\xf5\x15\x0a\x7f\xe7\x10\x6a\x04\xa2\x1d\xbf\x36\x13\x71\xed\xb2\xd6\x4c\xd0\xd4\xc5\x22\x18\x8d\xfa\x44\x98\x0a\xdb\xf6\xe2\xe5\xf4\x0e\x00\x00\xff\xff\xf9\x94\xd3\xae\x49\x01\x00\x00") + +func createDbSqlBytes() ([]byte, error) { + return bindataRead( + _createDbSql, + "create-db.sql", + ) +} + +func createDbSql() (*asset, error) { + bytes, err := createDbSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "create-db.sql", size: 329, mode: os.FileMode(420), modTime: time.Unix(1569572906, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +// Asset loads and returns the asset for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func Asset(name string) ([]byte, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) + } + return a.bytes, nil + } + return nil, fmt.Errorf("Asset %s not found", name) +} + +// MustAsset is like Asset but panics when Asset would return an error. +// It simplifies safe initialization of global variables. +func MustAsset(name string) []byte { + a, err := Asset(name) + if err != nil { + panic("asset: Asset(" + name + "): " + err.Error()) + } + + return a +} + +// AssetInfo loads and returns the asset info for the given name. +// It returns an error if the asset could not be found or +// could not be loaded. +func AssetInfo(name string) (os.FileInfo, error) { + cannonicalName := strings.Replace(name, "\\", "/", -1) + if f, ok := _bindata[cannonicalName]; ok { + a, err := f() + if err != nil { + return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) + } + return a.info, nil + } + return nil, fmt.Errorf("AssetInfo %s not found", name) +} + +// AssetNames returns the names of the assets. +func AssetNames() []string { + names := make([]string, 0, len(_bindata)) + for name := range _bindata { + names = append(names, name) + } + return names +} + +// _bindata is a table, holding each asset generator, mapped to its name. +var _bindata = map[string]func() (*asset, error){ + "create-db.sql": createDbSql, +} + +// AssetDir returns the file names below a certain +// directory embedded in the file by go-bindata. +// For example if you run go-bindata on data/... and data contains the +// following hierarchy: +// data/ +// foo.txt +// img/ +// a.png +// b.png +// then AssetDir("data") would return []string{"foo.txt", "img"} +// AssetDir("data/img") would return []string{"a.png", "b.png"} +// AssetDir("foo.txt") and AssetDir("notexist") would return an error +// AssetDir("") will return []string{"data"}. +func AssetDir(name string) ([]string, error) { + node := _bintree + if len(name) != 0 { + cannonicalName := strings.Replace(name, "\\", "/", -1) + pathList := strings.Split(cannonicalName, "/") + for _, p := range pathList { + node = node.Children[p] + if node == nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + } + } + if node.Func != nil { + return nil, fmt.Errorf("Asset %s not found", name) + } + rv := make([]string, 0, len(node.Children)) + for childName := range node.Children { + rv = append(rv, childName) + } + return rv, nil +} + +type bintree struct { + Func func() (*asset, error) + Children map[string]*bintree +} + +var _bintree = &bintree{nil, map[string]*bintree{ + "create-db.sql": &bintree{createDbSql, map[string]*bintree{}}, +}} + +// RestoreAsset restores an asset under the given directory +func RestoreAsset(dir, name string) error { + data, err := Asset(name) + if err != nil { + return err + } + info, err := AssetInfo(name) + if err != nil { + return err + } + err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) + if err != nil { + return err + } + err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) + if err != nil { + return err + } + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil +} + +// RestoreAssets restores an asset under the given directory recursively +func RestoreAssets(dir, name string) error { + children, err := AssetDir(name) + // File + if err != nil { + return RestoreAsset(dir, name) + } + // Dir + for _, child := range children { + err = RestoreAssets(dir, filepath.Join(name, child)) + if err != nil { + return err + } + } + return nil +} + +func _filePath(dir, name string) string { + cannonicalName := strings.Replace(name, "\\", "/", -1) + return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) +} diff --git a/pkg/ideaLog/collector.go b/pkg/ideaLog/collector.go index 91dbd713..602654d2 100644 --- a/pkg/ideaLog/collector.go +++ b/pkg/ideaLog/collector.go @@ -18,11 +18,11 @@ import ( func ConfigureCollectFromDirCommand(app *kingpin.Application, log *zap.Logger) { command := app.Command("collect", "Collect reports from idea.log files.") - dir := command.Flag("dir", "The input directory.").Short('i').Required().String() + dirs := command.Flag("dir", "The input directory.").Short('i').Required().Strings() dbPath := command.Flag("db", "The output SQLite database file.").Short('o').Required().String() machine := command.Flag("machine", "The name of machine to associate report with.").Short('m').Required().String() command.Action(func(context *kingpin.ParseContext) error { - err := collectFromDir(*dir, *dbPath, *machine, log) + err := collectFromDirs(*dirs, *dbPath, *machine, log) if err != nil { return err } @@ -31,7 +31,7 @@ func ConfigureCollectFromDirCommand(app *kingpin.Application, log *zap.Logger) { }) } -func collectFromDir(dir string, dbPath string, machine string, logger *zap.Logger) error { +func collectFromDirs(dirs []string, dbPath string, machine string, logger *zap.Logger) error { taskContext, cancel := context.WithCancel(context.Background()) defer cancel() @@ -42,11 +42,6 @@ func collectFromDir(dir string, dbPath string, machine string, logger *zap.Logge cancel() }() - files, err := filepath.Glob(dir + "/idea*.log*") - if err != nil { - return errors.WithStack(err) - } - reportAnalyzer, err := analyzer.CreateReportAnalyzer(dbPath, machine, taskContext, logger) if err != nil { return err @@ -63,12 +58,8 @@ func collectFromDir(dir string, dbPath string, machine string, logger *zap.Logge defer util.Close(reportAnalyzer, logger) - for _, file := range files { - if taskContext.Err() != nil { - return nil - } - - err := collectFromLogFile(file, logger, reportAnalyzer, taskContext) + for _, dir := range dirs { + err = collectFromDir(dir, taskContext, logger, reportAnalyzer) if err != nil { return err } @@ -88,6 +79,25 @@ func collectFromDir(dir string, dbPath string, machine string, logger *zap.Logge } } +func collectFromDir(dir string, taskContext context.Context, logger *zap.Logger, reportAnalyzer *analyzer.ReportAnalyzer) error { + files, err := filepath.Glob(dir + "/idea*.log*") + if err != nil { + return errors.WithStack(err) + } + + for _, file := range files { + if taskContext.Err() != nil { + return nil + } + + err := collectFromLogFile(file, logger, reportAnalyzer, taskContext) + if err != nil { + return err + } + } + return nil +} + func collectFromLogFile(filePath string, log *zap.Logger, reportAnalyzer *analyzer.ReportAnalyzer, taskContext context.Context) error { file, err := os.Open(filePath) if err != nil { diff --git a/pkg/model/model.go b/pkg/model/model.go index 0fa13612..caf9c360 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -25,7 +25,8 @@ type TraceEvent struct { // in microseconds Timestamp int `json:"ts"` - Thread string `json:"tid"` + // in old reports (v10) can be int instead of string + //Thread string `json:"tid"` Category string `json:"cat"` } @@ -40,14 +41,15 @@ type Activity struct { } // computed metrics +// prefix i_ is used for instant events type Metrics struct { // value - not duration, but start, because it is instant event and not duration event - Splash int `json:"splash"` + Splash int `json:"i_splash"` - Bootstrap int `json:"bootstrap"` - AppInitPreparation int `json:"appInitPreparation"` - AppInit int `json:"appInit"` - PluginDescriptorsLoading int `json:"pluginDescriptorsLoading"` + Bootstrap int `json:"bootstrap"` + AppInitPreparation int `json:"appInitPreparation"` + AppInit int `json:"appInit"` + PluginDescriptorLoading int `json:"pluginDescriptorLoading"` ProjectComponentCreation int `json:"projectComponentCreation"` AppComponentCreation int `json:"appComponentCreation"` diff --git a/pkg/server/server.go b/pkg/server/server.go index 765f6d68..1ada8784 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -125,7 +125,13 @@ type StatsServer struct { } func (t *StatsServer) handleStatsRequest(w http.ResponseWriter, r *http.Request) { - statement, err := t.db.Prepare("select machine, generated_time, metrics from report order by machine, generated_time") + //noinspection SqlResolve + statement, err := t.db.Prepare(` +select machine, generated_time, metrics, + json_extract(raw_report, '$.productCode') as productCode, + json_extract(raw_report, '$.build') as build +from report order by machine, generated_time + `) if err != nil { t.logger.Error("cannot query", zap.Error(err)) http.Error(w, err.Error(), 503) @@ -156,7 +162,9 @@ func (t *StatsServer) handleStatsRequest(w http.ResponseWriter, r *http.Request) var machine sqlite3.RawString var generatedTime int64 var metrics sqlite3.RawString - err = statement.Scan(&machine, &generatedTime, &metrics) + var productCode string + var build string + err = statement.Scan(&machine, &generatedTime, &metrics, &productCode, &build) if err != nil { t.logger.Error("cannot query", zap.Error(err)) http.Error(w, err.Error(), 503) @@ -194,6 +202,17 @@ func (t *StatsServer) handleStatsRequest(w http.ResponseWriter, r *http.Request) // seconds to milliseconds jsonWriter.WriteInt64(generatedTime * 1000) jsonWriter.WriteMore() + + if len(productCode) != 0 { + jsonWriter.WriteObjectField("_p") + jsonWriter.WriteString(productCode) + jsonWriter.WriteMore() + + jsonWriter.WriteObjectField("_v") + jsonWriter.WriteString(build) + jsonWriter.WriteMore() + } + // skip first '{' jsonWriter.WriteRaw(string(metrics[1:])) }