Skip to content

Commit

Permalink
Merge 2d052b9 into 52a213d
Browse files Browse the repository at this point in the history
  • Loading branch information
talal committed Jul 4, 2019
2 parents 52a213d + 2d052b9 commit 94c62ff
Show file tree
Hide file tree
Showing 33 changed files with 3,260 additions and 5 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,10 @@ All components receive configuration via environment variables. The following va
| Variable | Default | Explanation |
| -------- | ------- | ----------- |
| `CASTELLUM_ASSET_MANAGERS` | *(required)* | A comma-separated list of all asset managers that can be enabled. This configures what kinds of assets Castellum can handle. See [`docs/asset-managers/`](./docs/asset-managers/) for which asset managers exist. |
| `CASTELLUM_DB_URI` | *(required)* | A [libpq connection URI][pq-uri] that locates the Limes database. The non-URI "connection string" format is not allowed; it must be a URI. |
| `CASTELLUM_DB_URI` | *(required)* | A [libpq connection URI][pq-uri] that locates the Castellum database. The non-URI "connection string" format is not allowed; it must be a URI. |
| `CASTELLUM_HTTP_LISTEN_ADDRESS` | `:8080` | Listen address for the internal HTTP server. For `castellum observer/worker`, this just exposes Prometheus metrics on `/metrics`. For `castelum api`, this also exposes [the REST API](./docs/api-spec.md). |
| `CASTELLUM_OSLO_POLICY_PATH` | *(required)* | Path to the `policy.json` file for this service. See [*Oslo policy*](#oslo-policy) for details. |
| `CASTELLUM_SENTRY_DSN` | *(optional)* | DSN for your Sentry project. If this variable is configured then Castellum will report failed backend operations to Sentry. |
| `OS_...` | *(required)* | A full set of OpenStack auth environment variables for Castellum's service user. See [documentation for openstackclient][os-env] for details. |

### Oslo policy
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.12

require (
github.com/databus23/goslo.policy v0.0.0-20170317131957-3ae74dd07ebf
github.com/getsentry/sentry-go v0.1.1
github.com/golang-migrate/migrate v3.5.4+incompatible // indirect
github.com/gophercloud/gophercloud v0.1.0
github.com/gophercloud/utils v0.0.0-20190313033024-0bcc8e728cb5
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/databus23/goslo.policy v0.0.0-20170317131957-3ae74dd07ebf h1:9V+7mLoEGsdQRnHH7idSfsNcvYFBF3y41AgK3r6whzA=
github.com/databus23/goslo.policy v0.0.0-20170317131957-3ae74dd07ebf/go.mod h1:l9nkDMRWNXo519AKv09j14YhyHW1YsAKAa1kG6R1p3Y=
github.com/getsentry/sentry-go v0.1.1 h1:5jcHHCJ+EUK+X8aoDbJUzZU1kG4ZFk2xX15BKv29v10=
github.com/getsentry/sentry-go v0.1.1/go.mod h1:2QfSdvxz4IZGyB5izm1TtADFhlhfj1Dcesrg8+A/T9Y=
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
github.com/golang-migrate/migrate v3.5.4+incompatible h1:R7OzwvCJTCgwapPCiX6DyBiu2czIUMDCB118gFTKTUA=
github.com/golang-migrate/migrate v3.5.4+incompatible/go.mod h1:IsVUlFN5puWOmXrqjgGUfIRIbU7mr8oNBE2tyERd9Wk=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
Expand All @@ -26,6 +29,8 @@ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/pingcap/errors v0.11.1/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
Expand Down
2 changes: 1 addition & 1 deletion internal/core/assets.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
"github.com/sapcc/castellum/internal/db"
)

//AssetStatus shows the current state of an asset. It is returned by AssetManager.GetProjectAssetStatus().
//AssetStatus shows the current state of an asset. It is returned by AssetManager.GetAssetStatus().
type AssetStatus struct {
Size uint64
UsagePercent uint32
Expand Down
14 changes: 13 additions & 1 deletion internal/tasks/asset_resize.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,19 @@ func (c Context) ExecuteNextResize() (targetAssetType db.AssetType, returnedErro
outcome := db.OperationOutcomeSucceeded
errorMessage := ""
if err != nil {
logg.Error("cannot resize %s %s to size %d: %s", res.AssetType, asset.UUID, op.NewSize, err.Error())
e := setAssetSizeError{
scopeUUID: res.ScopeUUID,
assetType: string(res.AssetType),
assetUUID: asset.UUID,
newSize: op.NewSize,
inner: err,
}
logg.Error(e.Error())

if c.SentryHub != nil {
captureSentryException(c.SentryHub, e)
}

outcome = db.OperationOutcomeFailed
errorMessage = err.Error()
}
Expand Down
11 changes: 10 additions & 1 deletion internal/tasks/asset_scrape.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,16 @@ func (c Context) ScrapeNextAsset(assetType db.AssetType, maxCheckedAt time.Time)
if dbErr != nil {
return dbErr
}
return fmt.Errorf("cannot query status of %s %s: %s", assetType, asset.UUID, err.Error())
e := getAssetStatusError{
scopeUUID: res.ScopeUUID,
assetType: string(assetType),
assetUUID: asset.UUID,
inner: err,
}
if c.SentryHub != nil {
captureSentryException(c.SentryHub, e)
}
return e
}

//update asset attributes - We have four separate cases here, which
Expand Down
15 changes: 15 additions & 0 deletions internal/tasks/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ package tasks
import (
"time"

"github.com/getsentry/sentry-go"
"github.com/sapcc/castellum/internal/core"
"gopkg.in/gorp.v2"
)
Expand All @@ -38,9 +39,23 @@ type Context struct {
//When Blocker is not nil, tasks that support concurrent operation will
//withhold operations until this channel is closed.
Blocker <-chan struct{}

//A local Sentry Hub isolated from the global Sentry namespace.
SentryHub *sentry.Hub
}

//ApplyDefaults injects the regular runtime dependencies into this Context.
func (c *Context) ApplyDefaults() {
c.TimeNow = time.Now
}

//CloneSentryHub clones the current Sentry Hub from the global namespace and
//assigns it to this Context.
//
//This function needs to be called separately for each goroutine to avoid
//overwriting Sentry's Scope.
func (c *Context) CloneSentryHub() {
if sendEventsToSentry {
c.SentryHub = sentry.CurrentHub().Clone()
}
}
123 changes: 123 additions & 0 deletions internal/tasks/log.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/******************************************************************************
*
* Copyright 2019 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/

package tasks

import (
"fmt"
"os"

"github.com/getsentry/sentry-go"
"github.com/sapcc/go-bits/logg"
)

//sendEventsToSentry tells whether events should be sent to a Sentry server.
var sendEventsToSentry bool

func init() {
dsn := os.Getenv("CASTELLUM_SENTRY_DSN")
sendEventsToSentry = (dsn != "")
if sendEventsToSentry {
err := sentry.Init(sentry.ClientOptions{Dsn: dsn})
if err != nil {
logg.Error("Sentry initialization failed: %s", err.Error())
}
}
}

//sentryException is the interface that different error types must implement
//in order to generate custom context information for a Sentry event.
type sentryException interface {
generateTags() map[string]string
Error() string
}

//captureSentryException is a convenient wrapper around sentry.CaptureException().
//It takes a local *sentry.Hub and generates the relevant tags for the
//custom error type before sending the event to a Sentry server.
func captureSentryException(hub *sentry.Hub, se sentryException) {
hub.WithScope(func(scope *sentry.Scope) {
scope.SetTags(se.generateTags())
hub.CaptureException(se)
})
}

//listAssetsError contains parameters for creating the respective Sentry event.
type listAssetsError struct {
scopeUUID string
assetType string
inner error
}

//Error implements the tasks.sentryException interface.
func (e listAssetsError) Error() string {
return fmt.Sprintf("cannot list %s assets in project %s: %s", e.assetType, e.scopeUUID, e.inner.Error())
}

//generateTags implements the tasks.sentryException interface.
func (e listAssetsError) generateTags() map[string]string {
return map[string]string{
"scope_uuid": e.scopeUUID,
"asset_type": e.assetType,
}
}

//getAssetStatusError contains parameters for creating the respective Sentry event.
type getAssetStatusError struct {
scopeUUID string
assetType string
assetUUID string
inner error
}

//Error implements the tasks.sentryException interface.
func (e getAssetStatusError) Error() string {
return fmt.Sprintf("cannot query status of %s %s: %s", e.assetType, e.assetUUID, e.inner.Error())
}

//generateTags implements the tasks.sentryException interface.
func (e getAssetStatusError) generateTags() map[string]string {
return map[string]string{
"scope_uuid": e.scopeUUID,
"asset_type": e.assetType,
"asset_uuid": e.assetUUID,
}
}

//setAssetSizeError contains parameters for creating the respective Sentry event.
type setAssetSizeError struct {
scopeUUID string
assetType string
assetUUID string
newSize uint64
inner error
}

//generateTags implements the tasks.sentryException interface.
func (e setAssetSizeError) generateTags() map[string]string {
return map[string]string{
"scope_uuid": e.scopeUUID,
"asset_type": e.assetType,
"asset_uuid": e.assetUUID,
}
}

//Error implements the tasks.sentryException interface.
func (e setAssetSizeError) Error() string {
return fmt.Sprintf("cannot resize %s %s to size %d: %s", e.assetType, e.assetUUID, e.newSize, e.inner.Error())
}
19 changes: 18 additions & 1 deletion internal/tasks/resource_scrape.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,15 @@ func (c Context) ScrapeNextResource(assetType db.AssetType, maxScrapedAt time.Ti
//check which assets exist in this resource in OpenStack
assetUUIDs, err := manager.ListAssets(res)
if err != nil {
return fmt.Errorf("cannot list %s assets in project %s: %s", assetType, res.ScopeUUID, err.Error())
e := listAssetsError{
scopeUUID: res.ScopeUUID,
assetType: string(assetType),
inner: err,
}
if c.SentryHub != nil {
captureSentryException(c.SentryHub, e)
}
return e
}
logg.Debug("scraped %d assets for %s resource for project %s", len(assetUUIDs), assetType, res.ScopeUUID)
isExistingAsset := make(map[string]bool, len(assetUUIDs))
Expand Down Expand Up @@ -133,6 +141,15 @@ func (c Context) ScrapeNextResource(assetType db.AssetType, maxScrapedAt time.Ti

err = c.DB.Insert(&dbAsset)
if err != nil {
e := getAssetStatusError{
scopeUUID: res.ScopeUUID,
assetType: string(assetType),
assetUUID: assetUUID,
inner: err,
}
if c.SentryHub != nil {
captureSentryException(c.SentryHub, e)
}
return err
}
}
Expand Down
3 changes: 3 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,11 @@ func runObserver(dbi *gorp.DbMap, team core.AssetManagerTeam, httpListenAddr str

for _, manager := range team {
for _, assetType := range manager.AssetTypes() {
c.CloneSentryHub()
go jobLoop(func() error {
return c.ScrapeNextResource(assetType, time.Now().Add(-30*time.Minute))
})
c.CloneSentryHub()
go jobLoop(func() error {
return c.ScrapeNextAsset(assetType, time.Now().Add(-5*time.Minute))
})
Expand Down Expand Up @@ -240,6 +242,7 @@ func runWorker(dbi *gorp.DbMap, team core.AssetManagerTeam, httpListenAddr strin
c := tasks.Context{DB: dbi, Team: team}
c.ApplyDefaults()
c.InitializeResizingCounters()
c.CloneSentryHub()

go jobLoop(func() error {
_, err := c.ExecuteNextResize()
Expand Down
12 changes: 12 additions & 0 deletions vendor/github.com/getsentry/sentry-go/.craft.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions vendor/github.com/getsentry/sentry-go/.gitignore

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions vendor/github.com/getsentry/sentry-go/.golangci.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 28 additions & 0 deletions vendor/github.com/getsentry/sentry-go/.travis.yml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 94c62ff

Please sign in to comment.