Merge branch 'master' into backend_plugins

* master:
  changelog: adds note about closing #10131
  Explicitly specify default region in CloudWatch datasource (#9440)
  wait for all sub routines to finish
  changelog: adds ntoe about closing #10111
  postgres: change $__timeGroup macro to include "AS time" column alias (#10119)
  fixes broken test
  Solves problem with Github authentication restriction by organization membership when the organization's access policy is set to "Access restricted". "Access restricted" policy should not stop user to authenticate.

CHANGELOG.md

@@ -33,9 +33,14 @@ From `/etc/grafana/datasources` to `/etc/grafana/provisioning/datasources` when
 * **Dashboard**: Make it possible to start dashboards from search and dashboard list panel [#1871](https://github.com/grafana/grafana/issues/1871)
 * **Annotations**: Posting annotations now return the id of the annotation [#9798](https://github.com/grafana/grafana/issues/9798)
 * **Systemd**: Use systemd notification ready flag [#10024](https://github.com/grafana/grafana/issues/10024), thx [@jgrassler](https://github.com/jgrassler)
+* **Github**: Use organizations_url provided from github to verify user belongs in org. [#10111](https://github.com/grafana/grafana/issues/10111), thx 
+[@adiletmaratov](https://github.com/adiletmaratov)
+* **Backend**: Fixed bug where Grafana exited before all sub routines where finished [#10131](https://github.com/grafana/grafana/issues/10131)
+
 ## Tech
 * **RabbitMq**: Remove support for publishing events to RabbitMQ [#9645](https://github.com/grafana/grafana/issues/9645)
 
+
 ## Fixes
 * **Sensu**: Send alert message to sensu output [#9551](https://github.com/grafana/grafana/issues/9551), thx [@cjchand](https://github.com/cjchand)
 * **Singlestat**: suppress error when result contains no datapoints [#9636](https://github.com/grafana/grafana/issues/9636), thx [@utkarshcmu](https://github.com/utkarshcmu)

docs/sources/features/datasources/cloudwatch.md

@@ -78,11 +78,14 @@ CloudWatch Datasource Plugin provides the following queries you can specify in t
 edit view. They allow you to fill a variable's options list with things like `region`, `namespaces`, `metric names`
 and `dimension keys/values`.
 
+In place of `region` you can specify `default` to use the default region configured in the datasource for the query,
+e.g. `metrics(AWS/DynamoDB, default)` or `dimension_values(default, ..., ..., ...)`.
+
 Name | Description
 ------- | --------
 *regions()* | Returns a list of regions AWS provides their service.
 *namespaces()* | Returns a list of namespaces CloudWatch support.
-*metrics(namespace, [region])* | Returns a list of metrics in the namespace. (specify region for custom metrics)
+*metrics(namespace, [region])* | Returns a list of metrics in the namespace. (specify region or use "default" for custom metrics)
 *dimension_keys(namespace)* | Returns a list of dimension keys in the namespace.
 *dimension_values(region, namespace, metric, dimension_key)* | Returns a list of dimension values matching the specified `region`, `namespace`, `metric` and `dimension_key`.
 *ebs_volume_ids(region, instance_id)* | Returns a list of volume ids matching the specified `region`, `instance_id`.

docs/sources/features/datasources/postgres.md

@@ -48,7 +48,7 @@ Macro example | Description
 *$__timeFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name. For example, *extract(epoch from dateColumn) BETWEEN 1494410783 AND 1494497183*
 *$__timeFrom()* | Will be replaced by the start of the currently active time selection. For example, *to_timestamp(1494410783)*
 *$__timeTo()* | Will be replaced by the end of the currently active time selection. For example, *to_timestamp(1494497183)*
-*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *(extract(epoch from "dateColumn")/300)::bigint*300*
+*$__timeGroup(dateColumn,'5m')* | Will be replaced by an expression usable in GROUP BY clause. For example, *(extract(epoch from dateColumn)/300)::bigint*300 AS time*
 *$__unixEpochFilter(dateColumn)* | Will be replaced by a time range filter using the specified column name with times represented as unix timestamp. For example, *dateColumn > 1494410783 AND dateColumn < 1494497183*
 *$__unixEpochFrom()* | Will be replaced by the start of the currently active time selection as unix timestamp. For example, *1494410783*
 *$__unixEpochTo()* | Will be replaced by the end of the currently active time selection as unix timestamp. For example, *1494497183*
@@ -94,7 +94,7 @@ Example with `metric` column
 
 ```sql
 SELECT
-  $__timeGroup(time_date_time,'5m') as time,
+  $__timeGroup(time_date_time,'5m'),
   min(value_double),
   'min' as metric
 FROM test_data
@@ -107,7 +107,7 @@ Example with multiple columns:
 
 ```sql
 SELECT
-  $__timeGroup(time_date_time,'5m') as time,
+  $__timeGroup(time_date_time,'5m'),
   min(value_double) as min_value,
   max(value_double) as max_value
 FROM test_data

pkg/api/http_server.go

@@ -95,7 +95,7 @@ func (hs *HttpServer) Start(ctx context.Context) error {
 
 func (hs *HttpServer) Shutdown(ctx context.Context) error {
 	err := hs.httpSrv.Shutdown(ctx)
-	hs.log.Info("stopped http server")
+	hs.log.Info("Stopped HTTP server")
 	return err
 }
 

pkg/cmd/grafana-server/main.go

@@ -14,8 +14,8 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 
+	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/metrics"
-	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/setting"
 
 	_ "github.com/grafana/grafana/pkg/services/alerting/conditions"
@@ -40,9 +40,6 @@ var homePath = flag.String("homepath", "", "path to grafana install/home path, d
 var pidFile = flag.String("pidfile", "", "path to pid file")
 var exitChan = make(chan int)
 
-func init() {
-}
-
 func main() {
 	v := flag.Bool("v", false, "prints current version and exits")
 	profile := flag.Bool("profile", false, "Turn on pprof profiling")
@@ -82,12 +79,28 @@ func main() {
 	setting.BuildStamp = buildstampInt64
 
 	metrics.M_Grafana_Version.WithLabelValues(version).Set(1)
-
+	shutdownCompleted := make(chan int)
 	server := NewGrafanaServer()
-	server.Start()
+
+	go listenToSystemSignals(server, shutdownCompleted)
+
+	go func() {
+		code := 0
+		if err := server.Start(); err != nil {
+			log.Error2("Startup failed", "error", err)
+			code = 1
+		}
+
+		exitChan <- code
+	}()
+
+	code := <-shutdownCompleted
+	log.Info2("Grafana shutdown completed.", "code", code)
+	log.Close()
+	os.Exit(code)
 }
 
-func listenToSystemSignals(server models.GrafanaServer) {
+func listenToSystemSignals(server *GrafanaServerImpl, shutdownCompleted chan int) {
 	signalChan := make(chan os.Signal, 1)
 	ignoreChan := make(chan os.Signal, 1)
 	code := 0
@@ -97,10 +110,12 @@ func listenToSystemSignals(server models.GrafanaServer) {
 
 	select {
 	case sig := <-signalChan:
-		// Stops trace if profiling has been enabled
-		trace.Stop()
+		trace.Stop() // Stops trace if profiling has been enabled
 		server.Shutdown(0, fmt.Sprintf("system signal: %s", sig))
+		shutdownCompleted <- 0
 	case code = <-exitChan:
+		trace.Stop() // Stops trace if profiling has been enabled
 		server.Shutdown(code, "startup error")
+		shutdownCompleted <- code
 	}
 }

pkg/cmd/grafana-server/server.go

@@ -11,7 +11,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	"github.com/grafana/grafana/pkg/services/provisioning"
 
 	"golang.org/x/sync/errgroup"
@@ -20,7 +19,6 @@ import (
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/login"
 	"github.com/grafana/grafana/pkg/metrics"
-	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/services/alerting"
 	"github.com/grafana/grafana/pkg/services/cleanup"
@@ -33,7 +31,7 @@ import (
 	"github.com/grafana/grafana/pkg/tracing"
 )
 
-func NewGrafanaServer() models.GrafanaServer {
+func NewGrafanaServer() *GrafanaServerImpl {
 	rootCtx, shutdownFn := context.WithCancel(context.Background())
 	childRoutines, childCtx := errgroup.WithContext(rootCtx)
 
@@ -54,9 +52,7 @@ type GrafanaServerImpl struct {
 	httpServer *api.HttpServer
 }
 
-func (g *GrafanaServerImpl) Start() {
-	go listenToSystemSignals(g)
-
+func (g *GrafanaServerImpl) Start() error {
 	g.initLogging()
 	g.writePIDFile()
 
@@ -75,16 +71,12 @@ func (g *GrafanaServerImpl) Start() {
 	defer pluginCloser()
 
 	if err := provisioning.Init(g.context, setting.HomePath, setting.Cfg); err != nil {
-		logger.Error("Failed to provision Grafana from config", "error", err)
-		g.Shutdown(1, "Startup failed")
-		return
+		return fmt.Errorf("Failed to provision Grafana from config. error: %v", err)
 	}
 
 	tracingCloser, err := tracing.Init(setting.Cfg)
 	if err != nil {
-		g.log.Error("Tracing settings is not valid", "error", err)
-		g.Shutdown(1, "Startup failed")
-		return
+		return fmt.Errorf("Tracing settings is not valid. error: %v", err)
 	}
 	defer tracingCloser.Close()
 
@@ -99,13 +91,12 @@ func (g *GrafanaServerImpl) Start() {
 	g.childRoutines.Go(func() error { return cleanUpService.Run(g.context) })
 
 	if err = notifications.Init(); err != nil {
-		g.log.Error("Notification service failed to initialize", "error", err)
-		g.Shutdown(1, "Startup failed")
-		return
+		return fmt.Errorf("Notification service failed to initialize. error: %v", err)
 	}
 
-	SendSystemdNotification("READY=1")
-	g.startHttpServer()
+	sendSystemdNotification("READY=1")
+
+	return g.startHttpServer()
 }
 
 func initSql() {
@@ -129,16 +120,16 @@ func (g *GrafanaServerImpl) initLogging() {
 	setting.LogConfigurationInfo()
 }
 
-func (g *GrafanaServerImpl) startHttpServer() {
+func (g *GrafanaServerImpl) startHttpServer() error {
 	g.httpServer = api.NewHttpServer()
 
 	err := g.httpServer.Start(g.context)
 
 	if err != nil {
-		g.log.Error("Fail to start server", "error", err)
-		g.Shutdown(1, "Startup failed")
-		return
+		return fmt.Errorf("Fail to start server. error: %v", err)
 	}
+
+	return nil
 }
 
 func (g *GrafanaServerImpl) Shutdown(code int, reason string) {
@@ -151,10 +142,9 @@ func (g *GrafanaServerImpl) Shutdown(code int, reason string) {
 
 	g.shutdownFn()
 	err = g.childRoutines.Wait()
-
-	g.log.Info("Shutdown completed", "reason", err)
-	log.Close()
-	os.Exit(code)
+	if err != nil && err != context.Canceled {
+		g.log.Error("Server shutdown completed with an error", "error", err)
+	}
 }
 
 func (g *GrafanaServerImpl) writePIDFile() {
@@ -179,7 +169,7 @@ func (g *GrafanaServerImpl) writePIDFile() {
 	g.log.Info("Writing PID file", "path", *pidFile, "pid", pid)
 }
 
-func SendSystemdNotification(state string) error {
+func sendSystemdNotification(state string) error {
 	notifySocket := os.Getenv("NOTIFY_SOCKET")
 
 	if notifySocket == "" {

pkg/services/provisioning/dashboards/config_reader_test.go

@@ -8,7 +8,7 @@ import (
 
 var (
 	simpleDashboardConfig string = "./test-configs/dashboards-from-disk"
-	brokenConfigs         string = "./test-configs/borken-configs"
+	brokenConfigs         string = "./test-configs/broken-configs"
 )
 
 func TestDashboardsAsConfig(t *testing.T) {

pkg/social/github_oauth.go

@@ -58,12 +58,12 @@ func (s *SocialGithub) IsTeamMember(client *http.Client) bool {
 	return false
 }
 
-func (s *SocialGithub) IsOrganizationMember(client *http.Client) bool {
+func (s *SocialGithub) IsOrganizationMember(client *http.Client, organizationsUrl string) bool {
 	if len(s.allowedOrganizations) == 0 {
 		return true
 	}
 
-	organizations, err := s.FetchOrganizations(client)
+	organizations, err := s.FetchOrganizations(client, organizationsUrl)
 	if err != nil {
 		return false
 	}
@@ -167,12 +167,12 @@ func (s *SocialGithub) HasMoreRecords(headers http.Header) (string, bool) {
 
 }
 
-func (s *SocialGithub) FetchOrganizations(client *http.Client) ([]string, error) {
+func (s *SocialGithub) FetchOrganizations(client *http.Client, organizationsUrl string) ([]string, error) {
 	type Record struct {
 		Login string `json:"login"`
 	}
 
-	response, err := HttpGet(client, fmt.Sprintf(s.apiUrl+"/orgs"))
+	response, err := HttpGet(client, organizationsUrl)
 	if err != nil {
 		return nil, fmt.Errorf("Error getting organizations: %s", err)
 	}
@@ -193,10 +193,12 @@ func (s *SocialGithub) FetchOrganizations(client *http.Client) ([]string, error)
 }
 
 func (s *SocialGithub) UserInfo(client *http.Client) (*BasicUserInfo, error) {
+
 	var data struct {
-		Id    int    `json:"id"`
-		Login string `json:"login"`
-		Email string `json:"email"`
+		Id               int    `json:"id"`
+		Login            string `json:"login"`
+		Email            string `json:"email"`
+		OrganizationsUrl string `json:"organizations_url"`
 	}
 
 	response, err := HttpGet(client, s.apiUrl)
@@ -219,7 +221,7 @@ func (s *SocialGithub) UserInfo(client *http.Client) (*BasicUserInfo, error) {
 		return nil, ErrMissingTeamMembership
 	}
 
-	if !s.IsOrganizationMember(client) {
+	if !s.IsOrganizationMember(client, data.OrganizationsUrl) {
 		return nil, ErrMissingOrganizationMembership
 	}
 

pkg/tsdb/cloudwatch/credentials.go

@@ -141,6 +141,11 @@ func ec2RoleProvider(sess *session.Session) credentials.Provider {
 }
 
 func (e *CloudWatchExecutor) getDsInfo(region string) *DatasourceInfo {
+	defaultRegion := e.DataSource.JsonData.Get("defaultRegion").MustString()
+	if region == "default" {
+		region = defaultRegion
+	}
+
 	authType := e.DataSource.JsonData.Get("authType").MustString()
 	assumeRoleArn := e.DataSource.JsonData.Get("assumeRoleArn").MustString()
 	accessKey := ""

pkg/tsdb/postgres/macros.go

@@ -89,7 +89,7 @@ func (m *PostgresMacroEngine) evaluateMacro(name string, args []string) (string,
 		if err != nil {
 			return "", fmt.Errorf("error parsing interval %v", args[1])
 		}
-		return fmt.Sprintf("(extract(epoch from \"%s\")/%v)::bigint*%v", args[0], interval.Seconds(), interval.Seconds()), nil
+		return fmt.Sprintf("(extract(epoch from %s)/%v)::bigint*%v AS time", args[0], interval.Seconds(), interval.Seconds()), nil
 	case "__unixEpochFilter":
 		if len(args) == 0 {
 			return "", fmt.Errorf("missing time column argument for macro %v", name)

pkg/tsdb/postgres/macros_test.go

@@ -45,7 +45,7 @@ func TestMacroEngine(t *testing.T) {
 			sql, err := engine.Interpolate(timeRange, "GROUP BY $__timeGroup(time_column,'5m')")
 			So(err, ShouldBeNil)
 
-			So(sql, ShouldEqual, "GROUP BY (extract(epoch from \"time_column\")/300)::bigint*300")
+			So(sql, ShouldEqual, "GROUP BY (extract(epoch from time_column)/300)::bigint*300 AS time")
 		})
 
 		Convey("interpolate __timeTo function", func() {