render_handler.go

package http

import (
	"net/http"
	"strconv"
	"time"

	"github.com/go-graphite/carbonapi/carbonapipb"
	"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
	"github.com/go-graphite/carbonapi/date"
	"github.com/go-graphite/carbonapi/expr"
	"github.com/go-graphite/carbonapi/expr/functions/cairo/png"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
	"github.com/go-graphite/carbonapi/util/ctx"

	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
	"github.com/lomik/zapwriter"
	"github.com/satori/go.uuid"
	"go.uber.org/zap"
)

func cleanupParams(r *http.Request) {
	// make sure the cache key doesn't say noCache, because it will never hit
	r.Form.Del("noCache")

	// jsonp callback names are frequently autogenerated and hurt our cache
	r.Form.Del("jsonp")

	// Strip some cache-busters.  If you don't want to cache, use noCache=1
	r.Form.Del("_salt")
	r.Form.Del("_ts")
	r.Form.Del("_t") // Used by jquery.graphite.js
}

func setError(w http.ResponseWriter, accessLogDetails *carbonapipb.AccessLogDetails, msg string, status int) {
	http.Error(w, http.StatusText(status)+": "+msg, status)
	accessLogDetails.Reason = msg
	accessLogDetails.HTTPCode = int32(status)
}

func getFormat(r *http.Request) string {
	format := r.FormValue("format")

	if format == "" && (parser.TruthyBool(r.FormValue("rawData")) || parser.TruthyBool(r.FormValue("rawdata"))) {
		format = rawFormat
	}

	if format == "" {
		format = pngFormat
	}

	return format
}

func getCacheTimeout(logger *zap.Logger, r *http.Request) int32 {
	cacheTimeout := config.Config.Cache.DefaultTimeoutSec

	if tstr := r.FormValue("cacheTimeout"); tstr != "" {
		t, err := strconv.Atoi(tstr)
		if err != nil {
			logger.Error("failed to parse cacheTimeout",
				zap.String("cache_string", tstr),
				zap.Error(err),
			)
		} else {
			cacheTimeout = int32(t)
		}
	}

	return cacheTimeout
}

func renderHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now()
	uuid := uuid.NewV4()

	// TODO: Migrate to context.WithTimeout
	// ctx, _ := context.WithTimeout(context.TODO(), config.Config.ZipperTimeout)
	ctx := ctx.SetUUID(r.Context(), uuid.String())
	username, _, _ := r.BasicAuth()

	logger := zapwriter.Logger("render").With(
		zap.String("carbonapi_uuid", uuid.String()),
		zap.String("username", username),
	)

	srcIP, srcPort := splitRemoteAddr(r.RemoteAddr)

	accessLogger := zapwriter.Logger("access")
	var accessLogDetails = &carbonapipb.AccessLogDetails{
		Handler:       "render",
		Username:      username,
		CarbonapiUUID: uuid.String(),
		URL:           r.URL.RequestURI(),
		PeerIP:        srcIP,
		PeerPort:      srcPort,
		Host:          r.Host,
		Referer:       r.Referer(),
		URI:           r.RequestURI,
	}

	logAsError := false
	defer func() {
		deferredAccessLogging(accessLogger, accessLogDetails, t0, logAsError)
	}()

	size := 0
	ApiMetrics.Requests.Add(1)

	err := r.ParseForm()
	if err != nil {
		setError(w, accessLogDetails, err.Error(), http.StatusBadRequest)
		logAsError = true
		return
	}

	targets := r.Form["target"]
	from := r.FormValue("from")
	until := r.FormValue("until")
	template := r.FormValue("template")
	useCache := !parser.TruthyBool(r.FormValue("noCache"))
	format := getFormat(r)

	var jsonp string

	if format == jsonFormat {
		// TODO(dgryski): check jsonp only has valid characters
		jsonp = r.FormValue("jsonp")
	}

	cacheTimeout := getCacheTimeout(logger, r)

	cleanupParams(r)

	cacheKey := r.Form.Encode()

	// normalize from and until values
	qtz := r.FormValue("tz")
	from32 := date.DateParamToEpoch(from, qtz, timeNow().Add(-24*time.Hour).Unix(), config.Config.DefaultTimeZone)
	until32 := date.DateParamToEpoch(until, qtz, timeNow().Unix(), config.Config.DefaultTimeZone)

	accessLogDetails.UseCache = useCache
	accessLogDetails.FromRaw = from
	accessLogDetails.From = from32
	accessLogDetails.UntilRaw = until
	accessLogDetails.Until = until32
	accessLogDetails.Tz = qtz
	accessLogDetails.CacheTimeout = cacheTimeout
	accessLogDetails.Format = format
	accessLogDetails.Targets = targets
	if useCache {
		tc := time.Now()
		response, err := config.Config.QueryCache.Get(cacheKey)
		td := time.Since(tc).Nanoseconds()
		ApiMetrics.RenderCacheOverheadNS.Add(td)

		accessLogDetails.CarbonzipperResponseSizeBytes = 0
		accessLogDetails.CarbonapiResponseSizeBytes = int64(len(response))

		if err == nil {
			ApiMetrics.RequestCacheHits.Add(1)
			writeResponse(w, response, format, jsonp)
			accessLogDetails.FromCache = true
			return
		}
		ApiMetrics.RequestCacheMisses.Add(1)
	}

	if from32 == until32 {
		setError(w, accessLogDetails, "Invalid or empty time range", http.StatusBadRequest)
		logAsError = true
		return
	}

	var results []*types.MetricData
	errors := make(map[string]string)
	metricMap := make(map[parser.MetricRequest][]*types.MetricData)

	var metrics []string
	// Iterate over all targets, check if we need to fetch them
	// TODO(civil): replace loop with 'for targetIdx := 0; targetIdx < len(targets); targetIdx++'
	var targetIdx = 0
	for targetIdx < len(targets) {
		var target = targets[targetIdx]
		targetIdx++

		exp, e, err := parser.ParseExpr(target)

		// if expression cannot be parsed return error
		if err != nil || e != "" {
			msg := buildParseErrorString(target, e, err)
			setError(w, accessLogDetails, msg, http.StatusBadRequest)
			logAsError = true
			return
		}

		// Splitting requests into batches is now done by carbonzipper
		pathExprTimeMap := make(map[string]requestInterval)
		var req pb.MultiFetchRequest
		for _, m := range exp.Metrics() {
			metrics = append(metrics, m.Metric)
			mFetch := m
			mFetch.From += from32
			mFetch.Until += until32

			if _, ok := metricMap[mFetch]; ok {
				// already fetched this metric for this request
				continue
			}

			req.Metrics = append(req.Metrics, pb.FetchRequest{
				Name:           m.Metric,
				PathExpression: m.Metric,
				StartTime:      mFetch.From,
				StopTime:       mFetch.Until,
			})
			pathExprTimeMap[m.Metric] = requestInterval{from: mFetch.From, until: mFetch.Until}
		}

		// Do we need to fetch anything?
		if len(req.Metrics) > 0 {
			ApiMetrics.RenderRequests.Add(1)
			config.Config.Limiter.Enter()

			r, stats, err := config.Config.ZipperInstance.Render(ctx, req)
			if stats != nil {
				accessLogDetails.ZipperRequests += stats.ZipperRequests
				accessLogDetails.TotalMetricsCount += stats.TotalMetricsCount
			}
			if err != nil {
				errors[target] = err.Error()
			}

			config.Config.Limiter.Leave()
			for _, m := range r {
				var from int64
				var until int64
				if m.RequestStartTime != 0 && m.RequestStopTime != 0 {
					from = m.RequestStartTime
					until = m.RequestStopTime
				} else {
					from = pathExprTimeMap[m.PathExpression].from
					until = pathExprTimeMap[m.PathExpression].until
				}

				mFetch := parser.MetricRequest{
					Metric: m.PathExpression,
					From:   from,
					Until:  until,
				}

				d := metricMap[mFetch]
				metricMap[mFetch] = append(d, m)

			}
			for i := range r {
				size += r[i].Size()
			}

			for mFetch := range metricMap {
				expr.SortMetrics(metricMap[mFetch], mFetch)
			}
		}
		accessLogDetails.Metrics = metrics

		var rewritten bool
		var newTargets []string

		rewritten, newTargets, err = expr.RewriteExpr(exp, from32, until32, metricMap)
		if err != nil && err != parser.ErrSeriesDoesNotExist {
			errors[target] = err.Error()
			accessLogDetails.Reason = err.Error()
			logAsError = true
			return
		}

		if rewritten {
			targets = append(targets, newTargets...)
		} else {
			func() {
				defer func() {
					if r := recover(); r != nil {
						logger.Error("panic during eval:",
							zap.String("cache_key", cacheKey),
							zap.Any("reason", r),
							zap.Stack("stack"),
						)
					}
				}()
				expressions, err := expr.EvalExpr(exp, from32, until32, metricMap)
				if err != nil && err != parser.ErrSeriesDoesNotExist {
					errors[target] = err.Error()
					accessLogDetails.Reason = err.Error()
					logAsError = true
					return
				}
				results = append(results, expressions...)
			}()
		}
	}

	var body []byte

	if len(results) == 0 {
		logger.Info("empty response or no response")
		results = append(results, &types.MetricData{})
	}

	switch format {
	case jsonFormat:
		if maxDataPoints, _ := strconv.Atoi(r.FormValue("maxDataPoints")); maxDataPoints != 0 {
			types.ConsolidateJSON(maxDataPoints, results)
		}

		body = types.MarshalJSON(results)
	case protobufFormat, protobuf3Format:
		body, err = types.MarshalProtobuf(results)
		if err != nil {
			setError(w, accessLogDetails, err.Error(), http.StatusInternalServerError)
			logAsError = true
			return
		}
	case rawFormat:
		body = types.MarshalRaw(results)
	case csvFormat:
		body = types.MarshalCSV(results)
	case pickleFormat:
		body = types.MarshalPickle(results)
	case pngFormat:
		body = png.MarshalPNGRequest(r, results, template)
	case svgFormat:
		body = png.MarshalSVGRequest(r, results, template)
	}

	writeResponse(w, body, format, jsonp)

	if len(results) != 0 {
		tc := time.Now()
		config.Config.QueryCache.Set(cacheKey, body, cacheTimeout)
		td := time.Since(tc).Nanoseconds()
		ApiMetrics.RenderCacheOverheadNS.Add(td)
	}

	gotErrors := len(errors) > 0
	accessLogDetails.HaveNonFatalErrors = gotErrors
}