Add support for sourcemap uploads

CHANGELOG.asciidoc

@@ -13,6 +13,7 @@ https://github.com/elastic/apm-server/compare/71df0d96445df35afe27f38bcf734a0828
 ==== Bug fixes
 
 ==== Added
+- Support for uploading sourcemaps {pull}302[302].
 
 ==== Deprecated
 

NOTICE.txt

@@ -336,6 +336,38 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+--------------------------------------------------------------------
+Dependency: github.com/go-sourcemap/sourcemap
+Revision: 4dcf5f6442018e298fdd505370345ef78552c31c
+License type (autodetected): BSD 2-clause license
+./vendor/github.com/go-sourcemap/sourcemap/LICENSE:
+--------------------------------------------------------------------
+Copyright (c) 2016 The github.com/go-sourcemap/sourcemap Contributors.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 --------------------------------------------------------------------
 Dependency: github.com/golang/protobuf
 Revision: 2463fd833024b2782962b18060ce642af2bdf1f4

beater/handlers.go

@@ -26,6 +26,7 @@ import (
 
 	err "github.com/elastic/apm-server/processor/error"
 	"github.com/elastic/apm-server/processor/healthcheck"
+	"github.com/elastic/apm-server/processor/sourcemap"
 	"github.com/elastic/apm-server/processor/transaction"
 	"github.com/elastic/beats/libbeat/monitoring"
 )
@@ -36,6 +37,7 @@ const (
 	BackendErrorsURL        = "/v1/errors"
 	FrontendErrorsURL       = "/v1/client-side/errors"
 	HealthCheckURL          = "/healthcheck"
+	SourcemapsURL           = "/v1/client-side/sourcemaps"
 
 	rateLimitCacheSize       = 1000
 	rateLimitBurstMultiplier = 2
@@ -70,6 +72,7 @@ var (
 		BackendErrorsURL:        {backendHandler, err.NewProcessor},
 		FrontendErrorsURL:       {frontendHandler, err.NewProcessor},
 		HealthCheckURL:          {healthCheckHandler, healthcheck.NewProcessor},
+		SourcemapsURL:           {sourcemapHandler, sourcemap.NewProcessor},
 	}
 )
 
@@ -87,15 +90,22 @@ func newMuxer(config Config, report reporter) *http.ServeMux {
 func backendHandler(pf ProcessorFactory, config Config, report reporter) http.Handler {
 	return logHandler(
 		authHandler(config.SecretToken,
-			processRequestHandler(pf, config, report)))
+			processRequestHandler(pf, config, report, decodeLimitJSONData(config.MaxUnzippedSize))))
 }
 
 func frontendHandler(pf ProcessorFactory, config Config, report reporter) http.Handler {
 	return logHandler(
 		killSwitchHandler(config.Frontend.isEnabled(),
 			ipRateLimitHandler(config.Frontend.RateLimit,
 				corsHandler(config.Frontend.AllowOrigins,
-					processRequestHandler(pf, config, report)))))
+					processRequestHandler(pf, config, report, decodeLimitJSONData(config.MaxUnzippedSize))))))
+}
+
+func sourcemapHandler(pf ProcessorFactory, config Config, report reporter) http.Handler {
+	return logHandler(
+		killSwitchHandler(config.Frontend.isEnabled(),
+			authHandler(config.SecretToken,
+				processRequestHandler(pf, config, report, sourcemap.DecodeSourcemapFormData))))
 }
 
 func healthCheckHandler(_ ProcessorFactory, _ Config, _ reporter) http.Handler {
@@ -247,35 +257,25 @@ func corsHandler(allowedOrigins []string, h http.Handler) http.Handler {
 	})
 }
 
-func processRequestHandler(pf ProcessorFactory, config Config, report reporter) http.Handler {
+func processRequestHandler(pf ProcessorFactory, config Config, report reporter, decode decoder) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		code, err := processRequest(r, pf, config.MaxUnzippedSize, report)
+		code, err := processRequest(r, pf, config.MaxUnzippedSize, report, decode)
 		sendStatus(w, r, code, err)
 	})
 }
 
-func processRequest(r *http.Request, pf ProcessorFactory, maxSize int64, report reporter) (int, error) {
+func processRequest(r *http.Request, pf ProcessorFactory, maxSize int64, report reporter, decode decoder) (int, error) {
 
 	processor := pf()
 
 	if r.Method != "POST" {
 		return http.StatusMethodNotAllowed, errPOSTRequestOnly
 	}
 
-	reader, err := decodeData(r)
+	buf, err := decode(r)
 	if err != nil {
 		return http.StatusBadRequest, errors.New(fmt.Sprintf("Decoding error: %s", err.Error()))
 	}
-	defer reader.Close()
-
-	// Limit size of request to prevent for example zip bombs
-	limitedReader := io.LimitReader(reader, maxSize)
-	buf, err := ioutil.ReadAll(limitedReader)
-	if err != nil {
-		// If we run out of memory, for example
-		return http.StatusInternalServerError, errors.New(fmt.Sprintf("Data read error: %s", err.Error()))
-
-	}
 
 	if err = processor.Validate(buf); err != nil {
 		return http.StatusBadRequest, err
@@ -293,34 +293,47 @@ func processRequest(r *http.Request, pf ProcessorFactory, maxSize int64, report
 	return http.StatusAccepted, nil
 }
 
-func decodeData(req *http.Request) (io.ReadCloser, error) {
+type decoder func(req *http.Request) ([]byte, error)
 
-	if req.Header.Get("Content-Type") != "application/json" {
-		return nil, fmt.Errorf("invalid content type: %s", req.Header.Get("Content-Type"))
-	}
+func decodeLimitJSONData(maxSize int64) decoder {
+	return func(req *http.Request) ([]byte, error) {
 
-	reader := req.Body
-	if reader == nil {
-		return nil, fmt.Errorf("No content supplied")
-	}
+		contentType := req.Header.Get("Content-Type")
+		if contentType != "application/json" {
+			return nil, fmt.Errorf("invalid content type: %s", req.Header.Get("Content-Type"))
+		}
 
-	switch req.Header.Get("Content-Encoding") {
-	case "deflate":
-		var err error
-		reader, err = zlib.NewReader(reader)
-		if err != nil {
-			return nil, err
+		reader := req.Body
+		if reader == nil {
+			return nil, fmt.Errorf("No content supplied")
 		}
 
-	case "gzip":
-		var err error
-		reader, err = gzip.NewReader(reader)
+		switch req.Header.Get("Content-Encoding") {
+		case "deflate":
+			var err error
+			reader, err = zlib.NewReader(reader)
+			if err != nil {
+				return nil, err
+			}
+
+		case "gzip":
+			var err error
+			reader, err = gzip.NewReader(reader)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		// Limit size of request to prevent for example zip bombs
+		limitedReader := io.LimitReader(reader, maxSize)
+		buf, err := ioutil.ReadAll(limitedReader)
 		if err != nil {
-			return nil, err
+			// If we run out of memory, for example
+			return nil, errors.New(fmt.Sprintf("Data read error: %s", err.Error()))
 		}
-	}
 
-	return reader, nil
+		return buf, nil
+	}
 }
 
 func sendStatus(w http.ResponseWriter, r *http.Request, code int, err error) {

beater/handlers_test.go

@@ -1,13 +1,14 @@
 package beater
 
 import (
-	"bytes"
 	"errors"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
+	"bytes"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/apm-server/tests"
@@ -22,11 +23,7 @@ func TestDecode(t *testing.T) {
 	req.Header.Add("Content-Type", "application/json")
 	assert.Nil(t, err)
 
-	res, err := decodeData(req)
-	assert.Nil(t, err)
-	assert.NotNil(t, res)
-
-	body, err := ioutil.ReadAll(res)
+	body, err := decodeLimitJSONData(1024 * 1024)(req)
 	assert.Nil(t, err)
 	assert.Equal(t, transactionBytes, body)
 }

docs/data/elasticsearch/sourcemap.json

@@ -0,0 +1,51 @@
+{
+    "processor": {
+        "event": "sourcemap",
+        "name": "sourcemap"
+    },
+    "sourcemap": {
+        "app": {
+            "name": "app",
+            "version": "1"
+        },
+        "bundle_filepath": "js/bundle.js",
+        "sourcemap": {
+            "file": "bundle.js",
+            "mappings": "CAAS,SAAUA,GCInB,QAAAC,GAAAC,GAGA,GAAAC,EAAAD,GACA,MAAAC,GAAAD,GAAAE,OAGA,IAAAC,GAAAF,EAAAD,IACAE,WACAE,GAAAJ,EACAK,QAAA,EAUA,OANAP,GAAAE,GAAAM,KAAAH,EAAAD,QAAAC,IAAAD,QAAAH,GAGAI,EAAAE,QAAA,EAGAF,EAAAD,QAvBA,GAAAD,KAqCA,OATAF,GAAAQ,EAAAT,EAGAC,EAAAS,EAAAP,EAGAF,EAAAU,EAAA,GAGAV,EAAA,KDMM,SAASI,EAAQD,EAASH,GE3ChCA,EAAA,GAEAA,EAAA,GAEAW,OFmDM,SAASP,EAAQD,EAASH,GGxDhCI,EAAAD,QAAAH,EAAAU,EAAA,cH8DM,SAASN,EAAQD,GI9DvB,QAAAQ,KACAC,QAAAC,IAAAC,QAGAH",
+            "names": [
+                "modules",
+                "__webpack_require__",
+                "moduleId",
+                "installedModules",
+                "exports",
+                "module",
+                "id",
+                "loaded",
+                "call",
+                "m",
+                "c",
+                "p",
+                "foo",
+                "console",
+                "log",
+                "foobar"
+            ],
+            "sourceRoot": "",
+            "sources": [
+                "webpack:///bundle.js",
+                "webpack:///webpack/bootstrap 6002740481c9666b0d38",
+                "webpack:///./scripts/index.js",
+                "webpack:///./index.html",
+                "webpack:///./scripts/app.js"
+            ],
+            "sourcesContent": [
+                "/******/ (function(modules) { // webpackBootstrap\n/******/ \t// The module cache\n/******/ \tvar installedModules = {};\n/******/\n/******/ \t// The require function\n/******/ \tfunction __webpack_require__(moduleId) {\n/******/\n/******/ \t\t// Check if module is in cache\n/******/ \t\tif(installedModules[moduleId])\n/******/ \t\t\treturn installedModules[moduleId].exports;\n/******/\n/******/ \t\t// Create a new module (and put it into the cache)\n/******/ \t\tvar module = installedModules[moduleId] = {\n/******/ \t\t\texports: {},\n/******/ \t\t\tid: moduleId,\n/******/ \t\t\tloaded: false\n/******/ \t\t};\n/******/\n/******/ \t\t// Execute the module function\n/******/ \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n/******/\n/******/ \t\t// Flag the module as loaded\n/******/ \t\tmodule.loaded = true;\n/******/\n/******/ \t\t// Return the exports of the module\n/******/ \t\treturn module.exports;\n/******/ \t}\n/******/\n/******/\n/******/ \t// expose the modules object (__webpack_modules__)\n/******/ \t__webpack_require__.m = modules;\n/******/\n/******/ \t// expose the module cache\n/******/ \t__webpack_require__.c = installedModules;\n/******/\n/******/ \t// __webpack_public_path__\n/******/ \t__webpack_require__.p = \"\";\n/******/\n/******/ \t// Load entry module and return exports\n/******/ \treturn __webpack_require__(0);\n/******/ })\n/************************************************************************/\n/******/ ([\n/* 0 */\n/***/ function(module, exports, __webpack_require__) {\n\n\t// Webpack\n\t__webpack_require__(1)\n\t\n\t__webpack_require__(2)\n\t\n\tfoo()\n\n\n/***/ },\n/* 1 */\n/***/ function(module, exports, __webpack_require__) {\n\n\tmodule.exports = __webpack_require__.p + \"index.html\"\n\n/***/ },\n/* 2 */\n/***/ function(module, exports) {\n\n\tfunction foo() {\n\t    console.log(foobar)\n\t}\n\t\n\tfoo()\n\n\n/***/ }\n/******/ ]);\n\n\n/** WEBPACK FOOTER **\n ** bundle.js\n **/",
+                " \t// The module cache\n \tvar installedModules = {};\n\n \t// The require function\n \tfunction __webpack_require__(moduleId) {\n\n \t\t// Check if module is in cache\n \t\tif(installedModules[moduleId])\n \t\t\treturn installedModules[moduleId].exports;\n\n \t\t// Create a new module (and put it into the cache)\n \t\tvar module = installedModules[moduleId] = {\n \t\t\texports: {},\n \t\t\tid: moduleId,\n \t\t\tloaded: false\n \t\t};\n\n \t\t// Execute the module function\n \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n\n \t\t// Flag the module as loaded\n \t\tmodule.loaded = true;\n\n \t\t// Return the exports of the module\n \t\treturn module.exports;\n \t}\n\n\n \t// expose the modules object (__webpack_modules__)\n \t__webpack_require__.m = modules;\n\n \t// expose the module cache\n \t__webpack_require__.c = installedModules;\n\n \t// __webpack_public_path__\n \t__webpack_require__.p = \"\";\n\n \t// Load entry module and return exports\n \treturn __webpack_require__(0);\n\n\n\n/** WEBPACK FOOTER **\n ** webpack/bootstrap 6002740481c9666b0d38\n **/",
+                "// Webpack\nrequire('../index.html')\n\nrequire('./app')\n\nfoo()\n\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./scripts/index.js\n ** module id = 0\n ** module chunks = 0\n **/",
+                "module.exports = __webpack_public_path__ + \"index.html\"\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./index.html\n ** module id = 1\n ** module chunks = 0\n **/",
+                "function foo() {\n    console.log(foobar)\n}\n\nfoo()\n\n\n\n/*****************\n ** WEBPACK FOOTER\n ** ./scripts/app.js\n ** module id = 2\n ** module chunks = 0\n **/"
+            ],
+            "version": 3
+        }
+    }
+}

docs/fields.asciidoc

@@ -14,6 +14,7 @@ grouped in the following categories:
 
 * <<exported-fields-apm>>
 * <<exported-fields-apm-error>>
+* <<exported-fields-apm-sourcemap>>
 * <<exported-fields-apm-trace>>
 * <<exported-fields-apm-transaction>>
 * <<exported-fields-beat>>
@@ -445,6 +446,44 @@ type: keyword
 
 Equal to message, but with placeholders replaced.
 
+[[exported-fields-apm-sourcemap]]
+== APM Sourcemap fields
+
+Sourcemap files enriched with metadata
+
+
+
+[float]
+== app fields
+
+App fields.
+
+
+
+[float]
+=== `sourcemap.app.name`
+
+type: keyword
+
+The name of the app this sourcemap belongs to.
+
+
+[float]
+=== `sourcemap.app.version`
+
+type: keyword
+
+App version.
+
+
+[float]
+=== `sourcemap.bundle_filepath`
+
+type: keyword
+
+Location of the sourcemap relative to the file requesting it.
+
+
 [[exported-fields-apm-trace]]
 == APM Trace fields