diff --git a/src/sentry/static/sentry/bootstrap b/src/sentry/static/sentry/bootstrap index d9b502dfb876c4..de683e9003b5ec 160000 --- a/src/sentry/static/sentry/bootstrap +++ b/src/sentry/static/sentry/bootstrap @@ -1 +1 @@ -Subproject commit d9b502dfb876c40b0735008bac18049c7ee7b6d2 +Subproject commit de683e9003b5ec6860a1fcb768416356dd685ef5 diff --git a/src/sentry/tasks/fetch_source.py b/src/sentry/tasks/fetch_source.py index 495281f8165349..0f3e020c121efc 100644 --- a/src/sentry/tasks/fetch_source.py +++ b/src/sentry/tasks/fetch_source.py @@ -12,6 +12,7 @@ import re import urllib2 import zlib +import base64 from collections import namedtuple from urlparse import urljoin @@ -27,6 +28,8 @@ LINES_OF_CONTEXT = 5 CHARSET_RE = re.compile(r'charset=(\S+)') DEFAULT_ENCODING = 'utf-8' +BASE64_SOURCEMAP_PREAMBLE = 'data:application/json;base64,' +BASE64_PREAMBLE_LENGTH = len(BASE64_SOURCEMAP_PREAMBLE) UrlResult = namedtuple('UrlResult', ['url', 'headers', 'body']) @@ -161,11 +164,15 @@ def fetch_url(url): def fetch_sourcemap(url): - result = fetch_url(url) - if result == BAD_SOURCE: - return + if is_data_uri(url): + body = base64.b64decode(url[BASE64_PREAMBLE_LENGTH:]) + else: + result = fetch_url(url) + if result == BAD_SOURCE: + return + + body = result.body - body = result.body # According to spec (https://docs.google.com/document/d/1U1RGAehQwRypUTovF1KRlpiOFze0b-_2gc6fAH0KY0k/edit#heading=h.h7yy76c5il9v) # A SourceMap may be prepended with ")]}'" to cause a Javascript error. # If the file starts with that string, ignore the entire first line. @@ -179,6 +186,10 @@ def fetch_sourcemap(url): return index +def is_data_uri(url): + return url[:BASE64_PREAMBLE_LENGTH] == BASE64_SOURCEMAP_PREAMBLE + + def expand_javascript_source(data, **kwargs): """ Attempt to fetch source code for javascript frames. @@ -250,12 +261,18 @@ def expand_javascript_source(data, **kwargs): continue sourcemap = discover_sourcemap(result) - source_code[filename] = (result.body.splitlines(), sourcemap) # TODO: we're currently running splitlines twice - if sourcemap: - logger.debug('Found sourcemap %r for minified script %r', sourcemap, result.url) - elif sourcemap in sourmap_idxs or not sourcemap: + if not sourcemap: + source_code[filename] = (result.body.splitlines(), None) + continue + else: + logger.debug('Found sourcemap %r for minified script %r', sourcemap[:256], result.url) + + sourcemap_key = hashlib.md5(sourcemap).hexdigest() + source_code[filename] = (result.body.splitlines(), sourcemap_key) + + if sourcemap in sourmap_idxs: continue # pull down sourcemap @@ -264,13 +281,20 @@ def expand_javascript_source(data, **kwargs): logger.debug('Failed parsing sourcemap index: %r', sourcemap[:15]) continue - sourmap_idxs[sourcemap] = index + if is_data_uri(sourcemap): + sourmap_idxs[sourcemap_key] = (index, result.url) + else: + sourmap_idxs[sourcemap_key] = (index, sourcemap) # queue up additional source files for download for source in index.sources: next_filename = urljoin(result.url, source) if next_filename not in done_file_list: - pending_file_list.add(next_filename) + if index.content: + source_code[next_filename] = (index.content[source], None) + done_file_list.add(next_filename) + else: + pending_file_list.add(next_filename) has_changes = False for frame in frames: @@ -282,9 +306,9 @@ def expand_javascript_source(data, **kwargs): # may have had a failure pulling down the sourcemap previously if sourcemap in sourmap_idxs and frame.colno is not None: - state = find_source(sourmap_idxs[sourcemap], frame.lineno, frame.colno) - # TODO: is this urljoin right? (is it relative to the sourcemap or the originating file) - abs_path = urljoin(sourcemap, state.src) + index, relative_to = sourmap_idxs[sourcemap] + state = find_source(index, frame.lineno, frame.colno) + abs_path = urljoin(relative_to, state.src) logger.debug('Mapping compressed source %r to mapping in %r', frame.abs_path, abs_path) try: source, _ = source_code[abs_path] diff --git a/src/sentry/utils/sourcemaps.py b/src/sentry/utils/sourcemaps.py index ba21bc2d999af4..065781ec1b2872 100644 --- a/src/sentry/utils/sourcemaps.py +++ b/src/sentry/utils/sourcemaps.py @@ -17,7 +17,7 @@ SourceMap = namedtuple('SourceMap', ['dst_line', 'dst_col', 'src', 'src_line', 'src_col', 'name']) -SourceMapIndex = namedtuple('SourceMapIndex', ['states', 'keys', 'sources']) +SourceMapIndex = namedtuple('SourceMapIndex', ['states', 'keys', 'sources', 'content']) # Mapping of base64 letter -> integer value. B64 = dict( @@ -59,12 +59,10 @@ def parse_vlq(segment): return values -def parse_sourcemap(sourcemap): +def parse_sourcemap(smap): """ - Given a file-like object, yield SourceMap objects as they are read from it. + Given a sourcemap json object, yield SourceMap objects as they are read from it. """ - - smap = json.loads(sourcemap) sources = smap['sources'] sourceRoot = smap.get('sourceRoot') names = smap['names'] @@ -108,16 +106,27 @@ def parse_sourcemap(sourcemap): def sourcemap_to_index(sourcemap): + smap = json.loads(sourcemap) + state_list = [] key_list = [] src_list = set() + content = None + + if 'sourcesContent' in smap: + content = {} + for idx, source in enumerate(smap['sources']): + if smap['sourcesContent'][idx]: + content[source] = smap['sourcesContent'][idx].splitlines() + else: + content[source] = [] - for state in parse_sourcemap(sourcemap): + for state in parse_sourcemap(smap): state_list.append(state) key_list.append((state.dst_line, state.dst_col)) src_list.add(state.src) - return SourceMapIndex(state_list, key_list, src_list) + return SourceMapIndex(state_list, key_list, src_list, content) def find_source(indexed_sourcemap, lineno, colno): diff --git a/tests/sentry/tasks/fetch_source/tests.py b/tests/sentry/tasks/fetch_source/tests.py index d5498c40e45607..2e97bf3aada5d1 100644 --- a/tests/sentry/tasks/fetch_source/tests.py +++ b/tests/sentry/tasks/fetch_source/tests.py @@ -5,9 +5,12 @@ import mock from sentry.tasks.fetch_source import ( - UrlResult, expand_javascript_source, discover_sourcemap) + UrlResult, expand_javascript_source, discover_sourcemap, fetch_sourcemap) +from sentry.utils.sourcemaps import (SourceMap, SourceMapIndex) from sentry.testutils import TestCase +base64_sourcemap = 'data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZ2VuZXJhdGVkLmpzIiwic291cmNlcyI6WyIvdGVzdC5qcyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiO0FBQUEiLCJzb3VyY2VzQ29udGVudCI6WyJjb25zb2xlLmxvZyhcImhlbGxvLCBXb3JsZCFcIikiXX0=' + class DiscoverSourcemapTest(TestCase): # discover_sourcemap(result) @@ -42,7 +45,8 @@ class ExpandJavascriptSourceTest(TestCase): @mock.patch('sentry.models.Event.update') @mock.patch('sentry.tasks.fetch_source.fetch_url') @mock.patch('sentry.tasks.fetch_source.fetch_sourcemap') - def test_simple(self, fetch_sourcemap, fetch_url, update): + @mock.patch('sentry.tasks.fetch_source.discover_sourcemap') + def test_simple(self, discover_sourcemap, fetch_sourcemap, fetch_url, update): data = { 'sentry.interfaces.Exception': { 'values': [{ @@ -65,6 +69,7 @@ def test_simple(self, fetch_sourcemap, fetch_url, update): }], } } + discover_sourcemap.return_value = None fetch_sourcemap.return_value = None fetch_url.return_value.body = '\n'.join('hello world') @@ -82,3 +87,46 @@ def test_simple(self, fetch_sourcemap, fetch_url, update): assert frame['pre_context'] == [] assert frame['context_line'] == 'h' assert frame['post_context'] == ['e', 'l', 'l', 'o', ' '] + + @mock.patch('sentry.models.Event.update') + @mock.patch('sentry.tasks.fetch_source.fetch_url') + @mock.patch('sentry.tasks.fetch_source.discover_sourcemap') + def test_inlined_sources(self, discover_sourcemap, fetch_url, update): + data = { + 'sentry.interfaces.Exception': { + 'values': [{ + 'stacktrace': { + 'frames': [ + { + 'abs_path': 'http://example.com/test.js', + 'filename': 'test.js', + 'lineno': 1, + 'colno': 0, + }, + ], + }, + }], + } + } + discover_sourcemap.return_value = base64_sourcemap + fetch_url.return_value.body = '\n'.join('') + + expand_javascript_source(data) + fetch_url.assert_called_once_with('http://example.com/test.js') + + frame_list = data['sentry.interfaces.Exception']['values'][0]['stacktrace']['frames'] + frame = frame_list[0] + assert frame['pre_context'] == [] + assert frame['context_line'] == 'console.log("hello, World!")' + assert frame['post_context'] == [] + + +class FetchBase64SourcemapTest(TestCase): + def test_simple(self): + index = fetch_sourcemap(base64_sourcemap) + states = [SourceMap(1, 0, '/test.js', 0, 0, None)] + sources = set(['/test.js']) + keys = [(1, 0)] + content = {'/test.js': ['console.log("hello, World!")']} + + assert index == SourceMapIndex(states, keys, sources, content) diff --git a/tests/sentry/utils/sourcemaps/tests.py b/tests/sentry/utils/sourcemaps/tests.py index 03efc3b5eb01e2..3908813ce2b3b7 100644 --- a/tests/sentry/utils/sourcemaps/tests.py +++ b/tests/sentry/utils/sourcemaps/tests.py @@ -6,6 +6,8 @@ find_source) from sentry.testutils import TestCase +from sentry.utils import json + sourcemap = """{"version":3,"file":"file.min.js","sources":["file1.js","file2.js"],"names":["add","a","b","multiply","divide","c","e","Raven","captureException"],"mappings":"AAAA,QAASA,KAAIC,EAAGC,GACf,YACA,OAAOD,GAAIC,ECFZ,QAASC,UAASF,EAAGC,GACpB,YACA,OAAOD,GAAIC,EAEZ,QAASE,QAAOH,EAAGC,GAClB,YACA,KACC,MAAOC,UAASH,IAAIC,EAAGC,GAAID,EAAGC,GAAKG,EAClC,MAAOC,GACRC,MAAMC,iBAAiBF"}""" @@ -30,7 +32,8 @@ def test_simple(self): class ParseSourcemapTest(TestCase): def test_basic(self): - states = list(parse_sourcemap(sourcemap)) + smap = json.loads(sourcemap) + states = list(parse_sourcemap(smap)) assert states == [ SourceMap(dst_line=0, dst_col=0, src='file1.js', src_line=0, src_col=0, name=None),