source-to-source mappings! (same format as Closure Compiler)

pj/api_internal.py

@@ -2,7 +2,7 @@
 import ast
 
 from pyxc.importing import SourcePath, orderedModules
-from pyxc.transforming import Transformer
+from pyxc.transforming import Transformer, SourceMap, exportSourceMap
 from pyxc.util import topLevelNamesInBody
 
 import pj.js_ast
@@ -26,7 +26,7 @@ def codeToCode(py):
 
 
 #### Build Bundle
-def buildBundle(mainModule, path=None, includeSource=False, prependJs=None):
+def buildBundle(mainModule, path=None, createSourceMap=False, includeSource=False, prependJs=None):
 
     assert path
 
@@ -37,15 +37,21 @@ def buildBundle(mainModule, path=None, includeSource=False, prependJs=None):
     jsArr = []
     topLevelNames = set()
 
+    linemaps = []
+    mappings = []
+
     sourceDict = {}
 
+    i = 0
     for module in modules:
+        fileKey = str(i)
+        i += 1
         codePath = sourcePath.pathForModule(module)
         with open(codePath, 'rb') as f:
 
             py = str(f.read(), 'utf-8')
 
-            sourceDict[module] = {
+            sourceDict[fileKey] = {
                 'path': codePath,
                 'code': py,
                 'module': module,
@@ -63,7 +69,20 @@ def buildBundle(mainModule, path=None, includeSource=False, prependJs=None):
 
                 # py → js
                 jsAst = t.transformCode(py)
-                js = str(jsAst)
+                if createSourceMap:
+
+                    sm = SourceMap(fileKey, nextMappingId=len(mappings))
+                    sm.handleNode(jsAst)
+
+                    js = sm.getCode() + '\n'
+
+                    assert len(sm.linemaps) == len(js.split('\n')) - 1
+
+                    linemaps += sm.linemaps
+                    mappings += sm.mappings
+
+                else:
+                    js = str(jsAst)
 
             jsArr.append(js)
 
@@ -78,6 +97,10 @@ def buildBundle(mainModule, path=None, includeSource=False, prependJs=None):
                     '\n'.join(t.snippets), '\n\n',
                     varJs, '\n\n'])
 
+    linemaps = (
+                    ([[]] * (len(jsPrefix.split('\n')) - 1)) +
+                    linemaps)
+
     js = ''.join([
                     jsPrefix,
                     ''.join(jsArr),
@@ -87,8 +110,15 @@ def buildBundle(mainModule, path=None, includeSource=False, prependJs=None):
         'js': js,
     }
 
+    if createSourceMap:
+        info['sourceMap'] = exportSourceMap(linemaps, mappings, sourceDict)
+
     if includeSource:
-        info['sourceDict'] = sourceDict
+        info['sourceDict'] = dict(
+                                    (
+                                        sourceDict[k]['module'],
+                                        sourceDict[k])
+                                    for k in sourceDict)
 
     return info
 

pyxc/transforming.py

@@ -2,7 +2,7 @@
 from pyxc.util import usingPython3
 assert usingPython3()
 
-import sys, os, ast
+import sys, os, ast, json, hashlib
 from pyxc.util import rfilter, parentOf, randomToken
 
 from pyxc.pyxc_exceptions import NoTransformationForNode
@@ -19,6 +19,99 @@ def __str__(self):
                     str(x) for x in
                         self.emit(*self.transformedArgs))
 
+
+#### SourceMap
+
+class SourceMap:
+
+    def __init__(self, fileString, nextMappingId=0):
+
+        self.fileString = fileString
+
+        self.nextMappingId = nextMappingId
+        self.node_mappingId_map = {}
+        self.mappings = []
+
+        self.linemaps = [[]]
+        self.strings = []
+
+    def getCode(self):
+        return ''.join(self.strings)
+
+    def handleNode(self, node, parentMappingId=0):
+
+        mappingId = self.mappingIdForNode(node)
+        if mappingId is None:
+            mappingId = parentMappingId
+
+        arr = node.emit(*node.transformedArgs)
+        for x in arr:
+            if isinstance(x, str):
+                if x:
+                    # Store the string
+                    self.strings.append(x)
+
+                    # Update self.linemaps
+                    linemap = self.linemaps[-1]
+                    for c in x:
+                        linemap.append(mappingId)
+                        if c == '\n':
+                            linemap = []
+                            self.linemaps.append(linemap)
+            else:
+                assert isinstance(x, TargetNode)
+                self.handleNode(x, parentMappingId=mappingId)
+
+    def mappingIdForNode(self, node):
+        if node in self.node_mappingId_map:
+            return self.node_mappingId_map[node]
+        else:
+            lineno = getattr(node.pyNode, 'lineno', None)
+            col_offset = getattr(node.pyNode, 'col_offset', None)
+            if (lineno is None) or (col_offset is None):
+                return None
+            else:
+                mappingId = self.nextMappingId
+                self.nextMappingId += 1
+                self.node_mappingId_map[node] = mappingId
+                self.mappings.append([self.fileString, lineno, col_offset])
+                return mappingId
+
+
+def exportSourceMap(linemaps, mappings, sourceDict):
+
+    # Get filekeys from mappings
+    filekeys = []
+    filekeysSet = set()
+    for tup in mappings:
+        k = tup[0]
+        if k not in filekeysSet:
+            filekeysSet.add(k)
+            filekeys.append(k)
+
+    arr = ['/** Begin line maps. **/{ "file" : "", "count": %d }\n' % len(filekeys)]
+
+    for linemap in linemaps:
+        arr.append(json.dumps(linemap, separators=(',', ':')) + '\n')
+
+    arr.append('/** Begin file information. **/\n')
+    for k in filekeys:
+        sourceInfo = sourceDict[k]
+        arr.append('%s\n' % json.dumps([{
+            'module': sourceInfo['module'],
+            'sha1': hashlib.sha1(sourceInfo['code'].encode('utf-8')).hexdigest(),
+            'path': sourceInfo['path'],
+            'name': sourceInfo['path'].split('/')[-1],
+            'k': k,
+        }]))
+    arr.append('/** Begin mapping definitions. **/\n')
+
+    for mapping in mappings:
+        arr.append(json.dumps(mapping, separators=(',', ':')) + '\n')
+
+    return ''.join(arr)
+
+
 #### Transformer
 
 class Transformer: