Manage generated sources

apps/remix-ide-e2e/src/tests/debugger.test.ts

@@ -147,7 +147,27 @@ module.exports = {
     .waitForElementPresent('*[data-id="treeViewLoadMore"]')
     .click('*[data-id="treeViewLoadMore"]')
     .assert.containsText('*[data-id="solidityLocals"]', '149: 0 uint256')
-    .notContainsText('*[data-id="solidityLocals"]', '150: 0 uint256')
+    .notContainsText('*[data-id="solidityLocals"]', '150: 0 uint256')    
+  },
+
+  'Should debug using generated sources': function (browser: NightwatchBrowser) {
+    browser
+    .clickLaunchIcon('solidity')
+    .setSolidityCompilerVersion('soljson-v0.7.2+commit.51b20bc0.js')
+    .clickLaunchIcon('udapp')    
+    .testContracts('withGeneratedSources.sol', sources[4]['browser/withGeneratedSources.sol'], ['A'])
+    .createContract('')
+    .clickInstance(4)
+    .clickFunction('f - transact (not payable)', {types: 'uint256[] ', values: '[]'})
+    .debugTransaction(8)
+    .pause(2000)
+    .click('*[data-id="debuggerTransactionStartButton"]') // stop debugging
+    .click('*[data-id="debugGeneratedSourcesLabel"]') // select debug with generated sources
+    .click('*[data-id="debuggerTransactionStartButton"]') // start debugging
+    .pause(2000)
+    .getEditorValue((content) => {
+      browser.assert.ok(content.indexOf('if slt(sub(dataEnd, headStart), 32) { revert(0, 0) }') != -1, 'current displayed content is not a generated source')
+    })
     .end()
   },
 
@@ -227,6 +247,17 @@ const sources = [
     }
         `
     }
+  },
+  {
+    'browser/withGeneratedSources.sol': {
+      content: `
+      // SPDX-License-Identifier: GPL-3.0
+      pragma experimental ABIEncoderV2; 
+      contract A { 
+        function f(uint[] memory) public returns (uint256) { } 
+      }
+      `
+    }
   }
 ]
 
@@ -293,4 +324,4 @@ const localVariable_step717_ABIEncoder = {
       "error": "<decoding failed - no decoder for calldata>",
       "type": "bytes"
   }
-}
+}

apps/remix-ide/src/app/tabs/debugger/debuggerUI.js

@@ -22,6 +22,26 @@ var css = csjs`
   .statusMessage {
     margin-left: 15px;
   }
+
+  .debuggerConfig {
+    display: flex;
+    align-items: center;
+  }
+
+  .debuggerConfig label {
+    margin: 0;
+  }
+
+  .debuggerSection {
+    padding: 12px 24px 16px;
+  }
+
+  .debuggerLabel {
+    margin-bottom: 2px;
+    font-size: 11px;
+    line-height: 12px;
+    text-transform: uppercase;
+  }
 `
 
 class DebuggerUI {
@@ -32,7 +52,9 @@ class DebuggerUI {
     this.event = new EventManager()
 
     this.isActive = false
-
+    this.opt = {
+      debugWithGeneratedSources: false
+    }
     this.sourceHighlighter = new SourceHighlighter()
 
     this.startTxBrowser()
@@ -72,12 +94,31 @@ class DebuggerUI {
       this.isActive = isActive
     })
 
-    this.debugger.event.register('newSourceLocation', async (lineColumnPos, rawLocation) => {
+    this.debugger.event.register('newSourceLocation', async (lineColumnPos, rawLocation, generatedSources) => {
+      if (!lineColumnPos) return
       const contracts = await this.fetchContractAndCompile(
         this.currentReceipt.contractAddress || this.currentReceipt.to,
         this.currentReceipt)
       if (contracts) {
-        const path = contracts.getSourceName(rawLocation.file)
+        let path = contracts.getSourceName(rawLocation.file)
+        if (!path) {
+          // check in generated sources
+          for (const source of generatedSources) {
+            if (source.id === rawLocation.file) {
+              path = `browser/.debugger/generated-sources/${source.name}`
+              let content
+              try {
+                content = await this.debuggerModule.call('fileManager', 'getFile', path, source.contents)
+              } catch (e) {
+                console.log('unable to fetch generated sources, the file probably doesn\'t exist yet', e)
+              }
+              if (content !== source.contents) {
+                await this.debuggerModule.call('fileManager', 'setFile', path, source.contents)
+              }
+              break
+            }
+          }
+        }
         if (path) {
           await this.debuggerModule.call('editor', 'discardHighlight')
           await this.debuggerModule.call('editor', 'highlight', lineColumnPos, path)
@@ -137,7 +178,8 @@ class DebuggerUI {
           console.error(e)
         }
         return null
-      }
+      },
+      debugWithGeneratedSources: this.opt.debugWithGeneratedSources
     })
 
     this.listenToEvents()
@@ -167,7 +209,8 @@ class DebuggerUI {
             console.error(e)
           }
           return null
-        }
+        },
+        debugWithGeneratedSources: false
       })
       debug.debugger.traceManager.traceRetriever.getTrace(hash, (error, trace) => {
         if (error) return reject(error)
@@ -188,10 +231,16 @@ class DebuggerUI {
     var view = yo`
       <div>
         <div class="px-2">
+          <div class="mt-3">
+            <p class="mt-2 ${css.debuggerLabel}">Debugger Configuration</p>
+              <div class="mt-2 ${css.debuggerConfig} custom-control custom-checkbox">
+              <input class="custom-control-input" id="debugGeneratedSourcesInput" onchange=${(event) => { this.opt.debugWithGeneratedSources = event.target.checked }} type="checkbox" title="Debug with generated sources">
+              <label data-id="debugGeneratedSourcesLabel" class="form-check-label custom-control-label" for="debugGeneratedSourcesInput">Debug generated sources if available (from Solidity v0.7.2)</label>
+            </div>            
+          </div>       
           ${this.txBrowser.render()}
           ${this.stepManagerView}
-          ${this.debuggerHeadPanelsView}
-        </div>
+          ${this.debuggerHeadPanelsView}          
         <div class="${css.statusMessage}">${this.statusMessage}</div>
         ${this.debuggerPanelsView}
       </div>

apps/remix-ide/src/remixAppManager.js

@@ -10,7 +10,7 @@ const requiredModules = [ // services + layout views + system views
   'terminal', 'settings', 'pluginManager']
 
 export function isNative (name) {
-  const nativePlugins = ['vyper', 'workshops']
+  const nativePlugins = ['vyper', 'workshops', 'debugger']
   return nativePlugins.includes(name) || requiredModules.includes(name)
 }
 

libs/remix-astwalker/src/astWalker.ts

@@ -18,6 +18,14 @@ export function isAstNode(node: Record<string, unknown>): boolean {
   )
 }
 
+export function isYulAstNode(node: Record<string, unknown>): boolean {
+  return (
+    isObject(node) &&
+    'nodeType' in node &&
+    'src' in node
+  )
+}
+
 
 /**
  * Crawl the given AST through the function walk(ast, callback)
@@ -200,7 +208,7 @@ export class AstWalker extends EventEmitter {
   }
   // eslint-disable-next-line @typescript-eslint/ban-types, @typescript-eslint/explicit-module-boundary-types
   walkFullInternal(ast: AstNode, callback: Function) {
-    if (isAstNode(ast)) {
+    if (isAstNode(ast) || isYulAstNode(ast)) {
       // console.log(`XXX id ${ast.id}, nodeType: ${ast.nodeType}, src: ${ast.src}`);
       callback(ast);
       for (const k of Object.keys(ast)) {
@@ -223,7 +231,7 @@ export class AstWalker extends EventEmitter {
   // Normalizes parameter callback and calls walkFullInternal
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
   walkFull(ast: AstNode, callback: any) {
-    if (isAstNode(ast)) return this.walkFullInternal(ast, callback);
+    if (isAstNode(ast) || isYulAstNode(ast)) return this.walkFullInternal(ast, callback);
   }
 
   // eslint-disable-next-line @typescript-eslint/ban-types, @typescript-eslint/explicit-module-boundary-types

libs/remix-astwalker/src/sourceMappings.ts

@@ -1,4 +1,4 @@
-import { isAstNode, AstWalker } from './astWalker';
+import { isAstNode, isYulAstNode, AstWalker } from './astWalker';
 import { AstNode, LineColPosition, LineColRange, Location } from "./types";
 import { util } from "@remix-project/remix-lib";
 
@@ -31,7 +31,7 @@ export function lineColPositionFromOffset(offset: number, lineBreaks: Array<numb
  * @param astNode  The object to convert.
  */
 export function sourceLocationFromAstNode(astNode: AstNode): Location | null {
-  if (isAstNode(astNode) && astNode.src) {
+  if (isAstNode(astNode) && isYulAstNode(astNode) && astNode.src) {
     return sourceLocationFromSrc(astNode.src)
   }
   return null;

libs/remix-debug/src/Ethdebugger.js

@@ -26,6 +26,7 @@ const {SolidityProxy, stateDecoder, localDecoder, InternalCallTree} = require('.
 function Ethdebugger (opts) {
   this.compilationResult = opts.compilationResult || function (contractAddress) { return null }
   this.web3 = opts.web3
+  this.opts = opts
 
   this.event = new EventManager()
 
@@ -36,16 +37,26 @@ function Ethdebugger (opts) {
   this.solidityProxy = new SolidityProxy({getCurrentCalledAddressAt: this.traceManager.getCurrentCalledAddressAt.bind(this.traceManager), getCode: this.codeManager.getCode.bind(this.codeManager)})
   this.storageResolver = null
 
-  this.callTree = new InternalCallTree(this.event, this.traceManager, this.solidityProxy, this.codeManager, { includeLocalVariables: true })
+  const includeLocalVariables = true
+  this.callTree = new InternalCallTree(this.event, 
+      this.traceManager,
+      this.solidityProxy,
+      this.codeManager, 
+      { ...opts, includeLocalVariables})
 }
 
 Ethdebugger.prototype.setManagers = function () {
   this.traceManager = new TraceManager({web3: this.web3})
   this.codeManager = new CodeManager(this.traceManager)
   this.solidityProxy = new SolidityProxy({getCurrentCalledAddressAt: this.traceManager.getCurrentCalledAddressAt.bind(this.traceManager), getCode: this.codeManager.getCode.bind(this.codeManager)})
   this.storageResolver = null
+  const includeLocalVariables = true
 
-  this.callTree = new InternalCallTree(this.event, this.traceManager, this.solidityProxy, this.codeManager, { includeLocalVariables: true })
+  this.callTree = new InternalCallTree(this.event, 
+      this.traceManager, 
+      this.solidityProxy, 
+      this.codeManager, 
+      { ...this.opts, includeLocalVariables})
   this.event.trigger('managersChanged')
 }
 

libs/remix-debug/src/cmdline/index.js

@@ -79,6 +79,7 @@ class CmdLine {
     this.txHash = txNumber
     this.debugger.debug(null, txNumber, null, () => {
       this.debugger.event.register('newSourceLocation', (lineColumnPos, rawLocation) => {
+        if (!lineColumnPos) return
         this.lineColumnPos = lineColumnPos
         this.rawLocation = rawLocation
         this.events.emit('source', [lineColumnPos, rawLocation])

libs/remix-debug/src/debugger/debugger.js

@@ -18,7 +18,8 @@ function Debugger (options) {
 
   this.debugger = new Ethdebugger({
     web3: options.web3,
-    compilationResult: this.compilationResult
+    debugWithGeneratedSources: options.debugWithGeneratedSources,
+    compilationResult: this.compilationResult,
   })
 
   const {traceManager, callTree, solidityProxy} = this.debugger
@@ -59,8 +60,17 @@ Debugger.prototype.registerAndHighlightCodeItem = async function (index) {
 
     this.debugger.callTree.sourceLocationTracker.getValidSourceLocationFromVMTraceIndex(address, index, compilationResultForAddress.data.contracts).then((rawLocation) => {
       if (compilationResultForAddress && compilationResultForAddress.data) {
-        var lineColumnPos = this.offsetToLineColumnConverter.offsetToLineColumn(rawLocation, rawLocation.file, compilationResultForAddress.source.sources, compilationResultForAddress.data.sources)
-        this.event.trigger('newSourceLocation', [lineColumnPos, rawLocation])
+        const generatedSources = this.debugger.callTree.sourceLocationTracker.getGeneratedSourcesFromAddress(address)
+        const astSources = Object.assign({}, compilationResultForAddress.data.sources)
+        const sources = Object.assign({}, compilationResultForAddress.source.sources)
+        if (generatedSources) {
+          for (const genSource of generatedSources) {
+            astSources[genSource.name] = { id: genSource.id, ast: genSource.ast }
+            sources[genSource.name] = { content: genSource.contents }
+          }
+        }
+        var lineColumnPos = this.offsetToLineColumnConverter.offsetToLineColumn(rawLocation, rawLocation.file, sources, astSources)
+        this.event.trigger('newSourceLocation', [lineColumnPos, rawLocation, generatedSources])
       } else {
         this.event.trigger('newSourceLocation', [null])
       }