Transport debug data through block flattener

.circleci/parallel_bytecode_report.sh

@@ -45,7 +45,7 @@ echo "Preparing input files"
 python3 ../scripts/isolate_tests.py ../test/
 
 # FIXME: These cases crash because of https://github.com/ethereum/solidity/issues/13583
-rm ./*_bytecode_too_large_*.sol ./*_combined_too_large_*.sol
+rm ./*_bytecode_too_large_*.sol ./*_combined_too_large_*.sol ./*_initcode_too_large_*.sol
 
 if [[ $binary_type == native || $binary_type == "osx_intel" ]]; then
     interface=$(echo -e "standard-json\ncli" | circleci tests split)

libevmasm/Assembly.cpp

@@ -62,6 +62,7 @@ AssemblyItem const& Assembly::append(AssemblyItem _i)
 	if (!m_items.back().location().isValid() && m_currentSourceLocation.isValid())
 		m_items.back().setLocation(m_currentSourceLocation);
 	m_items.back().m_modifierDepth = m_currentModifierDepth;
+	m_items.back().setDebugAttributes(m_currentDebugAttributes);
 	return m_items.back();
 }
 
@@ -314,6 +315,9 @@ class Functionalizer
 
 		std::string expression = _item.toAssemblyText(m_assembly);
 
+		if (!_item.debugData()->attributes.empty())
+			expression += m_prefix + "  // @debug.set " + _item.debugData()->attributes.dump();
+
 		if (!(
 			_item.canBeFunctional() &&
 			_item.returnValues() <= 1 &&

libevmasm/Assembly.h

@@ -111,6 +111,11 @@ class Assembly
 	/// Changes the source location used for each appended item.
 	void setSourceLocation(langutil::SourceLocation const& _location) { m_currentSourceLocation = _location; }
 	langutil::SourceLocation const& currentSourceLocation() const { return m_currentSourceLocation; }
+
+	/// Changes debug data used for each appended item.
+	void setDebugAttributes(Json const& _debugAttributes) { m_currentDebugAttributes = _debugAttributes; }
+	Json currentDebugAttributes() const { return m_currentDebugAttributes; }
+
 	langutil::EVMVersion const& evmVersion() const { return m_evmVersion; }
 
 	/// Assembles the assembly into bytecode. The assembly should not be modified after this call, since the assembled version is cached.
@@ -246,6 +251,7 @@ class Assembly
 	/// currently
 	std::string m_name;
 	langutil::SourceLocation m_currentSourceLocation;
+	Json m_currentDebugAttributes;
 
 	// FIXME: This being static means that the strings won't be freed when they're no longer needed
 	static std::map<std::string, std::shared_ptr<std::string const>> s_sharedSourceNames;

libevmasm/AssemblyItem.h

@@ -194,6 +194,15 @@ class AssemblyItem
 		m_debugData = std::move(_debugData);
 	}
 
+	void setDebugAttributes(Json const& _debugAttributes)
+	{
+		if (!m_debugData)
+			m_debugData = langutil::DebugData::create({}, {}, {}, _debugAttributes);
+		else
+			m_debugData = langutil::DebugData::create(m_debugData->nativeLocation, m_debugData->originLocation, m_debugData->astID, _debugAttributes);
+	}
+
+
 	langutil::DebugData::ConstPtr debugData() const { return m_debugData; }
 
 	void setJumpType(JumpType _jumpType) { m_jumpType = _jumpType; }

liblangutil/DebugData.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include <liblangutil/SourceLocation.h>
+#include <libsolutil/JSON.h>
 #include <optional>
 #include <memory>
 
@@ -32,23 +33,27 @@ struct DebugData
 	explicit DebugData(
 		langutil::SourceLocation _nativeLocation = {},
 		langutil::SourceLocation _originLocation = {},
-		std::optional<int64_t> _astID = {}
+		std::optional<int64_t> _astID = {},
+		Json _attributes = {}
 	):
 		nativeLocation(std::move(_nativeLocation)),
 		originLocation(std::move(_originLocation)),
-		astID(_astID)
+		astID(_astID),
+		attributes(std::move(_attributes))
 	{}
 
 	static DebugData::ConstPtr create(
 		langutil::SourceLocation _nativeLocation,
 		langutil::SourceLocation _originLocation = {},
-		std::optional<int64_t> _astID = {}
+		std::optional<int64_t> _astID = {},
+		Json _attributes = {}
 	)
 	{
 		return std::make_shared<DebugData>(
 			std::move(_nativeLocation),
 			std::move(_originLocation),
-			_astID
+			_astID,
+			std::move(_attributes)
 		);
 	}
 
@@ -65,6 +70,8 @@ struct DebugData
 	langutil::SourceLocation originLocation;
 	/// ID in the (Solidity) source AST.
 	std::optional<int64_t> astID;
+	/// Additional debug data attributes.
+	Json attributes;
 };
 
 } // namespace solidity::langutil

libyul/AsmParser.cpp

@@ -64,11 +64,11 @@ langutil::DebugData::ConstPtr Parser::createDebugData() const
 	switch (m_useSourceLocationFrom)
 	{
 		case UseSourceLocationFrom::Scanner:
-			return DebugData::create(ParserBase::currentLocation(), ParserBase::currentLocation());
+			return DebugData::create(ParserBase::currentLocation(), ParserBase::currentLocation(), {}, m_currentDebugDataAttributes);
 		case UseSourceLocationFrom::LocationOverride:
-			return DebugData::create(m_locationOverride, m_locationOverride);
+			return DebugData::create(m_locationOverride, m_locationOverride, {}, m_currentDebugDataAttributes);
 		case UseSourceLocationFrom::Comments:
-			return DebugData::create(ParserBase::currentLocation(), m_locationFromComment, m_astIDFromComment);
+			return DebugData::create(ParserBase::currentLocation(), m_locationFromComment, m_astIDFromComment, m_currentDebugDataAttributes);
 	}
 	solAssert(false, "");
 }
@@ -122,8 +122,7 @@ std::unique_ptr<Block> Parser::parseInline(std::shared_ptr<Scanner> const& _scan
 	try
 	{
 		m_scanner = _scanner;
-		if (m_useSourceLocationFrom == UseSourceLocationFrom::Comments)
-			fetchDebugDataFromComment();
+		fetchDebugDataFromComment();
 		return std::make_unique<Block>(parseBlock());
 	}
 	catch (FatalError const&)
@@ -137,24 +136,20 @@ std::unique_ptr<Block> Parser::parseInline(std::shared_ptr<Scanner> const& _scan
 langutil::Token Parser::advance()
 {
 	auto const token = ParserBase::advance();
-	if (m_useSourceLocationFrom == UseSourceLocationFrom::Comments)
-		fetchDebugDataFromComment();
+	fetchDebugDataFromComment();
 	return token;
 }
 
 void Parser::fetchDebugDataFromComment()
 {
-	solAssert(m_sourceNames.has_value(), "");
-
 	static std::regex const tagRegex = std::regex(
-		R"~~((?:^|\s+)(@[a-zA-Z0-9\-_]+)(?:\s+|$))~~", // tag, e.g: @src
+		R"~~((?:^|\s+)(@[a-zA-Z0-9\-\._]+)(?:\s+|$))~~", // tag, e.g: @src
 		std::regex_constants::ECMAScript | std::regex_constants::optimize
 	);
 
 	std::string_view commentLiteral = m_scanner->currentCommentLiteral();
 	std::match_results<std::string_view::const_iterator> match;
 
-	langutil::SourceLocation originLocation = m_locationFromComment;
 	// Empty for each new node.
 	std::optional<int> astID;
 
@@ -165,10 +160,14 @@ void Parser::fetchDebugDataFromComment()
 
 		if (match[1] == "@src")
 		{
-			if (auto parseResult = parseSrcComment(commentLiteral, m_scanner->currentCommentLocation()))
-				tie(commentLiteral, originLocation) = *parseResult;
-			else
-				break;
+			if (m_useSourceLocationFrom == UseSourceLocationFrom::Comments)
+			{
+				solAssert(m_sourceNames.has_value(), "");
+				if (auto parseResult = parseSrcComment(commentLiteral, m_scanner->currentCommentLocation()))
+					tie(commentLiteral, m_locationFromComment) = *parseResult;
+				else
+					break;
+			}
 		}
 		else if (match[1] == "@ast-id")
 		{
@@ -177,15 +176,101 @@ void Parser::fetchDebugDataFromComment()
 			else
 				break;
 		}
+		else if (match[1] == "@debug.set")
+		{
+			if (auto parseResult = parseDebugDataAttributeOperationComment(match[1], commentLiteral, m_scanner->currentCommentLocation()))
+			{
+				commentLiteral = parseResult->first;
+				if (parseResult->second.has_value())
+					m_currentDebugDataAttributes = parseResult->second.value();
+			}
+			else
+				break;
+		}
+		else if (match[1] == "@debug.merge")
+		{
+			if (auto parseResult = parseDebugDataAttributeOperationComment(match[1], commentLiteral, m_scanner->currentCommentLocation()))
+			{
+				commentLiteral = parseResult->first;
+				if (parseResult->second.has_value())
+					m_currentDebugDataAttributes.merge_patch(parseResult->second.value());
+			}
+			else
+				break;
+		}
+		else if (match[1] == "@debug.patch")
+		{
+			if (auto parseResult = parseDebugDataAttributeOperationComment(match[1], commentLiteral, m_scanner->currentCommentLocation()))
+			{
+				commentLiteral = parseResult->first;
+				if (parseResult->second.has_value())
+					applyDebugDataAttributePatch(parseResult->second.value(), m_scanner->currentCommentLocation());
+			}
+			else
+				break;
+		}
 		else
 			// Ignore unrecognized tags.
 			continue;
 	}
 
-	m_locationFromComment = originLocation;
 	m_astIDFromComment = astID;
 }
 
+std::optional<std::pair<std::string_view, std::optional<Json>>> Parser::parseDebugDataAttributeOperationComment(
+	std::string const& _command,
+	std::string_view _arguments,
+	langutil::SourceLocation const& _location
+)
+{
+	std::optional<Json> jsonData;
+	try
+	{
+		jsonData = Json::parse(_arguments.begin(), _arguments.end(), nullptr, true);
+	}
+	catch (nlohmann::json::parse_error& e)
+	{
+		try
+		{
+			jsonData = Json::parse(_arguments.substr(0, e.byte - 1), nullptr, true);
+		}
+		catch(nlohmann::json::parse_error& ee)
+		{
+			m_errorReporter.syntaxError(
+				5721_error,
+				_location,
+			_command + ": Could not parse debug data: " + removeNlohmannInternalErrorIdentifier(ee.what())
+			);
+			jsonData.reset();
+		}
+		_arguments = _arguments.substr(e.byte - 1);
+	}
+	return {{_arguments, jsonData}};
+}
+
+void Parser::applyDebugDataAttributePatch(Json const& _jsonPatch, langutil::SourceLocation const& _location)
+{
+	try
+	{
+		if (_jsonPatch.is_object())
+		{
+			Json array = Json::array();
+			array.push_back(_jsonPatch);
+			m_currentDebugDataAttributes = m_currentDebugDataAttributes.patch(array);
+		}
+		else
+			m_currentDebugDataAttributes = m_currentDebugDataAttributes.patch(_jsonPatch);
+	}
+	catch(nlohmann::json::parse_error& ee)
+	{
+		m_errorReporter.syntaxError(
+			9426_error,
+			_location,
+			"@debug.patch: Could not patch debug data: " + removeNlohmannInternalErrorIdentifier(ee.what())
+		);
+	}
+}
+
 std::optional<std::pair<std::string_view, SourceLocation>> Parser::parseSrcComment(
 	std::string_view const _arguments,
 	langutil::SourceLocation const& _commentLocation

libyul/AsmParser.h

@@ -117,6 +117,14 @@ class Parser: public langutil::ParserBase
 		langutil::SourceLocation const& _commentLocation
 	);
 
+	std::optional<std::pair<std::string_view, std::optional<Json>>> parseDebugDataAttributeOperationComment(
+		std::string const& _command,
+		std::string_view _arguments,
+		langutil::SourceLocation const& _commentLocation
+	);
+
+	void applyDebugDataAttributePatch(Json const& _jsonPatch, langutil::SourceLocation const& _location);
+
 	/// Creates a DebugData object with the correct source location set.
 	langutil::DebugData::ConstPtr createDebugData() const;
 
@@ -163,6 +171,7 @@ class Parser: public langutil::ParserBase
 	UseSourceLocationFrom m_useSourceLocationFrom = UseSourceLocationFrom::Scanner;
 	ForLoopComponent m_currentForLoopComponent = ForLoopComponent::None;
 	bool m_insideFunction = false;
+	Json m_currentDebugDataAttributes = Json::object();
 };
 
 }