[pull] master from sqlmapproject:master

lib/controller/checks.py

@@ -1223,6 +1223,14 @@ def _(page):
         if conf.beep:
             beep()
 
+    def _search(regex):
+        # Note: on a rare (e.g. huge) response the regex engine itself can fail, and losing one
+        # advisory heuristic beats losing the whole run (e.g. #5994 and #6105)
+        try:
+            return re.search(regex, page or "")
+        except (SystemError, RuntimeError) as ex:
+            logger.debug("skipping heuristic check because of a regex engine failure ('%s')" % getSafeExString(ex))
+
     try:
         for match in re.finditer(FI_ERROR_REGEX, page or ""):
             if randStr1.lower() in match.group(0).lower():
@@ -1234,71 +1242,71 @@ def _(page):
 
                 break
     except (SystemError, RuntimeError) as ex:
-        logger.debug("Skipping FI heuristic due to regex failure: %s", getSafeExString(ex))
+        logger.debug("skipping heuristic check because of a regex engine failure ('%s')" % getSafeExString(ex))
 
-    if not conf.nosql and re.search(NOSQL_ERROR_REGEX, page or ""):
+    if not conf.nosql and _search(NOSQL_ERROR_REGEX):
         infoMsg = "heuristic (NoSQL) test shows that %sparameter '%s' might be vulnerable to NoSQL injection attacks (rerun with switch '--nosql')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
 
         if conf.beep:
             beep()
 
-    if not conf.graphql and re.search(GRAPHQL_ERROR_REGEX, page or ""):
+    if not conf.graphql and _search(GRAPHQL_ERROR_REGEX):
         infoMsg = "heuristic (GraphQL) test shows that %sparameter '%s' appears to be a GraphQL endpoint (rerun with switch '--graphql')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
 
         if conf.beep:
             beep()
 
-    if not conf.ldap and re.search(LDAP_ERROR_REGEX, page or ""):
+    if not conf.ldap and _search(LDAP_ERROR_REGEX):
         infoMsg = "heuristic (LDAP) test shows that %sparameter '%s' might be vulnerable to LDAP injection (rerun with switch '--ldap')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
 
         if conf.beep:
             beep()
 
-    if not conf.xpath and re.search(XPATH_ERROR_REGEX, page or ""):
+    if not conf.xpath and _search(XPATH_ERROR_REGEX):
         infoMsg = "heuristic (XPath) test shows that %sparameter '%s' might be vulnerable to XPath injection (rerun with switch '--xpath')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
 
         if conf.beep:
             beep()
 
-    if not conf.ssti and re.search(SSTI_ERROR_REGEX, page or ""):
+    if not conf.ssti and _search(SSTI_ERROR_REGEX):
         infoMsg = "heuristic (SSTI) test shows that %sparameter '%s' might be vulnerable to server-side template injection (rerun with switch '--ssti')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
 
         if conf.beep:
             beep()
 
-    if not conf.hql and re.search(HQL_ERROR_REGEX, page or ""):
+    if not conf.hql and _search(HQL_ERROR_REGEX):
         infoMsg = "heuristic (HQL) test shows that %sparameter '%s' might be vulnerable to HQL/JPQL (Hibernate ORM) injection (rerun with switch '--hql')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
 
         if conf.beep:
             beep()
 
-    if not conf.xslt and re.search(XSLT_ERROR_REGEX, page or ""):
+    if not conf.xslt and _search(XSLT_ERROR_REGEX):
         infoMsg = "heuristic (XSLT) test shows that %sparameter '%s' might be vulnerable to XSLT injection (rerun with switch '--xslt')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
         if conf.beep:
             beep()
 
-    if not conf.sparql and re.search(SPARQL_ERROR_REGEX, page or ""):
+    if not conf.sparql and _search(SPARQL_ERROR_REGEX):
         infoMsg = "heuristic (SPARQL) test shows that %sparameter '%s' might be vulnerable to SPARQL injection (rerun with switch '--sparql')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
 
         if conf.beep:
             beep()
 
-    if not conf.odata and re.search(ODATA_ERROR_REGEX, page or ""):
+    if not conf.odata and _search(ODATA_ERROR_REGEX):
         infoMsg = "heuristic (OData) test shows that %sparameter '%s' might be vulnerable to OData $filter injection (rerun with switch '--odata')" % ("%s " % paramType if paramType != parameter else "", parameter)
         logger.info(infoMsg)
 
         if conf.beep:
             beep()
 
-    if not conf.xxe and kb.postHint in (POST_HINT.XML, POST_HINT.SOAP) and re.search(XXE_ERROR_REGEX, page or ""):
+    if not conf.xxe and kb.postHint in (POST_HINT.XML, POST_HINT.SOAP) and _search(XXE_ERROR_REGEX):
         infoMsg = "heuristic (XXE) test shows that the XML request body might be vulnerable to XML External Entity injection (rerun with switch '--xxe')"
         logger.info(infoMsg)
 

lib/core/option.py

@@ -905,6 +905,13 @@ def _setTamperingFunctions():
             priority = PRIORITY.NORMAL if not hasattr(module, "__priority__") else module.__priority__
             priority = priority if priority is not None else PRIORITY.LOWEST
 
+            if not isinstance(priority, int):
+                warnMsg = "tamper module '%s' has an invalid value for '__priority__' " % filename[:-3]
+                warnMsg += "(assuming '%d')" % PRIORITY.NORMAL
+                logger.warning(warnMsg)
+
+                priority = PRIORITY.NORMAL
+
             for name, function in inspect.getmembers(module, inspect.isfunction):
                 if name == "tamper" and (hasattr(inspect, "signature") and all(_ in inspect.signature(function).parameters for _ in ("payload", "kwargs")) or inspect.getargspec(function).args and inspect.getargspec(function).keywords == "kwargs"):
                     found = True

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.9.2"
+VERSION = "1.10.9.4"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -1193,7 +1193,10 @@
     ("Python ElementTree", r"xml\.etree\.ElementTree\.(?:ParseError|Element)"),
     # NOT XSLT: a dedicated '--xslt' engine owns those errors now, and claiming them here made every
     # XSLT parser error suggest '--xpath' as well
-    ("Generic XPath", r"XPath.*?(?:error|exception|syntax)"),
+    # NOTE: the gap has to stay bounded (like in 'Handlebars' below). An unbounded '.*?' turns this
+    # into a quadratic scan of every long line that merely carries the word 'xpath' (e.g. minified
+    # JS/JSON), which took ~25s on a 400KB response - and blew up the regex engine itself (#6105)
+    ("Generic XPath", r"XPath[^\n]{0,100}?(?:error|exception|syntax)"),
     ("Generic XPath", r"Invalid XPath|XPath evaluation failed"),
 )
 
@@ -1337,7 +1340,7 @@
     ("Velocity", r"org\.apache\.velocity\.(?:runtime|exception)\.\w+|ParseErrorException|MethodInvocationException|ResourceNotFoundException"),
     ("Spring EL / Thymeleaf", r"org\.springframework\.expression\.\w+|org\.thymeleaf\.\w+|SpelEvaluationException|TemplateProcessingException|ExpressionParsingException"),
     ("Struts2 (OGNL)", r"ognl\.(?:OgnlException|NoSuchPropertyException|MethodFailedException|InappropriateExpressionException|ExpressionSyntaxException)|com\.opensymphony\.xwork2|org\.apache\.struts2|There is no Action mapped for|Struts (?:Problem Report|has detected an unhandled exception)"),
-    ("ERB", r"\(erb\):\d+|NameError.*undefined local variable"),
+    ("ERB", r"\(erb\):\d+|NameError[^\n]{0,100}?undefined local variable"),
     # NOTE: these must stay anchored to a diagnostic. The bare product names matched any page that
     # carries the word 'pug'/'jade'/'handlebars' (a surname, a colour, a <script src=> of the runtime),
     # and the bare 'ParseError' matched lxml.etree.XSLTParseError and ElementTree.ParseError
@@ -1361,7 +1364,7 @@
     ("Java (Xerces/JAXP)", r"(?:org\.xml\.sax\.SAXParseException|com\.sun\.org\.apache\.xerces|javax\.xml\.stream\.XMLStreamException|The (?:entity|element type) \"[^\"]*\" was referenced|DOCTYPE is disallowed when the feature|External (?:DTD|parsed entities|Entity): failed|\"[^\"]*\" must be declared|had to be read but the maximum)"),
     (".NET System.Xml", r"(?:System\.Xml\.XmlException|For security reasons DTD is prohibited|Reference to undeclared entity|An error occurred while parsing EntityName|XmlTextReaderImpl)"),
     ("Python expat", r"(?:xml\.parsers\.expat\.ExpatError|undefined entity|not well-formed \(invalid token\)|ExpatError)"),
-    ("Ruby Nokogiri/REXML", r"(?:Nokogiri::XML::SyntaxError|REXML::ParseException|Entity .* not defined)"),
+    ("Ruby Nokogiri/REXML", r"(?:Nokogiri::XML::SyntaxError|REXML::ParseException|Entity [^\n]{0,100}? not defined)"),
     ("Go encoding/xml", r"XML syntax error on line \d+"),
     # NOTE: 'unexpected end of ...' is what every parser says, not what an XML parser says. It matched
     # the "Unexpected end of query" of BaseX, the "Unexpected <EOF>" of GraphQL and the "Unexpected end
@@ -1417,7 +1420,7 @@
     ("Hibernate", r"(?:unexpected (?:token:|end of subtree|AST node)|Could not (?:resolve|interpret) (?:attribute|root entity|path|property))"),
     ("EclipseLink / JPQL", r"(?:org\.eclipse\.persistence\.exceptions\.JPQLException|Exception \[EclipseLink|Problem compiling \[|An exception occurred while creating a query)"),
     ("JPA / JPQL", r"(?:javax|jakarta)\.persistence\.(?:PersistenceException|Query(?:Syntax|Timeout)?Exception)"),
-    ("Generic HQL/JPQL", r"(?:HQL|JPQL|EJBQL)\b.*?(?:error|exception|syntax|not (?:mapped|resolve))"),
+    ("Generic HQL/JPQL", r"(?:HQL|JPQL|EJBQL)\b[^\n]{0,100}?(?:error|exception|syntax|not (?:mapped|resolve))"),
 )
 
 HQL_ERROR_REGEX = r"(?i)(?:%s)" % '|'.join(regex for _, regex in HQL_ERROR_SIGNATURES)
@@ -1484,7 +1487,7 @@
     # too - matching on it alone mislabelled them as RDF4J, so only the package name is kept
     ("RDF4J / GraphDB", r"org\.eclipse\.rdf4j|org\.openrdf\.query"),
     ("Blazegraph", r"com\.bigdata\.rdf|\bBlazegraph\b"),
-    ("rdflib", r"rdflib\.plugins\.sparql|\bParseException\b.*?(?:SPARQL|sparql)"),
+    ("rdflib", r"rdflib\.plugins\.sparql|\bParseException\b[^\n]{0,100}?(?:SPARQL|sparql)"),
     ("Stardog", r"com\.(?:complexible\.)?stardog"),
 )
 

tests/test_heuristic_signatures.py

@@ -26,6 +26,7 @@
 import os
 import re
 import sys
+import time
 import unittest
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
@@ -223,6 +224,26 @@ def test_nothing_fires_on_sql_errors_or_ordinary_pages(self):
             self.assertEqual(fired, (),
                              msg="%s output suggests %s: %r" % (backend, '/'.join("'--%s'" % _ for _ in fired), text))
 
+    def test_no_signature_has_an_unbounded_gap(self):
+        # an unbounded '.*' between a literal and a keyword is quadratic on a long line (minified
+        # JS/JSON, a one-line JSON error body), which is how a stray 'xpath' in a 400KB response
+        # cost ~25s per parameter - and made the regex engine itself blow up (#5994, #6105).
+        # The bounded form ('[^\n]{0,100}?') matches the same real errors in constant work
+        for name, regex in ENGINES:
+            found = re.search(r"(?<!\\)\.[*+]", regex)
+            self.assertIsNone(found, msg="'--%s' signatures carry an unbounded '%s' gap" % (name, found.group(0) if found else ""))
+
+    def test_signatures_stay_linear_on_a_long_line(self):
+        # the same invariant, measured: every engine has to survive a single-line response that
+        # carries the words its signatures start with, without any error actually being present
+        page = ("<div class=\"x\">lorem ipsum dolor sit amet consectetur adipiscing elit</div>" + ''.join("%s " % _ for _ in ("xpath", "HQL", "ParseException", "NameError", "Entity x", "handlebars", "no such file", "Twig"))) * 2000
+
+        for name, regex in ENGINES:
+            start = time.time()
+            re.search(regex, page)
+            elapsed = time.time() - start
+            self.assertLess(elapsed, 2, msg="'--%s' signatures took %.1fs on a %dKB single-line response" % (name, elapsed, len(page) // 1024))
+
     def test_every_engine_is_covered(self):
         # a new switch must arrive here with its own errors, or the matrix above proves nothing about it
         owners = set(owner for owner, _, _ in CORPUS)