[pull] master from sqlmapproject:master

extra/kerberos/client.py

@@ -309,6 +309,15 @@ def _asReq(realm, username, etypes, nonce, padata=None):
     parts.append(der.tagged(4, reqBody))
     return der.application(AS_REQ, der.sequence(*parts))
 
+def _selectEtype(etypes, hints):
+    """
+    The etype getTGT commits to after a preauth-required hint: OUR first offered etype that is also
+    KDC-hinted and supported, falling back to our top preference. The unauthenticated hint can only
+    reorder WITHIN what we offered - it can never pull us onto an etype we did not offer (anti-downgrade).
+    """
+
+    return next((_ for _ in etypes if _ in hints and _ in ENCTYPES), etypes[0])
+
 def getTGT(realm, username, password, kdcHost, kdcPort=88, etypes=DEFAULT_ETYPES, salt=None):
     """Run the AS exchange and return the TGT and its session key.
 
@@ -343,10 +352,7 @@ def getTGT(realm, username, password, kdcHost, kdcPort=88, etypes=DEFAULT_ETYPES
         # the hint is unauthenticated, so it may only choose among the etypes we actually offered, and
         # in *our* order of preference rather than the KDC's (otherwise it could force a downgrade)
         hints = _preauthHints(errorFields)
-        for offered in etypes:
-            if offered in hints and offered in ENCTYPES:
-                etype = offered
-                break
+        etype = _selectEtype(etypes, hints)
         chosenSalt, iterations = _hintFor(hints, etype, salt, chosenSalt)
 
     enc = _enctype(etype)

lib/core/common.py

@@ -183,6 +183,7 @@
 from lib.core.settings import STRUCTURAL_TAG_REGEX
 from lib.core.settings import SUPPORTED_DBMS
 from lib.core.settings import TEXT_TAG_REGEX
+from lib.core.settings import TIME_OUTLIER_MAD_COEFF
 from lib.core.settings import TIME_STDEV_COEFF
 from lib.core.settings import UNICODE_ENCODING
 from lib.core.settings import UNKNOWN_DBMS_VERSION
@@ -2009,7 +2010,10 @@ def parseUnionPage(page):
                 entry = entry.split(kb.chars.start)[-1]
 
             if kb.unionDuplicates:
-                key = entry.lower()
+                # Note: de-dup on the EXACT entry, not entry.lower() - the doubled emission repeats each
+                # row verbatim, so case-folding here would silently drop genuinely case-distinct rows
+                # (e.g. 'Admin' vs 'admin'). Force-uppercased pages are already lower-cased before parsing.
+                key = entry
                 if key not in keys:
                     keys.add(key)
                 else:
@@ -2877,6 +2881,38 @@ def wasLastResponseHTTPError():
     threadData = getCurrentThreadData()
     return threadData.lastHTTPError and threadData.lastHTTPError[0] == threadData.lastRequestUID
 
+def stripTimeOutliers(values):
+    """
+    Returns L{values} with high (spike) outliers removed, using a robust median/MAD cutoff.
+
+    A single network spike that lands in the time-response model would otherwise inflate both the
+    average and the standard deviation, exploding the delay threshold (avg + 7*stdev) so that genuine
+    time-delays are no longer recognized. MAD is robust to a minority of outliers, so the cutoff is
+    computed from the clean bulk even when the sample already contains a spike. On a clean model no
+    value exceeds median + 10*MAD, so it is returned unchanged (identical avg/stdev/threshold).
+
+    >>> len(stripTimeOutliers([0.1, 0.12] * 8 + [9.0]))   # a lone 9s spike is dropped (17->16)
+    16
+    >>> len(stripTimeOutliers([0.1, 0.12] * 8))           # a clean model is left intact (no-op)
+    16
+    """
+
+    if not values or len(values) < MIN_TIME_RESPONSES // 2:
+        return values
+
+    ordered = sorted(values)
+    median = ordered[len(ordered) // 2]
+    mad = sorted(abs(_ - median) for _ in values)[len(values) // 2]
+
+    if mad <= 0:  # degenerate (near-constant model) - nothing robust to trim on
+        return values
+
+    cutoff = median + TIME_OUTLIER_MAD_COEFF * 1.4826 * mad
+    retVal = [_ for _ in values if _ <= cutoff]
+
+    # never trim away the bulk (guards a genuinely wide/bimodal model from being gutted)
+    return retVal if len(retVal) >= max(MIN_TIME_RESPONSES // 2, len(values) // 2) else values
+
 def wasLastResponseDelayed():
     """
     Returns True if the last web request resulted in a time-delay
@@ -2886,7 +2922,10 @@ def wasLastResponseDelayed():
     # response times should be inside +-7*stdev([normal response times])
     # Math reference: http://www.answers.com/topic/standard-deviation
 
-    deviation = stdev(kb.responseTimes.get(kb.responseTimeMode, []))
+    # spike outliers (e.g. a GC pause / retransmit during baseline sampling) are stripped first, so a
+    # single lagging response can't inflate the model and hide every genuine delay behind it
+    sample = stripTimeOutliers(kb.responseTimes.get(kb.responseTimeMode, []))
+    deviation = stdev(sample)
     threadData = getCurrentThreadData()
 
     if deviation and not conf.direct and not conf.disableStats:
@@ -2895,7 +2934,7 @@ def wasLastResponseDelayed():
             warnMsg += "with less than %d response times" % MIN_TIME_RESPONSES
             logger.warning(warnMsg)
 
-        lowerStdLimit = average(kb.responseTimes[kb.responseTimeMode]) + TIME_STDEV_COEFF * deviation
+        lowerStdLimit = average(sample) + TIME_STDEV_COEFF * deviation
         retVal = (threadData.lastQueryDuration >= max(MIN_VALID_DELAYED_RESPONSE, lowerStdLimit))
 
         if not kb.testMode and retVal:
@@ -3859,7 +3898,7 @@ def joinValue(value, delimiter=','):
     """
 
     if isListLike(value):
-        retVal = delimiter.join(getText(_ if _ is not None else "None") for _ in value)
+        retVal = delimiter.join(getText(getUnicode(_) if _ is not None else "None") for _ in value)
     else:
         retVal = value
 
@@ -4264,9 +4303,14 @@ def decodeStringEscape(value):
     retVal = value
 
     if value and '\\' in value:
-        charset = "\\%s" % string.whitespace.replace(" ", "")
-        for _ in charset:
+        # Note: shield an escaped backslash ('\\\\') behind a marker BEFORE decoding the whitespace
+        # escapes, then restore it - otherwise decoding '\\\\' -> '\\' first turns a literal '\\n'
+        # into a newline (i.e. the round-trip with encodeStringEscape was not lossless)
+        _marker = "\x00"
+        retVal = retVal.replace("\\\\", _marker)
+        for _ in string.whitespace.replace(" ", ""):
             retVal = retVal.replace(repr(_).strip("'"), _)
+        retVal = retVal.replace(_marker, "\\")
 
     return retVal
 

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.228"
+VERSION = "1.10.7.240"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -232,6 +232,12 @@
 # Coefficient used for a time-based query delay checking (must be >= 7)
 TIME_STDEV_COEFF = 7
 
+# Robust (median/MAD) cutoff for discarding spike outliers from the time-response model before
+# computing avg/stdev - a single network spike landing in the baseline would otherwise inflate the
+# delay threshold and miss genuine delays. Deliberately wide (~10 robust sigmas) so a clean model is
+# left untouched (identical threshold) and only true outliers are dropped.
+TIME_OUTLIER_MAD_COEFF = 10
+
 # Minimum response time that can be even considered as delayed (not a complete requirement)
 MIN_VALID_DELAYED_RESPONSE = 0.5
 

lib/parse/cmdline.py

@@ -1171,7 +1171,7 @@ def _format_action_invocation(self, action):
                 dataToStdout("[!] detected usage of long-option without a starting hyphen ('%s')\n" % argv[i])
                 raise SystemExit
 
-        for verbosity in (_ for _ in argv if re.search(r"\A\-v+\Z", _)):
+        for verbosity in [_ for _ in argv if re.search(r"\A\-v+\Z", _)]:  # Note: list (not a generator) - the loop mutates argv (del), which would desync a live generator and leave a stray '-v' for argparse to choke on
             try:
                 if argv.index(verbosity) == len(argv) - 1 or not argv[argv.index(verbosity) + 1].isdigit():
                     conf.verbose = verbosity.count('v')

lib/parse/configfile.py

@@ -84,14 +84,14 @@ def configFileParser(configFile):
 
     mandatory = False
 
-    for option in ("direct", "url", "logFile", "bulkFile", "googleDork", "requestFile", "wizard"):
+    for option in ("direct", "url", "logFile", "bulkFile", "googleDork", "requestFile", "openApiFile", "wizard"):
         if config.has_option("Target", option) and config.get("Target", option) or cmdLineOptions.get(option):
             mandatory = True
             break
 
     if not mandatory:
         errMsg = "missing a mandatory option in the configuration file "
-        errMsg += "(direct, url, logFile, bulkFile, googleDork, requestFile or wizard)"
+        errMsg += "(direct, url, logFile, bulkFile, googleDork, requestFile, openApiFile or wizard)"
         raise SqlmapMissingMandatoryOptionException(errMsg)
 
     for family, optionData in optDict.items():

lib/utils/har.py

@@ -116,7 +116,7 @@ def toDict(self):
             "httpVersion": self.httpVersion,
             "method": self.method,
             "url": self.url,
-            "headers": [dict(name=key.capitalize(), value=value) for key, value in self.headers.items()],
+            "headers": [dict(name="-".join(_.capitalize() for _ in key.split("-")), value=value) for key, value in self.headers.items()],
             "cookies": [],
             "queryString": [],
             "headersSize": -1,
@@ -202,7 +202,7 @@ def toDict(self):
             "httpVersion": self.httpVersion,
             "status": self.status,
             "statusText": self.statusText,
-            "headers": [dict(name=key.capitalize(), value=value) for key, value in self.headers.items() if key.lower() != "uri"],
+            "headers": [dict(name="-".join(_.capitalize() for _ in key.split("-")), value=value) for key, value in self.headers.items() if key.lower() != "uri"],
             "cookies": [],
             "content": content,
             "headersSize": -1,

lib/utils/safe2bin.py

@@ -55,6 +55,9 @@ def safecharencode(value):
 
     if isinstance(value, string_types):
         if any(_ not in SAFE_CHARS for _ in value):
+            # NOTE (checked twice, do NOT "fix" by deleting): this marker keeps an already-`\x`-carrying
+            # value (incl. getUnicode's 'reversible' \xNN) from double-escaping to `\\x` in console/CSV.
+            # Dropping it only swaps that for the rare literal-`\x`-in-data round-trip - a bad trade.
             retVal = retVal.replace(HEX_ENCODED_PREFIX, HEX_ENCODED_PREFIX_MARKER)
             retVal = retVal.replace('\\', SLASH_MARKER)
 

plugins/generic/databases.py

@@ -418,9 +418,9 @@ def getTables(self, bruteForce=None):
                     else:
                         for index in indexRange:
                             if Backend.isDbms(DBMS.SYBASE):
-                                query = _query % (db, (kb.data.cachedTables[-1] if kb.data.cachedTables else " "))
+                                query = _query % (db, (tables[-1] if tables else " "))
                             elif Backend.getIdentifiedDbms() in (DBMS.MAXDB, DBMS.ACCESS, DBMS.MCKOI, DBMS.EXTREMEDB):
-                                query = _query % (kb.data.cachedTables[-1] if kb.data.cachedTables else " ")
+                                query = _query % (tables[-1] if tables else " ")
                             elif Backend.getIdentifiedDbms() in (DBMS.SQLITE, DBMS.FIREBIRD):
                                 query = _query % index
                             elif Backend.getIdentifiedDbms() in (DBMS.HSQLDB, DBMS.INFORMIX, DBMS.FRONTBASE, DBMS.VIRTUOSO):

plugins/generic/users.py

@@ -648,13 +648,16 @@ def getPrivileges(self, query2=False):
                         break
 
                 if privileges:
-                    kb.data.cachedUsersPrivileges[user] = list(privileges)
+                    # Note: 'user' may be a LIKE-wrapped form (e.g. '%root%') built for the MySQL
+                    # query above; key/record under the real name so the output isn't mislabelled
+                    # (and the retrievedUsers de-dup check, which compares the unwrapped name, works)
+                    kb.data.cachedUsersPrivileges[outuser] = list(privileges)
                 else:
                     warnMsg = "unable to retrieve the privileges "
                     warnMsg += "for user '%s'" % outuser
                     logger.warning(warnMsg)
 
-                retrievedUsers.add(user)
+                retrievedUsers.add(outuser)
 
         if not kb.data.cachedUsersPrivileges:
             errMsg = "unable to retrieve the privileges "

tamper/charunicodeescape.py

@@ -33,7 +33,13 @@ def tamper(payload, **kwargs):
                 retVal += "\\u00%s" % payload[i + 1:i + 3]
                 i += 3
             else:
-                retVal += '\\u%.4X' % ord(payload[i])
+                ordinal = ord(payload[i])
+                if ordinal > 0xFFFF:
+                    # non-BMP: emit a UTF-16 surrogate pair (a bare 5-hex '\uXXXXX' is invalid)
+                    ordinal -= 0x10000
+                    retVal += "\\u%04X\\u%04X" % (0xD800 + (ordinal >> 10), 0xDC00 + (ordinal & 0x3FF))
+                else:
+                    retVal += "\\u%04X" % ordinal
                 i += 1
 
     return retVal

tamper/if2case.py

@@ -14,6 +14,34 @@
 def dependencies():
     pass
 
+def _unwrap(expr):
+    """
+    Strips only FULLY-wrapping outer parentheses (e.g. '(1=1)' -> '1=1'), leaving a bare function
+    call such as 'SLEEP(5)' intact - unlike str.strip('()') which would drop its trailing ')'
+    """
+
+    expr = expr.strip()
+
+    while len(expr) > 1 and expr[0] == '(' and expr[-1] == ')':
+        depth = 0
+        wrapper = True
+
+        for i in xrange(len(expr)):
+            if expr[i] == '(':
+                depth += 1
+            elif expr[i] == ')':
+                depth -= 1
+                if depth == 0 and i != len(expr) - 1:  # the opening '(' closes before the end
+                    wrapper = False
+                    break
+
+        if not wrapper:
+            break
+
+        expr = expr[1:-1].strip()
+
+    return expr
+
 def tamper(payload, **kwargs):
     """
     Replaces instances like 'IF(A, B, C)' with 'CASE WHEN (A) THEN (B) ELSE (C) END' counterpart
@@ -62,9 +90,9 @@ def tamper(payload, **kwargs):
                         depth -= 1
 
             if len(commas) == 2 and end:
-                a = payload[index + len("IF("):commas[0]].strip("()")
-                b = payload[commas[0] + 1:commas[1]].lstrip().strip("()")
-                c = payload[commas[1] + 1:end].lstrip().strip("()")
+                a = _unwrap(payload[index + len("IF("):commas[0]])
+                b = _unwrap(payload[commas[0] + 1:commas[1]])
+                c = _unwrap(payload[commas[1] + 1:end])
                 newVal = "CASE WHEN (%s) THEN (%s) ELSE (%s) END" % (a, b, c)
                 payload = payload[:index] + newVal + payload[end + 1:]
             else:

tamper/overlongutf8.py

@@ -38,7 +38,12 @@ def tamper(payload, **kwargs):
                 i += 3
             else:
                 if payload[i] not in (string.ascii_letters + string.digits):
-                    retVal += "%%%.2X%%%.2X" % (0xc0 + (ord(payload[i]) >> 6), 0x80 + (ord(payload[i]) & 0x3f))
+                    ordinal = ord(payload[i])
+                    if ordinal <= 0x7FF:
+                        retVal += "%%%.2X%%%.2X" % (0xc0 + (ordinal >> 6), 0x80 + (ordinal & 0x3f))
+                    else:
+                        # the 2-byte overlong form can't hold code points > U+07FF; fall back to real UTF-8
+                        retVal += "".join("%%%.2X" % _ for _ in bytearray(payload[i].encode("utf8")))
                 else:
                     retVal += payload[i]
                 i += 1

tamper/overlongutf8more.py

@@ -37,7 +37,12 @@ def tamper(payload, **kwargs):
                 retVal += payload[i:i + 3]
                 i += 3
             else:
-                retVal += "%%%.2X%%%.2X" % (0xc0 + (ord(payload[i]) >> 6), 0x80 + (ord(payload[i]) & 0x3f))
+                ordinal = ord(payload[i])
+                if ordinal <= 0x7FF:
+                    retVal += "%%%.2X%%%.2X" % (0xc0 + (ordinal >> 6), 0x80 + (ordinal & 0x3f))
+                else:
+                    # the 2-byte overlong form can't hold code points > U+07FF; fall back to real UTF-8
+                    retVal += "".join("%%%.2X" % _ for _ in bytearray(payload[i].encode("utf8")))
                 i += 1
 
     return retVal

tests/test_error_engine.py

@@ -26,6 +26,7 @@
 from _testutils import bootstrap, set_dbms, reset_dbms
 bootstrap()
 
+from lib.core.common import getCurrentThreadData, setTechnique
 from lib.core.data import conf, kb
 from lib.core.datatype import AttribDict
 from lib.core.enums import PAYLOAD, PLACE
@@ -52,6 +53,7 @@ def setUp(self):
             "conf.paramDict": conf.get("paramDict"), "conf.base64Parameter": conf.get("base64Parameter"),
             "kb.errorChunkLength": kb.get("errorChunkLength"), "kb.testMode": kb.get("testMode"),
             "kb.forceWhere": kb.get("forceWhere"), "kb.technique": kb.get("technique"),
+            "td.technique": getCurrentThreadData().technique,
             "kb.inj": (kb.injection.place, kb.injection.parameter, kb.injection.data),
             "qp": Connect.queryPage,
         }
@@ -67,6 +69,7 @@ def setUp(self):
         kb.injection.place = PLACE.GET
         kb.injection.parameter = "id"
         kb.technique = PAYLOAD.TECHNIQUE.ERROR
+        setTechnique(PAYLOAD.TECHNIQUE.ERROR)   # getTechnique() prefers the thread-local; set it so a leaked one can't poison us
         kb.injection.data = {PAYLOAD.TECHNIQUE.ERROR: _make_vector()}
         set_dbms("MySQL")
 
@@ -81,6 +84,7 @@ def tearDown(self):
         kb.testMode = self._saved["kb.testMode"]
         kb.forceWhere = self._saved["kb.forceWhere"]
         kb.technique = self._saved["kb.technique"]
+        setTechnique(self._saved["td.technique"])
         kb.injection.place, kb.injection.parameter, kb.injection.data = self._saved["kb.inj"]
         Connect.queryPage = self._saved["qp"]
         eu.Request.queryPage = self._saved["qp"]
@@ -126,6 +130,7 @@ def setUp(self):
             "paramDict": conf.get("paramDict"), "base64Parameter": conf.get("base64Parameter"),
             "errorChunkLength": kb.get("errorChunkLength"), "testMode": kb.get("testMode"),
             "forceWhere": kb.get("forceWhere"), "technique": kb.get("technique"),
+            "td.technique": getCurrentThreadData().technique,
             "inj": (kb.injection.place, kb.injection.parameter, kb.injection.data),
             "qp": Connect.queryPage,
             "dbmsHandler": conf.get("dbmsHandler"), "forceDbms": conf.get("forceDbms"),
@@ -145,6 +150,7 @@ def setUp(self):
         kb.injection.place = PLACE.GET
         kb.injection.parameter = "id"
         kb.technique = PAYLOAD.TECHNIQUE.ERROR
+        setTechnique(PAYLOAD.TECHNIQUE.ERROR)   # getTechnique() prefers the thread-local; set it so a leaked one can't poison us
         kb.injection.data = {PAYLOAD.TECHNIQUE.ERROR: _make_vector()}
         # With testMode=False, getIdentifiedDbms() prefers conf.dbmsHandler._dbms and conf.forceDbms
         # over the forced DBMS below; a leaked handler/option (e.g. MSSQL) would make the chunk-length
@@ -164,6 +170,7 @@ def tearDown(self):
         kb.testMode = self._saved["testMode"]
         kb.forceWhere = self._saved["forceWhere"]
         kb.technique = self._saved["technique"]
+        setTechnique(self._saved["td.technique"])
         kb.injection.place, kb.injection.parameter, kb.injection.data = self._saved["inj"]
         Connect.queryPage = self._saved["qp"]
         eu.Request.queryPage = self._saved["qp"]