[pull] master from sqlmapproject:master

.gitignore

@@ -16,3 +16,4 @@ plugins/.DS_Store
 thirdparty/.DS_Store
 CLAUDE.md
 .coverage
+.codegraph/

lib/controller/checks.py

@@ -554,7 +554,8 @@ def genCmpPayload():
                             trueRawResponse = "%s%s" % (trueHeaders, truePage)
 
                             if conf.lengths:
-                                kb.trueLength = len(truePage)
+                                # under NULL connection the body is absent, so take the length HEAD/Range reported
+                                kb.trueLength = threadData.lastComparisonPageLength if kb.nullConnection else len(truePage)
                                 trueResult = True
 
                             if trueResult and not (truePage == falsePage and not any((kb.nullConnection, conf.code))):

lib/core/agent.py

@@ -1041,8 +1041,14 @@ def limitCondition(self, expression, dump=False):
         else:
             limitRegExp2 = None
 
+        # DBMSes whose paging is a simple appended "LIMIT ... OFFSET ..." (limitQuery '% (1, num)' group,
+        # limitregexp 'query2' for a bare LIMIT, groupstart=2/stop=1): a user-supplied LIMIT must be parsed
+        # AND stripped here, else limitQuery appends a SECOND LIMIT (e.g. '... LIMIT 3 LIMIT 1 OFFSET n' on
+        # ClickHouse) and per-row extraction (blind/error) reads bogus offsets - only UNION was unaffected.
+        limitOffsetAppendDbmses = (DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE, DBMS.H2, DBMS.CLICKHOUSE, DBMS.CRATEDB, DBMS.SPANNER, DBMS.HANA)
+
         if (limitRegExp or limitRegExp2) or (Backend.getIdentifiedDbms() in (DBMS.MSSQL, DBMS.SYBASE) and topLimit):
-            if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE, DBMS.H2):
+            if Backend.getIdentifiedDbms() in limitOffsetAppendDbmses:
                 limitGroupStart = queries[Backend.getIdentifiedDbms()].limitgroupstart.query
                 limitGroupStop = queries[Backend.getIdentifiedDbms()].limitgroupstop.query
 
@@ -1084,7 +1090,7 @@ def limitCondition(self, expression, dump=False):
 
                 # From now on we need only the expression until the " LIMIT "
                 # (or equivalent, depending on the back-end DBMS) word
-                if Backend.getIdentifiedDbms() in (DBMS.MYSQL, DBMS.PGSQL, DBMS.SQLITE):
+                if Backend.getIdentifiedDbms() in limitOffsetAppendDbmses:
                     stopLimit += startLimit
                     if expression.find(queries[Backend.getIdentifiedDbms()].limitstring.query) > 0:
                         _ = expression.index(queries[Backend.getIdentifiedDbms()].limitstring.query)

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.210"
+VERSION = "1.10.7.228"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -909,7 +909,7 @@
 HASHDB_END_TRANSACTION_RETRIES = 3
 
 # Unique milestone value used for forced deprecation of old HashDB values (e.g. when changing the hash/serialization mechanism)
-HASHDB_MILESTONE_VALUE = "MvKpZrBqTn"  # python -c 'import random, string; print "".join(random.sample(string.ascii_letters, 10))'
+HASHDB_MILESTONE_VALUE = "CvHUbaSNZL"  # python -c 'import random, string; print "".join(random.sample(string.ascii_letters, 10))'
 
 # Warn user of possible delay due to large page dump in full UNION query injections
 LARGE_OUTPUT_THRESHOLD = 1024 ** 2

lib/core/target.py

@@ -266,7 +266,7 @@ def _attr(m):
                 if not (kb.processUserMarks and kb.customInjectionMark in conf.data):
                     conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE, conf.data)
                     conf.data = conf.data.replace(kb.customInjectionMark, ASTERISK_MARKER)
-                    conf.data = re.sub(r"(?si)(Content-Disposition:[^\n]+\s+name=\"(?P<name>[^\"]+)\"(?:[^f|^b]|f(?!ilename=)|b(?!oundary=))*?)((%s)--)" % ("\r\n" if "\r\n" in conf.data else '\n'),
+                    conf.data = re.sub(r"(?si)(Content-Disposition:[^\n]+\s+name=\"(?P<name>[^\"]+)\"(?:[^fb]|f(?!ilename=)|b(?!oundary=))*?)((%s)--)" % ("\r\n" if "\r\n" in conf.data else '\n'),
                                        functools.partial(process, repl=r"\g<1>%s\g<3>" % kb.customInjectionMark), conf.data)
 
         # a gRPC-Web body that was NOT accepted as GRPC_WEB (declined prompt) must be sent verbatim,

lib/core/threads.py

@@ -51,6 +51,7 @@ def reset(self):
         self.lastComparisonHeaders = None
         self.lastComparisonCode = None
         self.lastComparisonRatio = None
+        self.lastComparisonPageLength = None
         self.lastPageTemplateCleaned = None
         self.lastPageTemplateJsonMinimized = None
         self.lastPageTemplateStructural = None

lib/parse/openapi.py

@@ -193,7 +193,11 @@ def _baseUrl(spec, origin=None, servers=None):
         variables = servers[0].get("variables")
         if isinstance(variables, dict):
             for name, meta in variables.items():
-                default = meta.get("default", "1") if isinstance(meta, dict) else "1"
+                meta = meta if isinstance(meta, dict) else {}
+                default = meta.get("default")
+                if default is None:               # 'default' is spec-required; when omitted, a declared enum value beats a placeholder host ('1')
+                    enum = meta.get("enum")
+                    default = enum[0] if isinstance(enum, list) and enum else "1"
                 url = url.replace("{%s}" % name, str(default))
         if re.match(r"(?i)[a-z][a-z0-9+.-]*://", url):    # absolute server URL -> used as declared (the host is NOT rewritten to the spec's own origin)
             return url.rstrip('/')

lib/request/comparison.py

@@ -85,6 +85,7 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
         threadData.lastComparisonHeaders = listToStrValue(_ for _ in headers.headers if not _.startswith("%s:" % URI_HTTP_HEADER)) if headers else ""
         threadData.lastComparisonPage = page
         threadData.lastComparisonCode = code
+        threadData.lastComparisonPageLength = pageLength  # NULL connection carries the length here (no body)
 
     if page is None and pageLength is None:
         return None
@@ -114,9 +115,10 @@ def _comparison(page, headers, code, getRatioValue, pageLength):
     if conf.code:
         return conf.code == code
 
-    # Response content length to match when the query is True
+    # Response content length to match when the query is True (NULL connection supplies the length via
+    # pageLength with no body, so fall back to it rather than len(None)=0 which would stall the oracle)
     if conf.lengths:
-        return len(page or "") == kb.trueLength
+        return (pageLength if page is None else len(page)) == kb.trueLength
 
     seqMatcher = threadData.seqMatcher
     seqMatcher.set_seq1(kb.pageTemplate)

lib/request/connect.py

@@ -644,7 +644,7 @@ class _(dict):
                         conf.data = getattr(conf.data, UNENCODED_ORIGINAL_VALUE, conf.data)
                     else:
                         post = urldecode(post, convall=True)
-                        post = encodeBase64(post)
+                        post = encodeBase64(post, safe=conf.base64Safe)
 
                 if target and cmdLineOptions.method or method and method not in (HTTPMETHOD.GET, HTTPMETHOD.POST):
                     req = MethodRequest(url, post, headers)

lib/request/inject.py

@@ -62,6 +62,7 @@
 from lib.core.settings import GET_VALUE_UPPERCASE_KEYWORDS
 from lib.core.settings import IS_TTY
 from lib.core.settings import INFERENCE_MARKER
+from lib.core.settings import INVALID_UNICODE_PRIVATE_AREA
 from lib.core.settings import MAX_TECHNIQUES_PER_VALUE
 from lib.core.settings import SQL_SCALAR_REGEX
 from lib.core.settings import UNICODE_ENCODING
@@ -575,6 +576,24 @@ def _etaTicker():
 
     return results
 
+def _pageCharsetCorrupted(value):
+    """
+    True if a retrieved value carries reversibly-decoded (undecodable) bytes - a sign that the
+    web page charset could not represent the DBMS data (cf. the 'reversible' codec). Such a
+    UNION/error value is silently corrupt and should be re-fetched via DBMS-side hexadecimal.
+    """
+
+    retVal = [False]
+
+    def _(item):
+        if not retVal[0] and isinstance(item, six.string_types):
+            if re.search(r"\\x[89a-f][0-9a-f]", item) or (INVALID_UNICODE_PRIVATE_AREA and any(0xF0000 <= ord(_) <= 0xF00FF for _ in item)):
+                retVal[0] = True
+        return item
+
+    applyFunctionRecursively(value, _)
+    return retVal[0]
+
 @lockedmethod
 @stackedmethod
 def getValue(expression, blind=True, union=True, error=True, time=True, fromUser=False, expected=None, batch=False, unpack=True, resumeValue=True, charsetType=None, firstChar=None, lastChar=None, dump=False, suppressOutput=None, expectingNone=False, safeCharEncode=True):
@@ -672,6 +691,28 @@ def getValue(expression, blind=True, union=True, error=True, time=True, fromUser
                     count += 1
                     found = (value is not None) or (value is None and expectingNone) or count >= MAX_TECHNIQUES_PER_VALUE
 
+                # Auto-recover from a page/DBMS charset mismatch: a UNION/error value carrying
+                # undecodable bytes (the page charset couldn't represent the DBMS data) is silently
+                # corrupt. Re-fetch it via DBMS-side hex, which travels as ASCII regardless of the
+                # page charset - no user '--hex'/'--encoding' knowledge required. Gated, so clean
+                # or ASCII data pays nothing.
+                if (found and not conf.hexConvert and not conf.binaryFields and expected not in (EXPECTED.BOOL, EXPECTED.INT)
+                        and getTechnique() in (PAYLOAD.TECHNIQUE.UNION, PAYLOAD.TECHNIQUE.ERROR, PAYLOAD.TECHNIQUE.QUERY)
+                        and Backend.getIdentifiedDbms() and hasattr(queries[Backend.getIdentifiedDbms()], "hex")
+                        and _pageCharsetCorrupted(value)):
+                    warnMsg = "retrieved data appears corrupted because of a charset mismatch between the "
+                    warnMsg += "DBMS and the web page. Re-fetching using hexadecimal encoding"
+                    singleTimeWarnMessage(warnMsg)
+
+                    conf.hexConvert = True
+                    try:
+                        _value = _goUnion(query, unpack, dump) if getTechnique() == PAYLOAD.TECHNIQUE.UNION else errorUse(query, dump)
+                    finally:
+                        conf.hexConvert = False
+
+                    if _value is not None:
+                        value = _value
+
                 if found and conf.dnsDomain:
                     _ = "".join(filterNone(key if isTechniqueAvailable(value) else None for key, value in {'E': PAYLOAD.TECHNIQUE.ERROR, 'Q': PAYLOAD.TECHNIQUE.QUERY, 'U': PAYLOAD.TECHNIQUE.UNION}.items()))
                     warnMsg = "option '--dns-domain' will be ignored "

lib/techniques/blind/inference.py

@@ -268,6 +268,7 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
     finalValue = None
     retrievedLength = 0
     columnKey = None
+    hexEncoded = False
 
     if payload is None:
         return 0, None
@@ -369,14 +370,18 @@ def bisection(payload, expression, length=None, charsetType=None, firstChar=None
         # rows for low-cardinality guessing and for its own per-column online Huffman model.
         columnKey = normalizedExpression(expression) if dump else None
 
+        # Whole-value equality shortcuts (low-card guess, litmus) compare a hex-DECODED value against a
+        # HEX()-wrapped expression -> always miss; skip them when hex-encoding (--hex / --binary-fields).
+        hexEncoded = bool(conf.hexConvert or kb.binaryField)
+
         # Low-cardinality whole-value guessing: when the distinct values already seen for this column are
         # few (<= LOW_CARDINALITY_THRESHOLD), confirm the current cell by equality against each of them
         # (one request on a hit) before per-character extraction - a large win on the enum/flag/status/
         # category/type columns that dominate real tables. Self-verifying (a wrong candidate simply fails).
         # Especially valuable for TIME-BASED blind: a hit confirms the whole value in a single delayed
         # request instead of ~7 delays/char x N chars. The repetition gate below ensures it only ever fires
         # on genuinely low-cardinality columns, so unique identifier names never pay a wasted probe/delay.
-        if columnKey is not None and not partialValue:
+        if columnKey is not None and not partialValue and not hexEncoded:
             # Snapshot the shared cache under the lock (value-parallel workers may mutate it concurrently).
             with kb.locks.prediction:
                 seen = dict(kb.lowCardCache.get(columnKey) or ())
@@ -1188,7 +1193,7 @@ def blindThread():
     # known-answer differential so an always-true / flaky / degraded channel that would otherwise dump
     # SILENT garbage instead raises a one-time "results may be unreliable" warning. First value is always
     # checked (catch it before a whole bad dump), then every ORACLE_LITMUS_CHECK_EVERY-th.
-    if (ORACLE_LITMUS_CHECK_EVERY and finalValue and not kb.reliabilityAlarm and not kb.bruteMode
+    if (ORACLE_LITMUS_CHECK_EVERY and finalValue and not kb.reliabilityAlarm and not kb.bruteMode and not hexEncoded
             and (columnKey is not None or kb.partRun in NAME_PREDICTION_CONTEXTS)):
         with kb.locks.prediction:
             kb.litmusCounter += 1

lib/techniques/error/use.py

@@ -87,7 +87,9 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                 testQuery = "RPAD('%s',%d,'%s')" % (testChar, current, testChar)
             else:
                 testQuery = "%s('%s',%d)" % ("REPEAT" if Backend.isDbms(DBMS.MYSQL) else "REPLICATE", testChar, current)
-                testQuery = "SELECT %s" % (agent.hexConvertField(testQuery) if conf.hexConvert else testQuery)
+                # chunk length is the channel's CHAR capacity (hex-independent); a hex-wrapped probe doubles
+                # the length and mis-detects (pins to the minimum), so probe plain even under --hex
+                testQuery = "SELECT %s" % testQuery
 
             result = unArrayizeValue(_oneShotErrorUse(testQuery, chunkTest=True))
             seen.add(current)
@@ -206,7 +208,7 @@ def _oneShotErrorUse(expression, field=None, chunkTest=False):
                 hashDBWrite(expression, "%s%s" % (retVal, PARTIAL_VALUE_MARKER))
             raise
 
-        retVal = decodeDbmsHexValue(retVal) if conf.hexConvert else retVal
+        retVal = decodeDbmsHexValue(retVal) if conf.hexConvert and not chunkTest else retVal
 
         if isinstance(retVal, six.string_types):
             retVal = htmlUnescape(retVal).replace("<br>", "\n")

lib/techniques/union/use.py

@@ -125,7 +125,8 @@ def _oneShotUnionUse(expression, unpack=True, limited=False):
                                 if fields:
                                     parts = []
                                     for row in json_data:
-                                        parts.append("%s%s%s" % (kb.chars.start, kb.chars.delimiter.join(getUnicode(row.get(field) or NULL) for field in fields), kb.chars.stop))
+                                        # is-None (not falsy) check: a real 0 / '' / False are values, only JSON null is NULL
+                                        parts.append("%s%s%s" % (kb.chars.start, kb.chars.delimiter.join(NULL if row.get(field) is None else getUnicode(row.get(field)) for field in fields), kb.chars.stop))
                                     retVal = "".join(parts)
                         except:
                             retVal = None

lib/utils/brute.py

@@ -293,7 +293,9 @@ def columnExistsThread():
             warnMsg = "no column(s) found"
             logger.warning(warnMsg)
         else:
-            columns = {}
+            # Note: a separate name from the 'columns' wordlist (a list reused across the
+            # conf.tbl loop); reusing it here would rebind it to a dict and break later tables
+            columnData = {}
 
             for column in threadData.shared.files:
                 if Backend.getIdentifiedDbms() in (DBMS.MYSQL,):
@@ -306,15 +308,15 @@ def columnExistsThread():
                     result = inject.checkBooleanExpression("%s" % safeStringFormat("EXISTS(SELECT %s FROM %s WHERE ROUND(%s)=ROUND(%s))", (column, table, column, column)))
 
                 if result:
-                    columns[column] = "numeric"
+                    columnData[column] = "numeric"
                 else:
-                    columns[column] = "non-numeric"
+                    columnData[column] = "non-numeric"
 
             if conf.db not in kb.data.cachedColumns:
                 kb.data.cachedColumns[conf.db] = {}
-            kb.data.cachedColumns[conf.db][table] = columns
+            kb.data.cachedColumns[conf.db][table] = columnData
 
-            for _ in ((conf.db, table, item[0], item[1]) for item in columns.items()):
+            for _ in ((conf.db, table, item[0], item[1]) for item in columnData.items()):
                 if _ not in kb.brute.columns:
                     kb.brute.columns.append(_)
 

lib/utils/hash.py

@@ -183,7 +183,7 @@ def mssql_new_passwd(password, salt, uppercase=False):  # since version '2012'
     """
 
     binsalt = decodeHex(salt)
-    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+    unistr = getUnicode(password).encode("utf-16-le")   # MSSQL hashes the password as UCS-2/UTF-16LE
 
     retVal = "0200%s%s" % (salt, sha512(unistr + binsalt).hexdigest())
 
@@ -200,7 +200,7 @@ def mssql_passwd(password, salt, uppercase=False):  # versions '2005' and '2008'
     """
 
     binsalt = decodeHex(salt)
-    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+    unistr = getUnicode(password).encode("utf-16-le")   # MSSQL hashes the password as UCS-2/UTF-16LE
 
     retVal = "0100%s%s" % (salt, sha1(unistr + binsalt).hexdigest())
 
@@ -218,7 +218,7 @@ def mssql_old_passwd(password, salt, uppercase=True):  # version '2000' and befo
     """
 
     binsalt = decodeHex(salt)
-    unistr = b"".join((_.encode(UNICODE_ENCODING) + b"\0") if ord(_) < 256 else _.encode(UNICODE_ENCODING) for _ in password)
+    unistr = getUnicode(password).encode("utf-16-le")   # MSSQL hashes the password as UCS-2/UTF-16LE
 
     retVal = "0100%s%s%s" % (salt, sha1(unistr + binsalt).hexdigest(), sha1(unistr.upper() + binsalt).hexdigest())
 

lib/utils/keysetdump.py

@@ -246,8 +246,15 @@ def _dumpComposite(tbl, colList, count, cursorCols, tableRef, entries, lengths):
         if prev is None:
             condition = "1=1"
         else:
-            # ANSI row-value (tuple) comparison advances the composite cursor lexicographically
-            condition = "(%s)>(%s)" % (orderExpr, ','.join(_lit(_) for _ in prev))
+            # Portable lexicographic seek predicate. ANSI row-value comparison ((a,b)>(x,y)) is not
+            # supported on MSSQL/Oracle - there it errored, stopping the walk after the first row -
+            # so expand it to (a>x) OR (a=x AND b>y) OR ... which uses only scalar comparisons.
+            ors = []
+            for i in xrange(len(fields)):
+                terms = ["%s=%s" % (fields[j], _lit(prev[j])) for j in xrange(i)]
+                terms.append("%s>%s" % (fields[i], _lit(prev[i])))
+                ors.append("(%s)" % " AND ".join(terms))
+            condition = "(%s)" % " OR ".join(ors)
 
         tup = []
         for field in fields:

plugins/generic/search.py

@@ -535,7 +535,10 @@ def searchColumn(self):
                 origDb = conf.db
                 origTbl = conf.tbl
 
-                for column, dbData in foundCols.items():
+                # Note: only the current column - foundCols also holds columns from earlier
+                # colList iterations, and re-walking them here re-issued their table lookups
+                # (O(n^2) blind requests) and duplicated their found tables
+                for column, dbData in ((column, foundCols[column]),):
                     colQuery = "%s%s" % (colCond, colCondParam)
                     colQuery = colQuery % unsafeSQLIdentificatorNaming(column)