diff --git a/docs/design/qwen4-exp-text-core.md b/docs/design/qwen4-exp-text-core.md index b21445469..b1ceb9a96 100644 --- a/docs/design/qwen4-exp-text-core.md +++ b/docs/design/qwen4-exp-text-core.md @@ -102,16 +102,23 @@ complete 1,224-name tensor shape/qtype contract. GGUF's split indexer query/key matrices are concatenated row-wise into Hugging Face's fused `index_qk_proj`; they are not Q/K-permuted. -Payload conversion deliberately fails before Hub download. The combined PLE +Payload conversion deliberately fails before tensor materialization. A +successful bounded-header preflight rejects the architecture after fetching at +most the configured initial byte range, but before downloading any complete GGUF +file or shard payload. That bounded response can include initial tensor bytes +when the metadata header is smaller than the requested range. If the range +request fails or the metadata header exceeds its bounded range, the intentional +best-effort fallback downloads the immutable file or complete shard set for full +local validation before reaching the same fail-closed guard. The combined PLE table is an enormous IQ4_NL embedding for which the graph has no compatible -native gather ABI. Routed experts are rank-3 banks with IQ1_S gate/up and -IQ4_NL down tensors, while the released runtime has neither a mixed-format -sparse native-block MoE ABI nor real-weight execution evidence. Treating these -as ordinary affine `MatMulNBits` would be incorrect. Explicit float -dequantization is also rejected because the PLE table alone expands beyond the -bounded single-tensor materialization policy. The exact header/config/mapping -support is therefore a fail-closed foundation for future runtime ABI work, not -a quantized execution claim. +native gather ABI. Routed experts are rank-3 banks with IQ1_S gate/up and IQ4_NL +down tensors, while the released runtime has neither a mixed-format sparse +native-block MoE ABI nor real-weight execution evidence. Treating these as +ordinary affine `MatMulNBits` would be incorrect. Explicit float dequantization +is also rejected because the PLE table alone expands beyond the bounded +single-tensor materialization policy. The exact header/config/mapping support is +therefore a fail-closed foundation for future runtime ABI work, not a quantized +execution claim. Released onnxruntime-genai and the current ONNX GenAI workflow schema cannot represent Qwen4-Exp's `ple_input_ids`, four-axis position state, and diff --git a/src/mobius/integrations/gguf/_builder.py b/src/mobius/integrations/gguf/_builder.py index ab099328d..cf29a8cee 100644 --- a/src/mobius/integrations/gguf/_builder.py +++ b/src/mobius/integrations/gguf/_builder.py @@ -88,17 +88,22 @@ class _GGUFPreflightRevision: revision: str header_info: GGUFHeaderInfo + complete_payload_downloaded: bool | None = None def __str__(self) -> str: return self.revision def __eq__(self, other: object) -> bool: if isinstance(other, _GGUFPreflightRevision): - return self.revision == other.revision and self.header_info == other.header_info + return ( + self.revision == other.revision + and self.header_info == other.header_info + and self.complete_payload_downloaded == other.complete_payload_downloaded + ) return isinstance(other, str) and self.revision == other def __hash__(self) -> int: - return hash((self.revision, self.header_info)) + return hash((self.revision, self.header_info, self.complete_payload_downloaded)) @dataclass(frozen=True, slots=True, eq=False) @@ -585,9 +590,15 @@ def read_response(response) -> list[bytes]: error, ) return _GGUFPreflightFallbackRevision(commit_hash) + data = b"".join(chunks) + remote_size = getattr(metadata, "size", None) + if isinstance(remote_size, int) and remote_size >= 0: + complete_payload_downloaded: bool | None = len(data) >= remote_size + else: + complete_payload_downloaded = None try: header_info = _gguf_header_info_from_header_prefix( - b"".join(chunks), + data, source=source, ) except GGUFHeaderTruncatedError: @@ -629,8 +640,14 @@ def read_response(response) -> list[bytes]: if architecture == "qwen4exp" and dispatch_architecture: from mobius.integrations.gguf._qwen4_exp import reject_qwen4exp_payload - reject_qwen4exp_payload() - return _GGUFPreflightRevision(commit_hash, header_info) + reject_qwen4exp_payload( + complete_payload_downloaded=complete_payload_downloaded, + ) + return _GGUFPreflightRevision( + commit_hash, + header_info, + complete_payload_downloaded, + ) def _preflight_hf_mmproj_companion_file( @@ -6348,7 +6365,15 @@ def _select_hf_gguf_set_from_split_headers( if primary_architecture == "qwen4exp": from mobius.integrations.gguf._qwen4_exp import reject_qwen4exp_payload - reject_qwen4exp_payload() + download_states = { + preflight.complete_payload_downloaded for preflight in preflights.values() + } + complete_payload_downloaded = ( + True if True in download_states else False if download_states == {False} else None + ) + reject_qwen4exp_payload( + complete_payload_downloaded=complete_payload_downloaded, + ) mismatched_architectures = { name: architecture for name, architecture in declared_architectures.items() diff --git a/src/mobius/integrations/gguf/_qwen4_exp.py b/src/mobius/integrations/gguf/_qwen4_exp.py index f2c54733c..e1bba6641 100644 --- a/src/mobius/integrations/gguf/_qwen4_exp.py +++ b/src/mobius/integrations/gguf/_qwen4_exp.py @@ -25,7 +25,11 @@ class Qwen4ExpGGUFImportError(NotImplementedError): """A Qwen4Exp payload has no truthful executable import route.""" -def _payload_blocker(*, keep_quantized: bool | None) -> Qwen4ExpGGUFImportError: +def _payload_blocker( + *, + keep_quantized: bool | None, + complete_payload_downloaded: bool | None = None, +) -> Qwen4ExpGGUFImportError: if keep_quantized is True: detail = ( "per_layer_token_embd.weight is an IQ4_NL embedding with no matching " @@ -49,17 +53,39 @@ def _payload_blocker(*, keep_quantized: bool | None) -> Qwen4ExpGGUFImportError: "expert-bank ABIs, while dense materialization exceeds the bounded-memory " "import route." ) + if complete_payload_downloaded is False: + download_detail = ( + "Only bounded GGUF preflight range data was fetched; no complete GGUF " + "file or shard payload was downloaded." + ) + elif complete_payload_downloaded is True: + download_detail = ( + "The bounded preflight response contained a complete GGUF file or shard payload." + ) + else: + download_detail = ( + "This rejection does not imply that no Hub payload was downloaded: " + "bounded-header preflight may fall back to downloading the immutable file " + "or shard set for full local validation." + ) return Qwen4ExpGGUFImportError( "Qwen3.8 Flash-Next GGUF payload import is intentionally fail-closed. " - f"{detail} No GGUF tensor payload was downloaded or materialized. The exact header, " - "configuration, shard closure, and tensor-name mapping remain supported " - "for preflight and future ABI work." + f"{detail} No GGUF tensor payload was materialized by the importer. " + f"{download_detail} The exact header, configuration, shard closure, and " + "tensor-name mapping remain supported for preflight and future ABI work." ) -def reject_qwen4exp_payload(*, keep_quantized: bool | None = None) -> None: - """Reject a header-identified Qwen4Exp payload before materializing tensors.""" - raise _payload_blocker(keep_quantized=keep_quantized) +def reject_qwen4exp_payload( + *, + keep_quantized: bool | None = None, + complete_payload_downloaded: bool | None = None, +) -> None: + """Reject a header-identified Qwen4Exp payload without materializing tensors.""" + raise _payload_blocker( + keep_quantized=keep_quantized, + complete_payload_downloaded=complete_payload_downloaded, + ) def _qtype_name(qtype: Any) -> str: diff --git a/src/mobius/integrations/gguf/_qwen4_exp_test.py b/src/mobius/integrations/gguf/_qwen4_exp_test.py index 9cc8f161f..70f44be69 100644 --- a/src/mobius/integrations/gguf/_qwen4_exp_test.py +++ b/src/mobius/integrations/gguf/_qwen4_exp_test.py @@ -244,8 +244,8 @@ def test_qwen4exp_header_fixture_matches_pinned_evidence(): @pytest.mark.parametrize( ("keep_quantized", "message"), [ - (True, r"IQ4_NL embedding.*rank-3 routed experts.*No GGUF tensor payload"), - (False, r"191 GiB.*bounded-memory route.*No GGUF tensor payload"), + (True, r"IQ4_NL embedding.*rank-3 routed experts.*may fall back to downloading"), + (False, r"191 GiB.*bounded-memory route.*may fall back to downloading"), ], ) def test_qwen4exp_payload_modes_fail_closed_before_raw_payload_access( @@ -278,35 +278,135 @@ def test_qwen4exp_hub_preflight_is_source_independent_and_forwards_revision(monk lambda _url: SimpleNamespace( commit_hash="a" * 40, location="https://cdn.example/model.gguf", + size=len(bounded_response) + 1, ), ) response = mock.MagicMock() - response.iter_bytes.return_value = [b"header"] + bounded_response = b"complete-metadata-header|initial-tensor-bytes" + response.iter_bytes.return_value = [bounded_response] response_context = mock.MagicMock() response_context.__enter__.return_value = response session = mock.MagicMock() session.stream.return_value = response_context monkeypatch.setattr(_builder, "get_session", lambda: session) - monkeypatch.setattr( - _builder, - "_gguf_header_info_from_header_prefix", - lambda *_args, **_kwargs: _builder.GGUFHeaderInfo( + inspected_ranges = [] + + def inspect_range(data, **_kwargs): + inspected_ranges.append(data) + return _builder.GGUFHeaderInfo( architecture="qwen4exp", tensor_count=0, split_no=0, split_count=3, split_tensors_count=1224, - ), + ) + + monkeypatch.setattr( + _builder, + "_gguf_header_info_from_header_prefix", + inspect_range, ) - with pytest.raises(Qwen4ExpGGUFImportError, match="intentionally fail-closed"): + with pytest.raises( + Qwen4ExpGGUFImportError, + match=r"intentionally fail-closed.*Only bounded GGUF preflight range data", + ): _builder._preflight_hf_gguf_file( "other/Qwen4Exp-GGUF", "renamed-00001-of-00003.gguf", revision="feature/revision", ) + assert inspected_ranges == [bounded_response] + assert b"initial-tensor-bytes" in inspected_ranges[0] hub_url.assert_called_once_with( "other/Qwen4Exp-GGUF", "renamed-00001-of-00003.gguf", revision="feature/revision", ) + + +def test_qwen4exp_preflight_reports_complete_file_that_fits_in_range(monkeypatch): + from mobius.integrations.gguf import _builder + + complete_file = b"small-complete-gguf" + monkeypatch.setattr(_builder, "hf_hub_url", lambda *_args, **_kwargs: "hub-url") + monkeypatch.setattr( + _builder, + "get_hf_file_metadata", + lambda _url: SimpleNamespace( + commit_hash="c" * 40, + location="https://cdn.example/small.gguf", + size=len(complete_file), + ), + ) + response = mock.MagicMock() + response.iter_bytes.return_value = [complete_file] + response_context = mock.MagicMock() + response_context.__enter__.return_value = response + session = mock.MagicMock() + session.stream.return_value = response_context + monkeypatch.setattr(_builder, "get_session", lambda: session) + monkeypatch.setattr( + _builder, + "_gguf_header_info_from_header_prefix", + lambda *_args, **_kwargs: _builder.GGUFHeaderInfo( + architecture="qwen4exp", + tensor_count=1, + split_no=None, + split_count=None, + split_tensors_count=None, + ), + ) + + with pytest.raises( + Qwen4ExpGGUFImportError, + match="bounded preflight response contained a complete GGUF file", + ): + _builder._preflight_hf_gguf_file( + "other/Qwen4Exp-GGUF", + "small.gguf", + ) + + +def test_qwen4exp_split_fallback_does_not_claim_complete_payload_was_not_downloaded( + monkeypatch, +): + from mobius.integrations.gguf import _builder + + commit_hash = "b" * 40 + shards = [ + "model-00001-of-00002.gguf", + "model-00002-of-00002.gguf", + ] + download_shards = mock.Mock(return_value="cached-primary.gguf") + monkeypatch.setattr( + _builder, + "_preflight_hf_gguf_file", + lambda *_args, **_kwargs: _builder._GGUFPreflightFallbackRevision(commit_hash), + ) + api = mock.Mock() + api.list_repo_files.return_value = shards + monkeypatch.setattr(_builder, "HfApi", mock.Mock(return_value=api)) + monkeypatch.setattr(_builder, "_download_hf_gguf_shards", download_shards) + + assert ( + _builder._resolve_gguf_path(f"other/Qwen4Exp-GGUF:{shards[0]}") + == "cached-primary.gguf" + ) + download_shards.assert_called_once_with( + api, + repo_id="other/Qwen4Exp-GGUF", + selected_filename=shards[0], + shard_filenames=shards, + revision=commit_hash, + ) + + with pytest.raises(Qwen4ExpGGUFImportError) as exc_info: + _builder._validate_gguf_model( + _HeaderFixture(), + source="cached-primary.gguf", + keep_quantized=True, + ) + message = str(exc_info.value) + assert "may fall back to downloading" in message + assert "no complete GGUF file or shard payload was downloaded" not in message