diff --git a/README-ZH.md b/README-ZH.md index 293d62a9f..511f736ad 100644 --- a/README-ZH.md +++ b/README-ZH.md @@ -13,6 +13,8 @@ mllm [📚 文档](https://ubiquitouslearning.github.io/mllm/) • [🚀 快速开始](#quick-starts) • [💡 示例](./examples/) • [🛠️ 安装](#installation) +[English](./README.md) + ## 最新动态 @@ -108,6 +110,7 @@ mllm 框架可以与主流社区框架的模型检查点无缝集成。通过 ml | [Qwen3-4B](https://github.com/QwenLM/Qwen3) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/Qwen3-4B-w4a8-i8mm-kai) | | | | [MiniCPM5-1B](https://huggingface.co/openbmb/MiniCPM5-1B) | [✔️ w4a8](./examples/minicpm5/README.md) | | | | [MiniCPM5-2B](https://huggingface.co/openbmb/MiniCPM5-2B) | [✔️ w4a8](./examples/minicpm5/README.md) | | | +| [Spark-X2.5-1.7B](https://huggingface.co/XHToken/Spark-X2.5-1.7B) | [✔️ FP32 / KAI W4A32](./examples/spark2_5/README.md) | | | | Qwen3.5-2B | | ✔️ W4A16 / W8A8 | | | Qwen3.5-4B | | ✔️ W4A16 / W8A8 | | | Qwen3-VL-2B-Instruct | | ✔️ W4A16 / W8A8 | | diff --git a/README.md b/README.md index 34893a80f..87538932a 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ mllm [📚 Documentation](https://ubiquitouslearning.github.io/mllm/) • [🚀 Quick Start](#quick-starts) • [💡 Examples](./examples/) • [🛠️ Installation](#installation) +[简体中文](./README-ZH.md) + ## Latest News @@ -109,6 +111,7 @@ The mllm framework integrates seamlessly with popular community frameworks' chec | [Qwen3.5-4B](https://huggingface.co/Qwen/Qwen3.5-4B) | [✔️ w4a8](./examples/qwen3_5/README.md) | | | | [MiniCPM5-1B](https://huggingface.co/openbmb/MiniCPM5-1B) | [✔️ w4a8](./examples/minicpm5/README.md) | | | | [MiniCPM5-2B](https://huggingface.co/openbmb/MiniCPM5-2B) | [✔️ w4a8](./examples/minicpm5/README.md) | | | +| [Spark-X2.5-1.7B](https://huggingface.co/XHToken/Spark-X2.5-1.7B) | [✔️ FP32 / KAI W4A32](./examples/spark2_5/README.md) | | | | [LFM2.5-2.6B](https://huggingface.co/LiquidAI/LFM2.5-2.6B) | [✔️ w4a8](./examples/lfm2/README.md) | | | | [DeepSeek-OCR](https://github.com/deepseek-ai/DeepSeek-OCR) | [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/DeepSeek-OCR-w4a8-i8mm-kai) | | | | [SmolLM3](https://huggingface.co/blog/smollm3)| [✔️ w4a8](https://www.modelscope.cn/models/mllmTeam/SmolLM3-3B-w4a8-i8mm-kai) | | | diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index ed1cbeb82..fbfcbca24 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -30,3 +30,5 @@ if(MLLM_QUALCOMM_QNN_AOT_ON_X86_ENABLE OR MLLM_BUILD_QNN_BACKEND) add_subdirectory(qwen2_qnn_aot) add_subdirectory(llama_qnn_aot) endif() + +add_subdirectory(spark2_5) diff --git a/examples/spark2_5/CMakeLists.txt b/examples/spark2_5/CMakeLists.txt new file mode 100644 index 000000000..4896e2020 --- /dev/null +++ b/examples/spark2_5/CMakeLists.txt @@ -0,0 +1,3 @@ +add_executable(mllm-spark25-runner main.cpp) +target_link_libraries(mllm-spark25-runner PRIVATE MllmRT MllmCPUBackend) +target_include_directories(mllm-spark25-runner PRIVATE ${MLLM_INCLUDE_DIR}) diff --git a/examples/spark2_5/README.md b/examples/spark2_5/README.md new file mode 100644 index 000000000..bed01869f --- /dev/null +++ b/examples/spark2_5/README.md @@ -0,0 +1,79 @@ +# Spark-X2.5-1.7B on CPU + +This example implements `XHToken/Spark-X2.5-1.7B` at revision +`448e61eb392c00f2c403185c5b56d5e0665bfaab`: 28 dense decoder layers, +8 query / 2 KV heads of dimension 256, headwise sigmoid attention gates, +GELU-gated MLPs, and shared source embedding/output weights. +Every fourth layer uses full attention with 64-dimensional partial RoPE; +the other layers use a 512-token causal window and full-dimensional RoPE. + +The runtime supports batch-1 CPU text generation, FP32 and ARM KAI W4A32. +The supplied configs cap context at 4096 tokens; `max_cache_length` can be +increased to 8192 for larger requests. The checkpoint's 1M positional range is +not a tested mobile capacity claim. Full-attention KV grows with context; +sliding layers retain only the 511 past tokens needed by their next forward. +Chunked prefill uses absolute positions and per-query window boundaries. + +## Convert + +Download the pinned checkpoint, including tokenizer.json and the index/shards. +Use separate paths for checkpoint, converted files, and build output. +The converter validates every tensor name, shape and source dtype, and refuses +to overwrite existing output or partial files. + +```bash +python examples/spark2_5/convert.py \ + --checkpoint /path/to/Spark-X2.5-1.7B \ + --output /path/to/spark-fp32.mllm + +# Requires the repository's built pymllm mobile binding; packing is portable +# and can run on an x86 host. Use the candidate's matching runtime libraries. +python examples/spark2_5/convert.py \ + --checkpoint /path/to/Spark-X2.5-1.7B \ + --output /path/to/spark-w4a32.mllm --quantized +``` + +KAI quantizes QKV, attention output, FFN and output-head matrices. Norms, +lookup embedding and the small 8-output gate remain FP32. The output head is +derived from `model.embedding.weight`; lookup and projection have different +storage formats, so tied checkpoint weights do not imply one runtime buffer. +W4A32 uses FP32 interfaces and dynamically quantizes activations to INT8 inside +KleidiAI. Its artifact must be paired with `config_1.7B_w4a32_kai.json`. + +## Build and run + +```bash +cmake -S . -B build -DMLLM_ENABLE_EXAMPLE=ON +cmake --build build --target mllm-spark25-runner +OMP_NUM_THREADS=1 build/bin/mllm-spark25-runner \ + --model_path /path/to/spark-fp32.mllm \ + --config_path examples/spark2_5/config_1.7B_fp32.json \ + --tokenizer_path /path/to/Spark-X2.5-1.7B/tokenizer.json \ + --prompt "Explain how a sliding-window attention cache works." \ + --enable_thinking false --max_new_tokens 128 --print_token_ids +``` + +For Android, use the repository's Android NDK build settings and build the +same runner target with `MLLM_BUILD_ARM_BACKEND=ON`. Select an ARM ISA supported +by the target phone, including the instructions required by the chosen KAI +implementation. + +The runner defaults to one CPU operation thread. For OpenMP builds, also set +`OMP_NUM_THREADS=1`, as shown above. Override operation threading with +`--engine_cpu_op_thread N`; multi-thread performance is not qualified by this +example. The CLI renders the official single-user-turn template, with an optional +system message. Thinking defaults to enabled; disabling it appends `` +in the generation prompt. Decoding is greedy and stops at EOS id 1 or the +requested generation limit. UTF-8 bytes are buffered across token boundaries. +The runner does not provide tool execution or a conversation-history API. + +## Tests and numerical contract + +`GELUFocused`, `SigmoidFocused`, `GroupedQueryAttentionFocused`, and `Spark25Focused` cover the +public operations, window/chunk semantics, synthetic model/reset, configuration +and tokenizer. Official tokenizer parity fixtures may be supplied through +`MLLM_SPARK_TOKENIZER_FIXTURES` (tokenizer.json and tokenizer_cases.json). +Spark requests erf GELU and accurate logistic Sigmoid; existing callers retain +their default approximations. FP32 runtime calculations are compared with an FP32 official-model oracle; +this is distinct from bitwise reproduction of the checkpoint's BF16 arithmetic. +Quantized quality and device execution require their own validation. diff --git a/examples/spark2_5/config_1.7B_fp32.json b/examples/spark2_5/config_1.7B_fp32.json new file mode 100644 index 000000000..904702ecd --- /dev/null +++ b/examples/spark2_5/config_1.7B_fp32.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Spark2_5ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_spark.Spark2_5Config", + "AutoModel": "modeling_spark.Spark2_5Model", + "AutoModelForCausalLM": "modeling_spark.Spark2_5ForCausalLM" + }, + "bos_token_id": 0, + "dtype": "bfloat16", + "eos_token_id": 1, + "gate_attn_act_mode": "sigmoid", + "head_dim": 256, + "headwise_attn_output_gate": true, + "hidden_act": "gelu", + "hidden_size": 2048, + "initializer_range": 0.0221, + "intermediate_size": 6656, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 1048576, + "mlp_bias": false, + "model_type": "spark2_5", + "num_attention_heads": 8, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 2, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "partial_rotary_factor": 0.25, + "rope_theta": 5000000 + }, + "sliding_attention": { + "partial_rotary_factor": 1.0, + "rope_theta": 10000 + } + }, + "sliding_window": 512, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": true, + "vocab_size": 131072, + "max_cache_length": 4096, + "linear_impl_type": "Default" +} diff --git a/examples/spark2_5/config_1.7B_w4a32_kai.json b/examples/spark2_5/config_1.7B_w4a32_kai.json new file mode 100644 index 000000000..5037161dc --- /dev/null +++ b/examples/spark2_5/config_1.7B_w4a32_kai.json @@ -0,0 +1,77 @@ +{ + "architectures": [ + "Spark2_5ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_spark.Spark2_5Config", + "AutoModel": "modeling_spark.Spark2_5Model", + "AutoModelForCausalLM": "modeling_spark.Spark2_5ForCausalLM" + }, + "bos_token_id": 0, + "dtype": "bfloat16", + "eos_token_id": 1, + "gate_attn_act_mode": "sigmoid", + "head_dim": 256, + "headwise_attn_output_gate": true, + "hidden_act": "gelu", + "hidden_size": 2048, + "initializer_range": 0.0221, + "intermediate_size": 6656, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 1048576, + "mlp_bias": false, + "model_type": "spark2_5", + "num_attention_heads": 8, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 2, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "full_attention": { + "partial_rotary_factor": 0.25, + "rope_theta": 5000000 + }, + "sliding_attention": { + "partial_rotary_factor": 1.0, + "rope_theta": 10000 + } + }, + "sliding_window": 512, + "tie_word_embeddings": true, + "transformers_version": "4.57.1", + "use_cache": true, + "vocab_size": 131072, + "max_cache_length": 4096, + "linear_impl_type": "KaiLinear_f32_qai8dxp_qsi4c32p_mxk_nxk_qai8dxp1x8_qsi4c32p8x8_1x8x32" +} diff --git a/examples/spark2_5/convert.py b/examples/spark2_5/convert.py new file mode 100644 index 000000000..890cb2e01 --- /dev/null +++ b/examples/spark2_5/convert.py @@ -0,0 +1,125 @@ +# Copyright (c) MLLM Team. +# Licensed under the MIT License. +"""Validate the official Spark 1.7B inventory and emit a float32 ModelFileV2. + +The tied output projection is materialized from model.embedding.weight. This +portable conversion needs torch and safetensors, not a compiled mllm binding. +""" +import argparse +import json +import os +from pathlib import Path +import struct + +import torch +from safetensors import safe_open + + +def expected_shapes(config): + c = config + if (c['model_type'], c['hidden_size'], c['num_hidden_layers'], c['intermediate_size'], + c['num_attention_heads'], c['num_key_value_heads'], c['head_dim'], c['vocab_size']) != ( + 'spark2_5', 2048, 28, 6656, 8, 2, 256, 131072): + raise ValueError('Only the official Spark-X2.5-1.7B geometry is supported') + required = { + 'hidden_act': 'gelu', 'gate_attn_act_mode': 'sigmoid', + 'headwise_attn_output_gate': True, 'attention_bias': False, 'mlp_bias': False, + 'tie_word_embeddings': True, 'sliding_window': 512, 'rms_norm_eps': 1e-6, + 'max_position_embeddings': 1048576, 'bos_token_id': 0, 'eos_token_id': 1, 'pad_token_id': 2, + 'layer_types': ['sliding_attention', 'sliding_attention', 'sliding_attention', 'full_attention'] * 7, + } + if any(c.get(k) != v for k, v in required.items()): + raise ValueError('Checkpoint architecture differs from Spark-X2.5-1.7B') + for kind, theta, factor in [('sliding_attention', 10000, 1.0), ('full_attention', 5000000, 0.25)]: + rope = c['rope_parameters'][kind] + if rope['rope_theta'] != theta or rope['partial_rotary_factor'] != factor or rope.get('rope_type', 'default') != 'default': + raise ValueError('Unsupported Spark RoPE configuration') + h, f = c['hidden_size'], c['intermediate_size'] + shapes = {'model.embedding.weight': (c['vocab_size'], h), 'model.norm.weight': (h,)} + for i in range(c['num_hidden_layers']): + prefix = f'model.layers.{i}.' + for name, shape in { + 'input_layernorm.weight': (h,), 'post_attention_layernorm.weight': (h,), + 'self_attn.q_k_v_proj.weight': (3072, h), 'self_attn.g_proj.weight': (8, h), + 'self_attn.out_proj.weight': (h, h), 'mlp.gate_proj.weight': (f, h), + 'mlp.up_proj.weight': (f, h), 'mlp.down_proj.weight': (h, f), + }.items(): + shapes[prefix+name] = shape + return shapes + + +def convert(checkpoint, output, quantized=False): + config = json.loads((checkpoint/'config.json').read_text()) + shapes = expected_shapes(config) + index = json.loads((checkpoint/'model.safetensors.index.json').read_text())['weight_map'] + if set(index) != set(shapes): + raise ValueError('Checkpoint tensor inventory differs from Spark 1.7B') + for shard in set(index.values()): + if Path(shard).name != shard: + raise ValueError('Invalid shard path') + with safe_open(checkpoint/shard, framework='pt') as f: + if set(f.keys()) != {n for n, s in index.items() if s == shard}: + raise ValueError('Shard/index inventory mismatch') + for name in f.keys(): + t = f.get_slice(name) + if tuple(t.get_shape()) != shapes[name] or t.get_dtype() != 'BF16': + raise ValueError(f'Unexpected shape/dtype: {name}') + if quantized: + # Use the repository's portable KAI packer through the normal Python API. + from pymllm.mobile.convertor.model_file_v2 import ModelFileV2 + from pymllm.mobile.quantize.pipeline import BUILTIN_QUANTIZE_PIPELINE + recipe = json.loads(Path(__file__).with_name('quant_cfg_1.7B_w4a32_kai.json').read_text()) + params = {} + for shard in sorted(set(index.values())): + with safe_open(checkpoint/shard, framework='pt') as f: + for name in f.keys(): + params[name] = f.get_tensor(name) + if not torch.isfinite(params[name]).all(): + raise ValueError(f"Nonfinite checkpoint tensor: {name}") + partial = output.with_name(output.name+'.partial') + if output.exists() or partial.exists(): + raise FileExistsError(output) + pipeline = BUILTIN_QUANTIZE_PIPELINE['w4a32_kai_pipeline']() + count = pipeline.stream_quantize_params_size(recipe, params) + writer = ModelFileV2(str(partial), 'Spark-X2.5-1.7B-w4a32', 'Streaming', + max_params_descriptor_buffer_num=count) + pipeline.stream_quantize(recipe, params, writer=writer, cast_left_2_fp32=True, verbose=True) + partial.rename(output) + return + names = sorted(shapes)+['lm_head.weight'] + # Packed descriptors match pymllm/mobile/convertor/model_file_v2.py. + # Reserve only the exact descriptor count; never overwrite an existing file. + partial = output.with_name(output.name+'.partial') + if output.exists() or partial.exists(): + raise FileExistsError(output) + with partial.open('xb') as out: + out.write(struct.pack('\n#include \nnamespace mllm::models::spark2_5 {\nstruct UnicodeRange { uint32_t first, last; uint8_t flags; };\ninline constexpr UnicodeRange unicode_ranges[] = {\n") + for a, b, v in ranges: + f.write(f" {{0x{a:X}, 0x{b:X}, {v}}},\n") + f.write("};\ninline uint8_t unicodeFlags(uint32_t cp) {\n size_t l=0,r=std::size(unicode_ranges);\n while(l +#include +#include +#include "mllm/mllm.hpp" +#include "mllm/models/spark2_5/modeling_spark2_5.hpp" +#include "mllm/models/spark2_5/tokenization_spark2_5.hpp" +using mllm::Argparse; +MLLM_MAIN({ + auto engine_args = mllm::engineArgAttach(); + auto& model_path = Argparse::add("-m|--model_path").required(true).help("Converted V2 model"); + auto& config_path = Argparse::add("-c|--config_path").required(true).help("Spark runtime config"); + auto& tokenizer_path = Argparse::add("-t|--tokenizer_path").required(true).help("Official tokenizer.json"); + auto& prompt = Argparse::add("-p|--prompt").required(true).help("Prompt text"); + auto& system = Argparse::add("--system").help("Optional system message"); + auto& thinking = Argparse::add("--enable_thinking").help("Enable reasoning (default true)"); + auto& count = Argparse::add("-g|--max_new_tokens").help("Maximum generated tokens (default 128)"); + auto& print_ids = Argparse::add("--print_token_ids").help("Write generated token IDs to stderr"); + for (int i = 1; i < argc; ++i) + if (std::string(argv[i]) == "--help" || std::string(argv[i]) == "-h") { + Argparse::printHelp(); + return 0; + } + Argparse::parse(argc, argv); + mllm::Context::instance().setCpuOpThreads(1); + mllm::configEngineWithArgs(engine_args); + try { + mllm::models::spark2_5::SparkConfig config(config_path.get()); + mllm::models::spark2_5::SparkTokenizer tokenizer(tokenizer_path.get()); + const int limit = count.isSet() ? count.get() : 128; + if (limit <= 0) throw std::invalid_argument("max_new_tokens must be positive"); + const auto thinking_value = thinking.isSet() ? thinking.get() : "true"; + if (thinking_value != "true" && thinking_value != "false") + throw std::invalid_argument("enable_thinking must be true or false"); + const auto inputs = tokenizer.convertMessage({prompt.get(), system.isSet() ? system.get() : "", thinking_value == "true"}); + const int prompt_tokens = inputs.at("sequence").shape()[1]; + if (limit > config.max_cache_length || prompt_tokens > config.max_cache_length - limit + 1) + throw std::invalid_argument("Prompt and generation exceed context capacity"); + auto params = mllm::load(model_path.get(), mllm::ModelFileVersion::kV2); + mllm::models::spark2_5::validateModelConfigMatch(config, params); + mllm::models::spark2_5::SparkForCausalLM model(config); + model.load(params); + model.resetState(); + mllm::preprocessor::StreamingUtf8Decoder decoder; + int generated = 0; + model.streamGenerate(inputs, {{"max_length", mllm::AnyValue(int(limit))}, {"do_sample", mllm::AnyValue(false)}}, + [&](int64_t id) { + ++generated; + if (print_ids.isSet() && print_ids.get()) fmt::print(stderr, "TOKEN_ID:{}\n", id); + fmt::print("{}", decoder.append(tokenizer.detokenizeBytes(id))); + std::fflush(stdout); + }); + fmt::print("{}\n", decoder.finish()); + fmt::print(stderr, "Prompt tokens: {}; generated tokens: {}\n", prompt_tokens, generated); + } catch (const std::exception& e) { + fmt::print(stderr, "Spark: {}\n", e.what()); + return 1; + } + return 0; +}) diff --git a/examples/spark2_5/quant_cfg_1.7B_w4a32_kai.json b/examples/spark2_5/quant_cfg_1.7B_w4a32_kai.json new file mode 100644 index 000000000..0eda757b1 --- /dev/null +++ b/examples/spark2_5/quant_cfg_1.7B_w4a32_kai.json @@ -0,0 +1,68 @@ +{ + "^model\\.layers\\.\\d+\\.self_attn\\.q_k_v_proj\\.weight$": { + "hints": { + "quant_method": "kai", + "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", + "kai_matmul_layout": "mxk_nxk", + "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", + "replace": true, + "shape": [ + 3072, + 2048 + ] + } + }, + "^model\\.layers\\.\\d+\\.self_attn\\.out_proj\\.weight$": { + "hints": { + "quant_method": "kai", + "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", + "kai_matmul_layout": "mxk_nxk", + "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", + "replace": true, + "shape": [ + 2048, + 2048 + ] + } + }, + "^model\\.layers\\.\\d+\\.mlp\\.(gate_proj|up_proj)\\.weight$": { + "hints": { + "quant_method": "kai", + "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", + "kai_matmul_layout": "mxk_nxk", + "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", + "replace": true, + "shape": [ + 6656, + 2048 + ] + } + }, + "^model\\.layers\\.\\d+\\.mlp\\.down_proj\\.weight$": { + "hints": { + "quant_method": "kai", + "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", + "kai_matmul_layout": "mxk_nxk", + "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", + "replace": true, + "shape": [ + 2048, + 6656 + ] + } + }, + "^model\\.embedding\\.weight$": { + "hints": { + "quant_method": "kai", + "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", + "kai_matmul_layout": "mxk_nxk", + "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", + "replace": false, + "shape": [ + 131072, + 2048 + ], + "rename": "lm_head.weight" + } + } +} diff --git a/mllm/backends/cpu/ops/GELUOp.cpp b/mllm/backends/cpu/ops/GELUOp.cpp index 7e85d7f98..bc147de3b 100644 --- a/mllm/backends/cpu/ops/GELUOp.cpp +++ b/mllm/backends/cpu/ops/GELUOp.cpp @@ -2,6 +2,8 @@ // Licensed under the MIT License. #include +#include +#include #include "mllm/backends/cpu/ops/GELUOp.hpp" #include "mllm/backends/cpu/kernels/Kernels.hpp" @@ -16,6 +18,13 @@ void CPUGELUOp::forward(const std::vector& inputs, std::vector& // Only Support Contiguous Tensor MLLM_RT_ASSERT(X.isContiguous()); + if (!options_.approximate) { + if (X.dtype() != kFloat32) throw std::invalid_argument("Exact CPU GELU requires float32"); + const auto* x = X.ptr(); + auto* y = Y.ptr(); + for (size_t i = 0; i < X.numel(); ++i) y[i] = 0.5F * x[i] * (1.0F + std::erf(x[i] * 0.7071067811865475F)); + return; + } switch (X.dtype()) { case kFloat32: { #if defined(MLLM_HOST_ARCH_X86_64) || defined(MLLM_HOST_ARCH_X86) diff --git a/mllm/backends/cpu/ops/GroupedQueryAttentionOp.cpp b/mllm/backends/cpu/ops/GroupedQueryAttentionOp.cpp index a9cc82b0e..b55773007 100644 --- a/mllm/backends/cpu/ops/GroupedQueryAttentionOp.cpp +++ b/mllm/backends/cpu/ops/GroupedQueryAttentionOp.cpp @@ -16,7 +16,8 @@ namespace mllm::cpu { namespace { template -void groupedQueryAttentionDirectStrided(const Tensor& query, const Tensor& key, const Tensor& value, Tensor& output) { +void groupedQueryAttentionDirectStrided(const Tensor& query, const Tensor& key, const Tensor& value, Tensor& output, + int32_t sliding_window) { const auto q_shape = query.shape(); const auto k_shape = key.shape(); const auto v_shape = value.shape(); @@ -70,11 +71,12 @@ void groupedQueryAttentionDirectStrided(const Tensor& query, const Tensor& key, std::vector scores(static_cast(k_shape[2])); for (int32_t query_index = 0; query_index < q_shape[2]; ++query_index) { const int32_t visible_keys = context_offset + query_index + 1; + const int32_t first_key = sliding_window > 0 ? std::max(0, visible_keys - sliding_window) : 0; const size_t query_row_index = query_row_base + query_index; const Scalar* query_row = query_row_data[query_row_index]; Scalar* output_row = output_row_data[query_row_index]; float maximum = std::numeric_limits::lowest(); - for (int32_t key_index = 0; key_index < visible_keys; ++key_index) { + for (int32_t key_index = first_key; key_index < visible_keys; ++key_index) { float dot = 0.0F; const Scalar* key_row = key_row_data[key_row_base + key_index]; // Keep the contiguous dot expression separate: folding it into the @@ -98,14 +100,14 @@ void groupedQueryAttentionDirectStrided(const Tensor& query, const Tensor& key, } float denominator = 0.0F; - for (int32_t key_index = 0; key_index < visible_keys; ++key_index) { + for (int32_t key_index = first_key; key_index < visible_keys; ++key_index) { scores[key_index] = std::exp(scores[key_index] - maximum); denominator += scores[key_index]; } const float inverse_denominator = 1.0F / denominator; for (int32_t value_dim = 0; value_dim < v_shape[3]; ++value_dim) { float accumulated = 0.0F; - for (int32_t key_index = 0; key_index < visible_keys; ++key_index) { + for (int32_t key_index = first_key; key_index < visible_keys; ++key_index) { accumulated += (scores[key_index] * static_cast(value_row_data[value_row_base + key_index][value_dim * v_stride[3]])) * inverse_denominator; @@ -119,8 +121,7 @@ void groupedQueryAttentionDirectStrided(const Tensor& query, const Tensor& key, // Scalar fallback for the decode variant, used when the vectorized decode // kernel declines the given geometry. -void groupedQueryAttentionDecodeFloat32Reference(const Tensor& query, const Tensor& key, const Tensor& value, - Tensor& output) { +void groupedQueryAttentionDecodeFloat32Reference(const Tensor& query, const Tensor& key, const Tensor& value, Tensor& output) { const auto q_shape = query.shape(); const auto k_shape = key.shape(); const auto v_shape = value.shape(); @@ -212,9 +213,10 @@ void CPUGroupedQueryAttentionOp::forward(const std::vector& inputs, std: switch (options_.implementation) { case aops::GroupedQueryAttentionImplementation::kDirectStrided: if (inputs[0].dtype() == kFloat32) { - groupedQueryAttentionDirectStrided(inputs[0], inputs[1], inputs[2], outputs[0]); + groupedQueryAttentionDirectStrided(inputs[0], inputs[1], inputs[2], outputs[0], options_.sliding_window); } else { - groupedQueryAttentionDirectStrided(inputs[0], inputs[1], inputs[2], outputs[0]); + groupedQueryAttentionDirectStrided(inputs[0], inputs[1], inputs[2], outputs[0], + options_.sliding_window); } return; case aops::GroupedQueryAttentionImplementation::kDecodeNativeKV: diff --git a/mllm/backends/cpu/ops/SigmoidOp.cpp b/mllm/backends/cpu/ops/SigmoidOp.cpp index 9bd8880f7..3ddf595f9 100644 --- a/mllm/backends/cpu/ops/SigmoidOp.cpp +++ b/mllm/backends/cpu/ops/SigmoidOp.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include +#include #include "mllm/backends/cpu/ops/SigmoidOp.hpp" #include "mllm/backends/cpu/kernels/Kernels.hpp" @@ -13,6 +14,10 @@ void CPUSigmoidOp::forward(const std::vector& inputs, std::vector()[i] = 1.0F / (1.0F + std::exp(-X.ptr()[i])); + return; + } switch (X.dtype()) { case kFloat32: { #if defined(MLLM_HOST_ARCH_X86_64) || defined(MLLM_HOST_ARCH_X86) diff --git a/mllm/compile/jit/binary/LinalgIRSerialization.cpp b/mllm/compile/jit/binary/LinalgIRSerialization.cpp index 2a8138853..5e6a13329 100644 --- a/mllm/compile/jit/binary/LinalgIRSerialization.cpp +++ b/mllm/compile/jit/binary/LinalgIRSerialization.cpp @@ -13,6 +13,8 @@ #include "mllm/core/aops/FlashAttention2Op.hpp" #include "mllm/core/aops/CausalDepthwiseConv1DOp.hpp" #include "mllm/core/aops/GroupedQueryAttentionOp.hpp" +#include "mllm/core/aops/GELUOp.hpp" +#include "mllm/core/aops/SigmoidOp.hpp" #include "mllm/core/aops/ParallelLinearOp.hpp" #include "mllm/core/aops/GatedDeltaRuleOp.hpp" #include "mllm/core/aops/KVCacheOp.hpp" @@ -83,6 +85,7 @@ nlohmann::json dumpLinalgIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { CASE(Conv2D) CASE(Conv3D) CASE(GELU) + CASE(Sigmoid) CASE(LayerNorm) CASE(MultimodalRoPE) CASE(VisionRoPE) @@ -164,7 +167,8 @@ nlohmann::json dumpGatedDeltaRuleOpIROptions(const ir::linalg::LinalgIROp::ptr_t nlohmann::json dumpGroupedQueryAttentionOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { const auto options = static_cast(op->getAOp())->options(); - return {{"implementation", aops::groupedQueryAttentionImplementation2Str(options.implementation)}}; + return {{"implementation", aops::groupedQueryAttentionImplementation2Str(options.implementation)}, + {"sliding_window", options.sliding_window}}; } nlohmann::json dumpParallelLinearOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { @@ -254,7 +258,6 @@ nlohmann::json dumpFlashAttention2OpIROptions(const ir::linalg::LinalgIROp::ptr_ {"D", options.D}, {"hp_exp", options.hp_exp}, {"causal_mask", options.causal_mask}}; } - nlohmann::json dumpRepeatOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { auto options = ((aops::RepeatOp*)op->getAOp())->options(); return {{"dim", options.dim}, {"repeat_times", options.repeat_times}}; @@ -295,7 +298,13 @@ nlohmann::json dumpConv3DOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { return j; } -nlohmann::json dumpGELUOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { return {}; } +nlohmann::json dumpGELUOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { + return {{"approximate", static_cast(op->getAOp())->options().approximate}}; +} + +nlohmann::json dumpSigmoidOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { + return {{"approximate", static_cast(op->getAOp())->options().approximate}}; +} nlohmann::json dumpLayerNormOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op) { auto options = ((aops::LayerNormOp*)op->getAOp())->options(); diff --git a/mllm/compile/jit/binary/LinalgIRSerialization.hpp b/mllm/compile/jit/binary/LinalgIRSerialization.hpp index ca3c32d48..dc201bd96 100644 --- a/mllm/compile/jit/binary/LinalgIRSerialization.hpp +++ b/mllm/compile/jit/binary/LinalgIRSerialization.hpp @@ -46,6 +46,7 @@ nlohmann::json dumpPermuteOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op); nlohmann::json dumpConv1DOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op); nlohmann::json dumpConv2DOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op); nlohmann::json dumpConv3DOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op); +nlohmann::json dumpSigmoidOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op); nlohmann::json dumpGELUOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op); nlohmann::json dumpLayerNormOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op); nlohmann::json dumpMultimodalRoPEOpIROptions(const ir::linalg::LinalgIROp::ptr_t& op); diff --git a/mllm/compile/jit/interpreter/AopsFromJson.cpp b/mllm/compile/jit/interpreter/AopsFromJson.cpp index f3ecdcbb4..7536b832a 100644 --- a/mllm/compile/jit/interpreter/AopsFromJson.cpp +++ b/mllm/compile/jit/interpreter/AopsFromJson.cpp @@ -29,6 +29,7 @@ #include "mllm/core/aops/RepeatOp.hpp" #include "mllm/core/aops/PermuteOp.hpp" #include "mllm/core/aops/GELUOp.hpp" +#include "mllm/core/aops/SigmoidOp.hpp" #include "mllm/core/aops/LayerNormOp.hpp" #include "mllm/core/aops/VisionRoPEOp.hpp" #include "mllm/core/aops/QuickGELUOp.hpp" @@ -121,6 +122,8 @@ BaseOp::ptr_t aopsFromJson(const nlohmann::json& json) { return __repeatFromJson(json); } else if (op_type == "Permute") { return __permuteFromJson(json); + } else if (op_type == "Sigmoid") { + return __sigmoidFromJson(json); } else if (op_type == "GELU") { return __geluFromJson(json); } else if (op_type == "LayerNorm") { @@ -648,6 +651,7 @@ BaseOp::ptr_t __causalDepthwiseConv1DFromJson(const nlohmann::json& json) { BaseOp::ptr_t __groupedQueryAttentionFromJson(const nlohmann::json& json) { aops::GroupedQueryAttentionOpOptions options; + if (json.contains("op_options")) options.sliding_window = json["op_options"].value("sliding_window", 0); if (json.contains("op_options") && json["op_options"].contains("implementation")) { options.implementation = aops::str2GroupedQueryAttentionImplementation(json["op_options"]["implementation"].get()); @@ -727,6 +731,7 @@ BaseOp::ptr_t __permuteFromJson(const nlohmann::json& json) { BaseOp::ptr_t __geluFromJson(const nlohmann::json& json) { aops::GELUOpOptions options; + if (json.contains("op_options")) options.approximate = json["op_options"].value("approximate", true); DeviceTypes backend = DeviceTypes::kCPU; if (json.contains("backend")) { backend = str2DeviceType(json["backend"]); } @@ -736,6 +741,18 @@ BaseOp::ptr_t __geluFromJson(const nlohmann::json& json) { return op; } +BaseOp::ptr_t __sigmoidFromJson(const nlohmann::json& json) { + aops::SigmoidOpOptions options; + if (json.contains("op_options")) options.approximate = json["op_options"].value("approximate", true); + + DeviceTypes backend = DeviceTypes::kCPU; + if (json.contains("backend")) { backend = str2DeviceType(json["backend"]); } + + // Use Context to create op + auto op = Context::instance().getBackend(backend)->createOp(OpTypes::kSigmoid, options); + return op; +} + BaseOp::ptr_t __layerNormFromJson(const nlohmann::json& json) { aops::LayerNormOpOptions options; diff --git a/mllm/compile/jit/interpreter/AopsFromJson.hpp b/mllm/compile/jit/interpreter/AopsFromJson.hpp index 87d6f2e71..165643f49 100644 --- a/mllm/compile/jit/interpreter/AopsFromJson.hpp +++ b/mllm/compile/jit/interpreter/AopsFromJson.hpp @@ -42,6 +42,7 @@ BaseOp::ptr_t __gatedDeltaRuleFromJson(const nlohmann::json& json); BaseOp::ptr_t __repeatFromJson(const nlohmann::json& json); BaseOp::ptr_t __permuteFromJson(const nlohmann::json& json); BaseOp::ptr_t __conv2dFromJson(const nlohmann::json& json); +BaseOp::ptr_t __sigmoidFromJson(const nlohmann::json& json); BaseOp::ptr_t __geluFromJson(const nlohmann::json& json); BaseOp::ptr_t __layerNormFromJson(const nlohmann::json& json); BaseOp::ptr_t __multimodalRopeFromJson(const nlohmann::json& json); diff --git a/mllm/core/aops/GELUOp.cpp b/mllm/core/aops/GELUOp.cpp index 7447906e8..3096e91cd 100644 --- a/mllm/core/aops/GELUOp.cpp +++ b/mllm/core/aops/GELUOp.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "mllm/core/aops/GELUOp.hpp" +#include #include "mllm/core/BaseOp.hpp" #include "mllm/core/Tensor.hpp" #include "mllm/utils/Common.hpp" @@ -25,7 +26,10 @@ void GELUOp::forward(const std::vector& inputs, std::vector& out } void GELUOp::reshape(const std::vector& inputs, std::vector& outputs) { + if (inputs.size() != 1) throw std::invalid_argument("GELU expects one input"); const auto& i = inputs[0]; + if (!options_.approximate && (i.dtype() != kFloat32 || i.device() != kCPU || !i.isContiguous())) + throw std::invalid_argument("Exact GELU requires contiguous float32 CPU input"); outputs.emplace_back(Tensor::empty(i.shape(), i.dtype(), i.device())); } diff --git a/mllm/core/aops/GELUOp.hpp b/mllm/core/aops/GELUOp.hpp index a5f89f104..3b3f500e0 100644 --- a/mllm/core/aops/GELUOp.hpp +++ b/mllm/core/aops/GELUOp.hpp @@ -8,7 +8,10 @@ namespace mllm::aops { -struct GELUOpOptions : public BaseOpOptions {}; +// Preserve the historical tanh approximation unless exact erf is requested. +struct GELUOpOptions : public BaseOpOptions { + bool approximate = true; +}; class GELUOp : public BaseOp { public: diff --git a/mllm/core/aops/GroupedQueryAttentionOp.cpp b/mllm/core/aops/GroupedQueryAttentionOp.cpp index a41b19411..64313b0d1 100644 --- a/mllm/core/aops/GroupedQueryAttentionOp.cpp +++ b/mllm/core/aops/GroupedQueryAttentionOp.cpp @@ -28,6 +28,9 @@ void GroupedQueryAttentionOp::forward(const std::vector& inputs, std::ve } void GroupedQueryAttentionOp::reshape(const std::vector& inputs, std::vector& outputs) { + if (options_.sliding_window < 0 + || (options_.sliding_window != 0 && options_.implementation != GroupedQueryAttentionImplementation::kDirectStrided)) + throw std::invalid_argument("Sliding window requires DirectStrided and a nonnegative window"); if (inputs.size() != 3) { throw std::invalid_argument("GroupedQueryAttention expects query, key, and value"); } const auto& query = inputs[0]; const auto& key = inputs[1]; diff --git a/mllm/core/aops/GroupedQueryAttentionOp.hpp b/mllm/core/aops/GroupedQueryAttentionOp.hpp index 884818775..1f2fbe9db 100644 --- a/mllm/core/aops/GroupedQueryAttentionOp.hpp +++ b/mllm/core/aops/GroupedQueryAttentionOp.hpp @@ -40,6 +40,9 @@ inline GroupedQueryAttentionImplementation str2GroupedQueryAttentionImplementati struct GroupedQueryAttentionOpOptions : public BaseOpOptions { GroupedQueryAttentionImplementation implementation = GroupedQueryAttentionImplementation::kDirectStrided; + // Zero: full causal history. Positive: at most this many keys, including self. + // Queries align with the last Sq positions of the chronological KV inputs. + int32_t sliding_window = 0; }; class GroupedQueryAttentionOp : public BaseOp { diff --git a/mllm/core/aops/SigmoidOp.cpp b/mllm/core/aops/SigmoidOp.cpp index a57d89255..40d22495f 100644 --- a/mllm/core/aops/SigmoidOp.cpp +++ b/mllm/core/aops/SigmoidOp.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "mllm/core/aops/SigmoidOp.hpp" +#include #include "mllm/core/BaseOp.hpp" #include "mllm/core/Tensor.hpp" #include "mllm/utils/Common.hpp" @@ -25,6 +26,10 @@ void SigmoidOp::forward(const std::vector& inputs, std::vector& } void SigmoidOp::reshape(const std::vector& inputs, std::vector& outputs) { + if (inputs.size() != 1) throw std::invalid_argument("Sigmoid expects one input"); + const auto& x = inputs[0]; + if (!options_.approximate && (x.dtype() != kFloat32 || x.device() != kCPU || !x.isContiguous())) + throw std::invalid_argument("Accurate Sigmoid requires contiguous float32 CPU input"); if (options_.isInplace()) { outputs.emplace_back(inputs[0]); } else { diff --git a/mllm/core/aops/SigmoidOp.hpp b/mllm/core/aops/SigmoidOp.hpp index 29c5651d5..0036e0686 100644 --- a/mllm/core/aops/SigmoidOp.hpp +++ b/mllm/core/aops/SigmoidOp.hpp @@ -8,7 +8,10 @@ namespace mllm::aops { -struct SigmoidOpOptions : public BaseOpOptions {}; +struct SigmoidOpOptions : public BaseOpOptions { + // Preserve the existing backend approximation unless accurate logistic is requested. + bool approximate = true; +}; class SigmoidOp : public BaseOp { public: diff --git a/mllm/models/spark2_5/configuration_spark2_5.hpp b/mllm/models/spark2_5/configuration_spark2_5.hpp new file mode 100644 index 000000000..9e8b7ee81 --- /dev/null +++ b/mllm/models/spark2_5/configuration_spark2_5.hpp @@ -0,0 +1,121 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. +#pragma once +#include +#include +#include "mllm/engine/ConfigFile.hpp" +#include +#include +#include "mllm/core/aops/LinearOp.hpp" +#include "mllm/core/ParameterFile.hpp" +#include + +namespace mllm::models::spark2_5 { +struct SparkConfig : protected ConfigFile { + SparkConfig() = default; + explicit SparkConfig(const std::string& path) : ConfigFile(path) { + const auto& config = data(); + vocab_size = config.at("vocab_size"); + hidden_size = config.at("hidden_size"); + intermediate_size = config.at("intermediate_size"); + num_hidden_layers = config.at("num_hidden_layers"); + num_attention_heads = config.at("num_attention_heads"); + num_key_value_heads = config.at("num_key_value_heads"); + head_dim = config.at("head_dim"); + sliding_window = config.at("sliding_window"); + max_position_embeddings = config.at("max_position_embeddings"); + rms_norm_eps = config.at("rms_norm_eps"); + max_cache_length = config.value("max_cache_length", 4096); + layer_types = config.at("layer_types").get>(); + const auto& rope = config.at("rope_parameters"); + for (const auto& type : {"full_attention", "sliding_attention"}) + if (rope.at(type).value("rope_type", std::string("default")) != "default") + throw std::invalid_argument("Only default Spark RoPE is supported"); + full_theta = rope.at("full_attention").at("rope_theta"); + sliding_theta = rope.at("sliding_attention").at("rope_theta"); + const double full_factor = rope.at("full_attention").at("partial_rotary_factor"); + const double sliding_factor = rope.at("sliding_attention").at("partial_rotary_factor"); + if (!std::isfinite(full_factor) || !std::isfinite(sliding_factor) || full_factor <= 0 || full_factor > 1 + || sliding_factor != 1.0 || head_dim <= 0 || full_factor * head_dim != std::floor(full_factor * head_dim)) + throw std::invalid_argument("Unsupported Spark RoPE geometry"); + rotary_dim = static_cast(head_dim * full_factor); + if (config.at("model_type") != "spark2_5" || config.at("hidden_act") != "gelu" + || config.at("gate_attn_act_mode") != "sigmoid" || !config.at("headwise_attn_output_gate").get() + || config.at("attention_bias").get() || config.at("mlp_bias").get() + || !config.at("tie_word_embeddings").get() || config.at("eos_token_id") != 1 || config.at("bos_token_id") != 0 + || config.at("pad_token_id") != 2) + throw std::invalid_argument("Unsupported Spark architecture or special tokens"); + const auto impl = config.value("linear_impl_type", std::string("Default")); + linear_impl_type = aops::str2LinearImplTypes(impl); + constexpr auto kai = aops::LinearImplTypes::kKaiLinear_f32_qai8dxp_qsi4c32p_mxk_nxk_qai8dxp1x8_qsi4c32p8x8_1x8x32; + if ((linear_impl_type != aops::LinearImplTypes::kDefault && linear_impl_type != kai) + || (linear_impl_type == aops::LinearImplTypes::kDefault && impl != "Default")) + throw std::invalid_argument("Unsupported Spark linear implementation"); + if (vocab_size < 3 || hidden_size <= 0 || intermediate_size <= 0 || num_hidden_layers <= 0 || num_attention_heads <= 0 + || num_key_value_heads <= 0 || num_attention_heads % num_key_value_heads || head_dim % 2 || rotary_dim < 2 + || rotary_dim % 2 || sliding_window < 2 || max_cache_length <= 0 || max_cache_length > max_position_embeddings + || !std::isfinite(rms_norm_eps) || rms_norm_eps <= 0 || !std::isfinite(full_theta) || full_theta <= 0 + || !std::isfinite(sliding_theta) || sliding_theta <= 0 || layer_types.size() != static_cast(num_hidden_layers)) + throw std::invalid_argument("Invalid Spark dimensions, cache or normalization"); + for (const auto& type : layer_types) { + if (type == "full_attention") + ++full_layers; + else if (type != "sliding_attention") + throw std::invalid_argument("Unknown Spark layer type"); + } + if (full_layers == 0) throw std::invalid_argument("Spark requires at least one full attention layer"); + } + int32_t vocab_size, hidden_size, intermediate_size, num_hidden_layers, num_attention_heads, num_key_value_heads; + int32_t head_dim, sliding_window, rotary_dim, max_position_embeddings, max_cache_length, full_layers = 0; + float rms_norm_eps, full_theta, sliding_theta; + std::vector layer_types; + aops::LinearImplTypes linear_impl_type; +}; + +inline void validateModelConfigMatch(const SparkConfig& c, const ParameterFile::ptr_t& params) { + if (c.vocab_size != 131072 || c.hidden_size != 2048 || c.intermediate_size != 6656 || c.num_hidden_layers != 28 + || c.num_attention_heads != 8 || c.num_key_value_heads != 2 || c.head_dim != 256 || c.rotary_dim != 64 + || c.sliding_window != 512 || c.full_theta != 5000000.0F || c.sliding_theta != 10000.0F || c.rms_norm_eps != 1e-6F + || c.max_position_embeddings != 1048576 || c.max_cache_length > 8192) + throw std::invalid_argument("Runner supports Spark-X2.5-1.7B with at most 8192 context tokens"); + for (int i = 0; i < 28; ++i) + if (c.layer_types[i] != (i % 4 == 3 ? "full_attention" : "sliding_attention")) + throw std::invalid_argument("Spark layer schedule differs from the official checkpoint"); + if (!params || params->version() != ModelFileVersion::kV2) throw std::invalid_argument("Spark requires ModelFileV2"); + const bool quantized = c.linear_impl_type != aops::LinearImplTypes::kDefault; + std::map shapes; + auto add = [&](const std::string& name, int n, int k, bool packed) { + if (packed && quantized) + shapes[name] = {n * (k / 32 * 18 + 8)}; + else + shapes[name] = k ? Tensor::shape_t{n, k} : Tensor::shape_t{n}; + }; + add("model.embedding.weight", 131072, 2048, false); + add("lm_head.weight", 131072, 2048, true); + add("model.norm.weight", 2048, 0, false); + for (int i = 0; i < 28; ++i) { + auto p = "model.layers." + std::to_string(i) + "."; + add(p + "input_layernorm.weight", 2048, 0, false); + add(p + "post_attention_layernorm.weight", 2048, 0, false); + add(p + "self_attn.q_k_v_proj.weight", 3072, 2048, true); + add(p + "self_attn.out_proj.weight", 2048, 2048, true); + add(p + "self_attn.g_proj.weight", 8, 2048, false); + add(p + "mlp.gate_proj.weight", 6656, 2048, true); + add(p + "mlp.up_proj.weight", 6656, 2048, true); + add(p + "mlp.down_proj.weight", 2048, 6656, true); + } + size_t count = 0; + for (const auto& entry : *params) { + ++count; + if (!shapes.contains(entry.first)) throw std::invalid_argument("Unexpected Spark parameter: " + entry.first); + } + if (count != shapes.size()) throw std::invalid_argument("Missing Spark parameters"); + for (const auto& [name, shape] : shapes) { + if (!params->has(name)) throw std::invalid_argument("Missing Spark parameter: " + name); + const auto t = params->pull(name); + const bool packed = quantized && shape.size() == 1 && shape[0] != 2048; + if (t.shape() != shape || t.dtype() != (packed ? kByte : kFloat32)) + throw std::invalid_argument("Spark model/config mismatch: " + name); + } +} +} // namespace mllm::models::spark2_5 diff --git a/mllm/models/spark2_5/modeling_spark2_5.hpp b/mllm/models/spark2_5/modeling_spark2_5.hpp new file mode 100644 index 000000000..c4d43a919 --- /dev/null +++ b/mllm/models/spark2_5/modeling_spark2_5.hpp @@ -0,0 +1,200 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. +#pragma once +#include "mllm/mllm.hpp" +#include "mllm/utils/Enumerate.hpp" +#include "mllm/models/ARGeneration.hpp" +#include "mllm/models/common/rope_tables.hpp" +#include "mllm/models/spark2_5/configuration_spark2_5.hpp" +#include "mllm/nn/Functional.hpp" +#include "mllm/nn/layers/GroupedQueryAttention.hpp" +#include "mllm/nn/lmcache/KVHeadStaticCache.hpp" + +namespace mllm::models::spark2_5 { + +class SparkMLP final : public nn::Module { + public: + SparkMLP() = default; + SparkMLP(const std::string& name, const SparkConfig& c) : Module(name) { + gate_ = reg("gate_proj", c.hidden_size, c.intermediate_size, false, c.linear_impl_type); + up_ = reg("up_proj", c.hidden_size, c.intermediate_size, false, c.linear_impl_type); + down_ = reg("down_proj", c.intermediate_size, c.hidden_size, false, c.linear_impl_type); + gelu_ = reg("act", aops::GELUOpOptions{.approximate = false}); + } + std::vector forward(const std::vector& x, const std::vector&) override { + return {down_(gelu_(gate_(x[0])) * up_(x[0]))}; + } + + private: + nn::Linear gate_, up_, down_; + nn::GELU gelu_; +}; + +class SparkAttention final : public nn::Module { + public: + SparkAttention() = default; + SparkAttention(const std::string& name, const SparkConfig& c) : Module(name), c_(c) { + qkv_ = reg("q_k_v_proj", c.hidden_size, (c.num_attention_heads + 2 * c.num_key_value_heads) * c.head_dim, false, + c.linear_impl_type); + gate_ = reg("g_proj", c.hidden_size, c.num_attention_heads, false); + out_ = reg("out_proj", c.num_attention_heads * c.head_dim, c.hidden_size, false, c.linear_impl_type); + sigmoid_ = reg("gate_sigmoid", aops::SigmoidOpOptions{.approximate = false}); + full_q_ = reg("full_q_rope", c.full_theta, c.max_position_embeddings, c.rotary_dim); + full_k_ = reg("full_k_rope", c.full_theta, c.max_position_embeddings, c.rotary_dim); + sliding_q_ = reg("sliding_q_rope", c.sliding_theta, c.max_position_embeddings, c.head_dim); + sliding_k_ = reg("sliding_k_rope", c.sliding_theta, c.max_position_embeddings, c.head_dim); + full_attn_ = reg("full_attention", aops::GroupedQueryAttentionImplementation::kDirectStrided); + sliding_attn_ = reg("sliding_attention", + aops::GroupedQueryAttentionImplementation::kDirectStrided, c.sliding_window); + } + void reset() { + history_k_ = Tensor(); + history_v_ = Tensor(); + } + int32_t retainedTokens() const { return history_k_.isNil() ? 0 : history_k_.shape()[2]; } + std::vector forward(const std::vector& x, const std::vector& args) override { + const int32_t s = x[0].shape()[1], qdim = c_.num_attention_heads * c_.head_dim, + kvdim = c_.num_key_value_heads * c_.head_dim; + auto qkv = qkv_(x[0]); + auto q = qkv[{kAll, kAll, {0, qdim}}].contiguous().view({1, s, c_.num_attention_heads, c_.head_dim}).transpose(1, 2); + auto k = + qkv[{kAll, kAll, {qdim, qdim + kvdim}}].contiguous().view({1, s, c_.num_key_value_heads, c_.head_dim}).transpose(1, 2); + auto v = qkv[{kAll, kAll, {qdim + kvdim, qdim + 2 * kvdim}}] + .contiguous() + .view({1, s, c_.num_key_value_heads, c_.head_dim}) + .transpose(1, 2); + Tensor output; + if (sliding) { + q = sliding_q_(q, x[3], x[4]); + k = sliding_k_(k, x[3], x[4]); + if (!history_k_.isNil()) { + k = nn::functional::concat({history_k_, k}, 2); + v = nn::functional::concat({history_v_, v}, 2); + } + output = sliding_attn_(q, k, v); + // Retain W-1 past tokens for the next call, after all current queries have attended. + const int32_t end = k.shape()[2], begin = std::max(0, end - c_.sliding_window + 1); + history_k_ = k[{kAll, kAll, {begin, end}, kAll}].contiguous().clone(); + history_v_ = v[{kAll, kAll, {begin, end}, kAll}].contiguous().clone(); + } else { + q = full_q_(q, x[1], x[2]); + k = full_k_(k, x[1], x[2]); + auto* cache = args[0].get(); + auto kv = cache->updateKVCache(full_slot, k, v); + output = full_attn_(q, kv[0], kv[1]); + } + auto gate = sigmoid_(gate_(x[0])).view({1, s, c_.num_attention_heads, 1}); + output = (output.transpose(1, 2) * gate).view({1, s, qdim}); + return {out_(output)}; + } + bool sliding = false; + int32_t full_slot = 0; + + private: + SparkConfig c_; + nn::Linear qkv_, gate_, out_; + nn::Sigmoid sigmoid_; + nn::RoPE full_q_, full_k_, sliding_q_, sliding_k_; + nn::GroupedQueryAttention full_attn_, sliding_attn_; + Tensor history_k_, history_v_; +}; + +class SparkDecoder final : public nn::Module { + public: + SparkDecoder() = default; + SparkDecoder(const std::string& name, const SparkConfig& c) : Module(name) { + attn = reg("self_attn", c); + mlp_ = reg("mlp", c); + in_norm_ = reg("input_layernorm", c.rms_norm_eps); + post_norm_ = reg("post_attention_layernorm", c.rms_norm_eps); + } + std::vector forward(const std::vector& x, const std::vector& args) override { + auto h = attn(in_norm_(x[0]), x[1], x[2], x[3], x[4], args[0])[0] + x[0]; + return {h + mlp_(post_norm_(h))[0]}; + } + SparkAttention attn; + + private: + SparkMLP mlp_; + nn::RMSNorm in_norm_, post_norm_; +}; + +class SparkText final : public nn::Module { + public: + SparkText() = default; + SparkText(const std::string& name, const SparkConfig& c) : Module(name) { + embedding_ = reg("embedding", c.vocab_size, c.hidden_size); + layers_ = reg>("layers", c.num_hidden_layers, c); + int slot = 0; + for (auto [i, layer] : enumerate(layers_.list())) { + layer.attn.sliding = c.layer_types[i] == "sliding_attention"; + if (!layer.attn.sliding) layer.attn.full_slot = slot++; + } + norm_ = reg("norm", c.rms_norm_eps); + } + std::vector forward(const std::vector& x, const std::vector& args) override { + auto h = embedding_(x[0]); + for (auto& layer : layers_.list()) h = layer(h, x[1], x[2], x[3], x[4], args[0])[0]; + return {norm_(h)}; + } + void reset() { + for (auto& layer : layers_.list()) layer.attn.reset(); + } + + private: + nn::Embedding embedding_; + nn::ModuleList layers_; + nn::RMSNorm norm_; +}; + +class SparkForCausalLM final : public ARGeneration, public nn::Module { + public: + explicit SparkForCausalLM(const SparkConfig& c) + : c_(c), cache_(c.max_cache_length, c.full_layers, c.num_key_value_heads, c.head_dim) { + eos_token_id_ = 1; + max_length_ = c.max_cache_length; + model_ = reg("model", c); + // Converter derives this Linear's weights from the same checkpoint embedding. + head_ = reg("lm_head", c.hidden_size, c.vocab_size, false, c.linear_impl_type); + registerBuffer("full_inv_freq", common::makeRoPEInvFreq(c.rotary_dim, c.full_theta)); + registerBuffer("sliding_inv_freq", common::makeRoPEInvFreq(c.head_dim, c.sliding_theta)); + } + ARGenerationOutputPast forward(const ARGenerationOutputPast& input, const ARGenerationArgs& args) override { + auto tokens = input.at("sequence"); + const auto shape = tokens.shape(); + if (shape.size() != 2 || shape[0] != 1 || shape[1] <= 0 || tokens.dtype() != kInt64 || tokens.device() != kCPU + || !tokens.isContiguous()) + throw std::invalid_argument("Spark expects contiguous int64 CPU tokens [1,S]"); + if (shape[1] > c_.max_cache_length - position_) throw std::invalid_argument("Spark context capacity exceeded"); + for (size_t i = 0; i < tokens.numel(); ++i) + if (tokens.ptr()[i] < 0 || tokens.ptr()[i] >= c_.vocab_size) + throw std::invalid_argument("Spark token outside vocabulary"); + auto positions = Tensor::empty({1, shape[1]}, kInt64, kCPU).alloc(); + for (int i = 0; i < shape[1]; ++i) positions.ptr()[i] = position_ + i; + auto [fs, fc] = common::makeRotaryPosEmbedding(positions, getBuffer("full_inv_freq")); + auto [ss, sc] = common::makeRotaryPosEmbedding(positions, getBuffer("sliding_inv_freq")); + try { + auto h = model_(tokens, fs, fc, ss, sc, AnyValue(&cache_))[0]; + position_ += shape[1]; + if (!all_logits) h = h[{kAll, {shape[1] - 1}, kAll}]; + return {{"sequence", head_(h)}, {"position_ids", positions}}; + } catch (...) { + resetState(); + throw; + } + } + void resetState() { + cache_.clearCache(); + model_.reset(); + position_ = 0; + } + bool all_logits = false; + + private: + SparkConfig c_; + SparkText model_; + nn::Linear head_; + nn::KVHeadStaticCache cache_; + int32_t position_ = 0; +}; +} // namespace mllm::models::spark2_5 diff --git a/mllm/models/spark2_5/tokenization_spark2_5.hpp b/mllm/models/spark2_5/tokenization_spark2_5.hpp new file mode 100644 index 000000000..a04a6e365 --- /dev/null +++ b/mllm/models/spark2_5/tokenization_spark2_5.hpp @@ -0,0 +1,201 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. +#pragma once +#include +#include +#include +#include "mllm/models/ARGeneration.hpp" +#include "mllm/models/spark2_5/unicode_ranges.hpp" +#include "mllm/preprocessor/StreamingUtf8Decoder.hpp" +#include "mllm/preprocessor/tokenizers/BPE.hpp" +#include "mllm/preprocessor/tokenizers/Unicode.hpp" + +namespace mllm::models::spark2_5 { +// The official pipeline first isolates numeric runs and CJK runs, then applies +// its ordered word/punctuation/whitespace alternatives, then isolates digits. +inline std::vector sparkPieces(const std::wstring& text) { + auto number = [](wchar_t c) { return (unicodeFlags(c) & 16) != 0; }; + auto cjk = [](wchar_t c) { return (c >= 0x4e00 && c <= 0x9fa5) || (c >= 0x3040 && c <= 0x30ff); }; + std::vector numeric, blocks, words, result; + size_t i = 0; + while (i < text.size()) { + size_t b = i++; + if (number(text[b])) { + while (i < text.size() && i - b < 3 && number(text[i])) ++i; + } else { + while (i < text.size() && !number(text[i])) ++i; + } + numeric.push_back(text.substr(b, i - b)); + } + for (const auto& t : numeric) { + i = 0; + while (i < t.size()) { + size_t b = i++; + bool kind = cjk(t[b]); + while (i < t.size() && cjk(t[i]) == kind) ++i; + blocks.push_back(t.substr(b, i - b)); + } + } + auto letter = [](wchar_t c) { return (unicodeFlags(c) & 1) != 0; }; + auto mark = [](wchar_t c) { return (unicodeFlags(c) & 2) != 0; }; + auto punct = [](wchar_t c) { return (unicodeFlags(c) & 12) != 0; }; + auto space = [](wchar_t c) { return (unicodeFlags(c) & 64) != 0; }; + auto asciiLetter = [](wchar_t c) { return (c >= L'A' && c <= L'Z') || (c >= L'a' && c <= L'z'); }; + const std::wstring asciiPunct = L"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"; + for (const auto& t : blocks) { + std::wstring unmatched; + i = 0; + while (i < t.size()) { + size_t b = i, e = i; + bool matched = true; + if (asciiPunct.find(t[i]) != std::wstring::npos && i + 1 < t.size() && asciiLetter(t[i + 1])) { + e = i + 2; + while (e < t.size() && asciiLetter(t[e])) ++e; + } else { + // [^\r\n\p{L}\p{P}\p{S}]? [\p{L}\p{M}]+ + size_t p = i; + if (t[p] != L'\r' && t[p] != L'\n' && !letter(t[p]) && !punct(t[p])) ++p; + if (p < t.size() && (letter(t[p]) || mark(t[p]))) { + e = p + 1; + while (e < t.size() && (letter(t[e]) || mark(t[e]))) ++e; + } else if (letter(t[i]) || mark(t[i])) { + e = i + 1; + while (e < t.size() && (letter(t[e]) || mark(t[e]))) ++e; + } else { + p = i + (t[i] == L' '); + if (p < t.size() && punct(t[p])) { + e = p + 1; + while (e < t.size() && punct(t[e])) ++e; + } else if (t[i] == L'\r' || t[i] == L'\n') + e = i + 1; + else if (space(t[i])) { + e = i + 1; + while (e < t.size() && space(t[e])) ++e; + if (e < t.size() && e - i > 1) --e; // \s+(?!\S) precedes \s+ + } else { + e = i + 1; + matched = false; + } + } + } + if (!matched) + unmatched += t.substr(b, e - b); + else { + if (!unmatched.empty()) { + words.push_back(std::move(unmatched)); + unmatched.clear(); + } + words.push_back(t.substr(b, e - b)); + } + i = e; + } + if (!unmatched.empty()) words.push_back(std::move(unmatched)); + } + for (const auto& t : words) { + i = 0; + while (i < t.size()) { + size_t b = i++; + if (!(unicodeFlags(t[b]) & 16)) + while (i < t.size() && !(unicodeFlags(t[i]) & 16)) ++i; + result.push_back(t.substr(b, i - b)); + } + } + return result; +} + +struct SparkMessage { + std::string prompt; + std::string system; + bool enable_thinking = true; +}; +class SparkTokenizer { + public: + explicit SparkTokenizer(const std::string& path) { + preprocessor::makeBytes2UnicodeMap(bytes_to_unicode_); + for (auto [b, c] : bytes_to_unicode_) unicode_to_bytes_[c] = b; + std::ifstream f(path); + if (!f) throw std::invalid_argument("Cannot read Spark tokenizer"); + const auto j = nlohmann::json::parse(f); + const auto expected = nlohmann::json::parse( + R"SPARK({"pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "\\p{N}{1,3}"}, "behavior": "Isolated", "invert": false}, {"type": "Split", "pattern": {"Regex": "[一-龥぀-ゟ゠-ヿ]+"}, "behavior": "Isolated", "invert": false}, {"type": "Split", "pattern": {"Regex": "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+|[\r\n]|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "Digits", "individual_digits": true}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "normalizer": {"type": "Sequence", "normalizers": []}, "post_processor": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": false, "use_regex": true}, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}})SPARK"); + for (const auto& [key, value] : expected.items()) + if (j.at(key) != value) throw std::invalid_argument("Unsupported Spark tokenizer pipeline: " + key); + if (j.at("model").at("type") != "BPE" || j.at("model").value("ignore_merges", false) + || j.at("model").value("byte_fallback", false)) + throw std::invalid_argument("Unsupported Spark BPE configuration"); + if (!bpe_.initFromSentencePieceJson(path)) throw std::invalid_argument("Invalid Spark BPE vocabulary"); + for (const auto& t : j.at("added_tokens")) { + const auto content = t.at("content").get(); + const int64_t id = t.at("id"); + if (t.value("lstrip", false) || t.value("rstrip", false) || t.value("single_word", false) || t.value("normalized", false)) + throw std::invalid_argument("Unsupported Spark added-token attributes"); + added_.push_back(preprocessor::utf8string2WideString(content)); + added_bytes_[id] = content; + } + for (const auto& [id, s] : std::vector>{{0, "<｜start▁of▁sentence｜>"}, + {1, "<｜end▁of▁sentence｜>"}, + {3, ""}, + {4, ""}, + {130976, "<|Bot|>"}}) + if (!added_bytes_.contains(id) || added_bytes_.at(id) != s) + throw std::invalid_argument("Incompatible Spark special tokens"); + std::sort(added_.begin(), added_.end(), [](const auto& a, const auto& b) { return a.size() > b.size(); }); + } + static std::string applyChatTemplate(const SparkMessage& m) { + if (m.prompt.empty()) throw std::invalid_argument("Spark prompt must not be empty"); + std::string s = "<｜start▁of▁sentence｜><|System|>\nyou are a helpful assistant."; + if (!m.system.empty()) s += "\n\n" + m.system; + return s + "<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>" + m.prompt + + "<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|>" + (m.enable_thinking ? "" : ""); + } + std::vector encode(const std::string& text) { + const auto w = preprocessor::utf8string2WideString(text); + std::vector ids; + auto normal = [&](size_t b, size_t e) { + for (const auto& p : sparkPieces(w.substr(b, e - b))) { + std::wstring mapped; + for (unsigned char c : preprocessor::wideString2Utf8String(p)) mapped += bytes_to_unicode_.at(c); + for (const auto& t : bpe_._bpe(mapped)) ids.push_back(bpe_._lookup_vocab(t)); + } + }; + size_t b = 0, i = 0; + while (i < w.size()) { + const std::wstring* match = nullptr; + for (const auto& t : added_) + if (w.compare(i, t.size(), t) == 0) { + match = &t; + break; + } + if (!match) { + ++i; + continue; + } + normal(b, i); + ids.push_back(bpe_._lookup_vocab(*match)); + i += match->size(); + b = i; + } + normal(b, w.size()); + return ids; + } + std::string detokenizeBytes(int64_t id) { + if (added_bytes_.contains(id)) return added_bytes_.at(id); + std::string s; + for (auto c : bpe_._lookup_inverse_vocab(id)) s += static_cast(unicode_to_bytes_.at(c)); + return s; + } + ARGenerationOutputPast convertMessage(const SparkMessage& m) { + const auto ids = encode(applyChatTemplate(m)); + auto t = Tensor::empty({1, static_cast(ids.size())}, kInt64, kCPU).alloc(); + std::copy(ids.begin(), ids.end(), t.ptr()); + return {{"sequence", t}}; + } + + private: + preprocessor::BPE bpe_; + std::unordered_map bytes_to_unicode_; + std::unordered_map unicode_to_bytes_; + std::vector added_; + std::unordered_map added_bytes_; +}; +} // namespace mllm::models::spark2_5 diff --git a/mllm/models/spark2_5/unicode_ranges.hpp b/mllm/models/spark2_5/unicode_ranges.hpp new file mode 100644 index 000000000..2056afced --- /dev/null +++ b/mllm/models/spark2_5/unicode_ranges.hpp @@ -0,0 +1,349 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. +// Generated by examples/spark2_5/generate_unicode_ranges.py; regex 2026.1.15. +#pragma once +#include +#include +namespace mllm::models::spark2_5 { +struct UnicodeRange { + uint32_t first, last; + uint8_t flags; +}; +inline constexpr UnicodeRange unicode_ranges[] = { + {0x9, 0xD, 64}, {0x20, 0x20, 64}, {0x21, 0x23, 4}, {0x24, 0x24, 8}, {0x25, 0x2A, 4}, + {0x2B, 0x2B, 8}, {0x2C, 0x2F, 4}, {0x30, 0x39, 48}, {0x3A, 0x3B, 4}, {0x3C, 0x3E, 8}, + {0x3F, 0x40, 4}, {0x41, 0x5A, 1}, {0x5B, 0x5D, 4}, {0x5E, 0x5E, 8}, {0x5F, 0x5F, 4}, + {0x60, 0x60, 8}, {0x61, 0x7A, 1}, {0x7B, 0x7B, 4}, {0x7C, 0x7C, 8}, {0x7D, 0x7D, 4}, + {0x7E, 0x7E, 8}, {0x85, 0x85, 64}, {0xA0, 0xA0, 64}, {0xA1, 0xA1, 4}, {0xA2, 0xA6, 8}, + {0xA7, 0xA7, 4}, {0xA8, 0xA9, 8}, {0xAA, 0xAA, 1}, {0xAB, 0xAB, 4}, {0xAC, 0xAC, 8}, + {0xAE, 0xB1, 8}, {0xB2, 0xB3, 16}, {0xB4, 0xB4, 8}, {0xB5, 0xB5, 1}, {0xB6, 0xB7, 4}, + {0xB8, 0xB8, 8}, {0xB9, 0xB9, 16}, {0xBA, 0xBA, 1}, {0xBB, 0xBB, 4}, {0xBC, 0xBE, 16}, + {0xBF, 0xBF, 4}, {0xC0, 0xD6, 1}, {0xD7, 0xD7, 8}, {0xD8, 0xF6, 1}, {0xF7, 0xF7, 8}, + {0xF8, 0x2C1, 1}, {0x2C2, 0x2C5, 8}, {0x2C6, 0x2D1, 1}, {0x2D2, 0x2DF, 8}, {0x2E0, 0x2E4, 1}, + {0x2E5, 0x2EB, 8}, {0x2EC, 0x2EC, 1}, {0x2ED, 0x2ED, 8}, {0x2EE, 0x2EE, 1}, {0x2EF, 0x2FF, 8}, + {0x300, 0x36F, 2}, {0x370, 0x374, 1}, {0x375, 0x375, 8}, {0x376, 0x377, 1}, {0x37A, 0x37D, 1}, + {0x37E, 0x37E, 4}, {0x37F, 0x37F, 1}, {0x384, 0x385, 8}, {0x386, 0x386, 1}, {0x387, 0x387, 4}, + {0x388, 0x38A, 1}, {0x38C, 0x38C, 1}, {0x38E, 0x3A1, 1}, {0x3A3, 0x3F5, 1}, {0x3F6, 0x3F6, 8}, + {0x3F7, 0x481, 1}, {0x482, 0x482, 8}, {0x483, 0x489, 2}, {0x48A, 0x52F, 1}, {0x531, 0x556, 1}, + {0x559, 0x559, 1}, {0x55A, 0x55F, 4}, {0x560, 0x588, 1}, {0x589, 0x58A, 4}, {0x58D, 0x58F, 8}, + {0x591, 0x5BD, 2}, {0x5BE, 0x5BE, 4}, {0x5BF, 0x5BF, 2}, {0x5C0, 0x5C0, 4}, {0x5C1, 0x5C2, 2}, + {0x5C3, 0x5C3, 4}, {0x5C4, 0x5C5, 2}, {0x5C6, 0x5C6, 4}, {0x5C7, 0x5C7, 2}, {0x5D0, 0x5EA, 1}, + {0x5EF, 0x5F2, 1}, {0x5F3, 0x5F4, 4}, {0x606, 0x608, 8}, {0x609, 0x60A, 4}, {0x60B, 0x60B, 8}, + {0x60C, 0x60D, 4}, {0x60E, 0x60F, 8}, {0x610, 0x61A, 2}, {0x61B, 0x61B, 4}, {0x61D, 0x61F, 4}, + {0x620, 0x64A, 1}, {0x64B, 0x65F, 2}, {0x660, 0x669, 48}, {0x66A, 0x66D, 4}, {0x66E, 0x66F, 1}, + {0x670, 0x670, 2}, {0x671, 0x6D3, 1}, {0x6D4, 0x6D4, 4}, {0x6D5, 0x6D5, 1}, {0x6D6, 0x6DC, 2}, + {0x6DE, 0x6DE, 8}, {0x6DF, 0x6E4, 2}, {0x6E5, 0x6E6, 1}, {0x6E7, 0x6E8, 2}, {0x6E9, 0x6E9, 8}, + {0x6EA, 0x6ED, 2}, {0x6EE, 0x6EF, 1}, {0x6F0, 0x6F9, 48}, {0x6FA, 0x6FC, 1}, {0x6FD, 0x6FE, 8}, + {0x6FF, 0x6FF, 1}, {0x700, 0x70D, 4}, {0x710, 0x710, 1}, {0x711, 0x711, 2}, {0x712, 0x72F, 1}, + {0x730, 0x74A, 2}, {0x74D, 0x7A5, 1}, {0x7A6, 0x7B0, 2}, {0x7B1, 0x7B1, 1}, {0x7C0, 0x7C9, 48}, + {0x7CA, 0x7EA, 1}, {0x7EB, 0x7F3, 2}, {0x7F4, 0x7F5, 1}, {0x7F6, 0x7F6, 8}, {0x7F7, 0x7F9, 4}, + {0x7FA, 0x7FA, 1}, {0x7FD, 0x7FD, 2}, {0x7FE, 0x7FF, 8}, {0x800, 0x815, 1}, {0x816, 0x819, 2}, + {0x81A, 0x81A, 1}, {0x81B, 0x823, 2}, {0x824, 0x824, 1}, {0x825, 0x827, 2}, {0x828, 0x828, 1}, + {0x829, 0x82D, 2}, {0x830, 0x83E, 4}, {0x840, 0x858, 1}, {0x859, 0x85B, 2}, {0x85E, 0x85E, 4}, + {0x860, 0x86A, 1}, {0x870, 0x887, 1}, {0x888, 0x888, 8}, {0x889, 0x88F, 1}, {0x897, 0x89F, 2}, + {0x8A0, 0x8C9, 1}, {0x8CA, 0x8E1, 2}, {0x8E3, 0x903, 2}, {0x904, 0x939, 1}, {0x93A, 0x93C, 2}, + {0x93D, 0x93D, 1}, {0x93E, 0x94F, 2}, {0x950, 0x950, 1}, {0x951, 0x957, 2}, {0x958, 0x961, 1}, + {0x962, 0x963, 2}, {0x964, 0x965, 4}, {0x966, 0x96F, 48}, {0x970, 0x970, 4}, {0x971, 0x980, 1}, + {0x981, 0x983, 2}, {0x985, 0x98C, 1}, {0x98F, 0x990, 1}, {0x993, 0x9A8, 1}, {0x9AA, 0x9B0, 1}, + {0x9B2, 0x9B2, 1}, {0x9B6, 0x9B9, 1}, {0x9BC, 0x9BC, 2}, {0x9BD, 0x9BD, 1}, {0x9BE, 0x9C4, 2}, + {0x9C7, 0x9C8, 2}, {0x9CB, 0x9CD, 2}, {0x9CE, 0x9CE, 1}, {0x9D7, 0x9D7, 2}, {0x9DC, 0x9DD, 1}, + {0x9DF, 0x9E1, 1}, {0x9E2, 0x9E3, 2}, {0x9E6, 0x9EF, 48}, {0x9F0, 0x9F1, 1}, {0x9F2, 0x9F3, 8}, + {0x9F4, 0x9F9, 16}, {0x9FA, 0x9FB, 8}, {0x9FC, 0x9FC, 1}, {0x9FD, 0x9FD, 4}, {0x9FE, 0x9FE, 2}, + {0xA01, 0xA03, 2}, {0xA05, 0xA0A, 1}, {0xA0F, 0xA10, 1}, {0xA13, 0xA28, 1}, {0xA2A, 0xA30, 1}, + {0xA32, 0xA33, 1}, {0xA35, 0xA36, 1}, {0xA38, 0xA39, 1}, {0xA3C, 0xA3C, 2}, {0xA3E, 0xA42, 2}, + {0xA47, 0xA48, 2}, {0xA4B, 0xA4D, 2}, {0xA51, 0xA51, 2}, {0xA59, 0xA5C, 1}, {0xA5E, 0xA5E, 1}, + {0xA66, 0xA6F, 48}, {0xA70, 0xA71, 2}, {0xA72, 0xA74, 1}, {0xA75, 0xA75, 2}, {0xA76, 0xA76, 4}, + {0xA81, 0xA83, 2}, {0xA85, 0xA8D, 1}, {0xA8F, 0xA91, 1}, {0xA93, 0xAA8, 1}, {0xAAA, 0xAB0, 1}, + {0xAB2, 0xAB3, 1}, {0xAB5, 0xAB9, 1}, {0xABC, 0xABC, 2}, {0xABD, 0xABD, 1}, {0xABE, 0xAC5, 2}, + {0xAC7, 0xAC9, 2}, {0xACB, 0xACD, 2}, {0xAD0, 0xAD0, 1}, {0xAE0, 0xAE1, 1}, {0xAE2, 0xAE3, 2}, + {0xAE6, 0xAEF, 48}, {0xAF0, 0xAF0, 4}, {0xAF1, 0xAF1, 8}, {0xAF9, 0xAF9, 1}, {0xAFA, 0xAFF, 2}, + {0xB01, 0xB03, 2}, {0xB05, 0xB0C, 1}, {0xB0F, 0xB10, 1}, {0xB13, 0xB28, 1}, {0xB2A, 0xB30, 1}, + {0xB32, 0xB33, 1}, {0xB35, 0xB39, 1}, {0xB3C, 0xB3C, 2}, {0xB3D, 0xB3D, 1}, {0xB3E, 0xB44, 2}, + {0xB47, 0xB48, 2}, {0xB4B, 0xB4D, 2}, {0xB55, 0xB57, 2}, {0xB5C, 0xB5D, 1}, {0xB5F, 0xB61, 1}, + {0xB62, 0xB63, 2}, {0xB66, 0xB6F, 48}, {0xB70, 0xB70, 8}, {0xB71, 0xB71, 1}, {0xB72, 0xB77, 16}, + {0xB82, 0xB82, 2}, {0xB83, 0xB83, 1}, {0xB85, 0xB8A, 1}, {0xB8E, 0xB90, 1}, {0xB92, 0xB95, 1}, + {0xB99, 0xB9A, 1}, {0xB9C, 0xB9C, 1}, {0xB9E, 0xB9F, 1}, {0xBA3, 0xBA4, 1}, {0xBA8, 0xBAA, 1}, + {0xBAE, 0xBB9, 1}, {0xBBE, 0xBC2, 2}, {0xBC6, 0xBC8, 2}, {0xBCA, 0xBCD, 2}, {0xBD0, 0xBD0, 1}, + {0xBD7, 0xBD7, 2}, {0xBE6, 0xBEF, 48}, {0xBF0, 0xBF2, 16}, {0xBF3, 0xBFA, 8}, {0xC00, 0xC04, 2}, + {0xC05, 0xC0C, 1}, {0xC0E, 0xC10, 1}, {0xC12, 0xC28, 1}, {0xC2A, 0xC39, 1}, {0xC3C, 0xC3C, 2}, + {0xC3D, 0xC3D, 1}, {0xC3E, 0xC44, 2}, {0xC46, 0xC48, 2}, {0xC4A, 0xC4D, 2}, {0xC55, 0xC56, 2}, + {0xC58, 0xC5A, 1}, {0xC5C, 0xC5D, 1}, {0xC60, 0xC61, 1}, {0xC62, 0xC63, 2}, {0xC66, 0xC6F, 48}, + {0xC77, 0xC77, 4}, {0xC78, 0xC7E, 16}, {0xC7F, 0xC7F, 8}, {0xC80, 0xC80, 1}, {0xC81, 0xC83, 2}, + {0xC84, 0xC84, 4}, {0xC85, 0xC8C, 1}, {0xC8E, 0xC90, 1}, {0xC92, 0xCA8, 1}, {0xCAA, 0xCB3, 1}, + {0xCB5, 0xCB9, 1}, {0xCBC, 0xCBC, 2}, {0xCBD, 0xCBD, 1}, {0xCBE, 0xCC4, 2}, {0xCC6, 0xCC8, 2}, + {0xCCA, 0xCCD, 2}, {0xCD5, 0xCD6, 2}, {0xCDC, 0xCDE, 1}, {0xCE0, 0xCE1, 1}, {0xCE2, 0xCE3, 2}, + {0xCE6, 0xCEF, 48}, {0xCF1, 0xCF2, 1}, {0xCF3, 0xCF3, 2}, {0xD00, 0xD03, 2}, {0xD04, 0xD0C, 1}, + {0xD0E, 0xD10, 1}, {0xD12, 0xD3A, 1}, {0xD3B, 0xD3C, 2}, {0xD3D, 0xD3D, 1}, {0xD3E, 0xD44, 2}, + {0xD46, 0xD48, 2}, {0xD4A, 0xD4D, 2}, {0xD4E, 0xD4E, 1}, {0xD4F, 0xD4F, 8}, {0xD54, 0xD56, 1}, + {0xD57, 0xD57, 2}, {0xD58, 0xD5E, 16}, {0xD5F, 0xD61, 1}, {0xD62, 0xD63, 2}, {0xD66, 0xD6F, 48}, + {0xD70, 0xD78, 16}, {0xD79, 0xD79, 8}, {0xD7A, 0xD7F, 1}, {0xD81, 0xD83, 2}, {0xD85, 0xD96, 1}, + {0xD9A, 0xDB1, 1}, {0xDB3, 0xDBB, 1}, {0xDBD, 0xDBD, 1}, {0xDC0, 0xDC6, 1}, {0xDCA, 0xDCA, 2}, + {0xDCF, 0xDD4, 2}, {0xDD6, 0xDD6, 2}, {0xDD8, 0xDDF, 2}, {0xDE6, 0xDEF, 48}, {0xDF2, 0xDF3, 2}, + {0xDF4, 0xDF4, 4}, {0xE01, 0xE30, 1}, {0xE31, 0xE31, 2}, {0xE32, 0xE33, 1}, {0xE34, 0xE3A, 2}, + {0xE3F, 0xE3F, 8}, {0xE40, 0xE46, 1}, {0xE47, 0xE4E, 2}, {0xE4F, 0xE4F, 4}, {0xE50, 0xE59, 48}, + {0xE5A, 0xE5B, 4}, {0xE81, 0xE82, 1}, {0xE84, 0xE84, 1}, {0xE86, 0xE8A, 1}, {0xE8C, 0xEA3, 1}, + {0xEA5, 0xEA5, 1}, {0xEA7, 0xEB0, 1}, {0xEB1, 0xEB1, 2}, {0xEB2, 0xEB3, 1}, {0xEB4, 0xEBC, 2}, + {0xEBD, 0xEBD, 1}, {0xEC0, 0xEC4, 1}, {0xEC6, 0xEC6, 1}, {0xEC8, 0xECE, 2}, {0xED0, 0xED9, 48}, + {0xEDC, 0xEDF, 1}, {0xF00, 0xF00, 1}, {0xF01, 0xF03, 8}, {0xF04, 0xF12, 4}, {0xF13, 0xF13, 8}, + {0xF14, 0xF14, 4}, {0xF15, 0xF17, 8}, {0xF18, 0xF19, 2}, {0xF1A, 0xF1F, 8}, {0xF20, 0xF29, 48}, + {0xF2A, 0xF33, 16}, {0xF34, 0xF34, 8}, {0xF35, 0xF35, 2}, {0xF36, 0xF36, 8}, {0xF37, 0xF37, 2}, + {0xF38, 0xF38, 8}, {0xF39, 0xF39, 2}, {0xF3A, 0xF3D, 4}, {0xF3E, 0xF3F, 2}, {0xF40, 0xF47, 1}, + {0xF49, 0xF6C, 1}, {0xF71, 0xF84, 2}, {0xF85, 0xF85, 4}, {0xF86, 0xF87, 2}, {0xF88, 0xF8C, 1}, + {0xF8D, 0xF97, 2}, {0xF99, 0xFBC, 2}, {0xFBE, 0xFC5, 8}, {0xFC6, 0xFC6, 2}, {0xFC7, 0xFCC, 8}, + {0xFCE, 0xFCF, 8}, {0xFD0, 0xFD4, 4}, {0xFD5, 0xFD8, 8}, {0xFD9, 0xFDA, 4}, {0x1000, 0x102A, 1}, + {0x102B, 0x103E, 2}, {0x103F, 0x103F, 1}, {0x1040, 0x1049, 48}, {0x104A, 0x104F, 4}, {0x1050, 0x1055, 1}, + {0x1056, 0x1059, 2}, {0x105A, 0x105D, 1}, {0x105E, 0x1060, 2}, {0x1061, 0x1061, 1}, {0x1062, 0x1064, 2}, + {0x1065, 0x1066, 1}, {0x1067, 0x106D, 2}, {0x106E, 0x1070, 1}, {0x1071, 0x1074, 2}, {0x1075, 0x1081, 1}, + {0x1082, 0x108D, 2}, {0x108E, 0x108E, 1}, {0x108F, 0x108F, 2}, {0x1090, 0x1099, 48}, {0x109A, 0x109D, 2}, + {0x109E, 0x109F, 8}, {0x10A0, 0x10C5, 1}, {0x10C7, 0x10C7, 1}, {0x10CD, 0x10CD, 1}, {0x10D0, 0x10FA, 1}, + {0x10FB, 0x10FB, 4}, {0x10FC, 0x1248, 1}, {0x124A, 0x124D, 1}, {0x1250, 0x1256, 1}, {0x1258, 0x1258, 1}, + {0x125A, 0x125D, 1}, {0x1260, 0x1288, 1}, {0x128A, 0x128D, 1}, {0x1290, 0x12B0, 1}, {0x12B2, 0x12B5, 1}, + {0x12B8, 0x12BE, 1}, {0x12C0, 0x12C0, 1}, {0x12C2, 0x12C5, 1}, {0x12C8, 0x12D6, 1}, {0x12D8, 0x1310, 1}, + {0x1312, 0x1315, 1}, {0x1318, 0x135A, 1}, {0x135D, 0x135F, 2}, {0x1360, 0x1368, 4}, {0x1369, 0x137C, 16}, + {0x1380, 0x138F, 1}, {0x1390, 0x1399, 8}, {0x13A0, 0x13F5, 1}, {0x13F8, 0x13FD, 1}, {0x1400, 0x1400, 4}, + {0x1401, 0x166C, 1}, {0x166D, 0x166D, 8}, {0x166E, 0x166E, 4}, {0x166F, 0x167F, 1}, {0x1680, 0x1680, 64}, + {0x1681, 0x169A, 1}, {0x169B, 0x169C, 4}, {0x16A0, 0x16EA, 1}, {0x16EB, 0x16ED, 4}, {0x16EE, 0x16F0, 16}, + {0x16F1, 0x16F8, 1}, {0x1700, 0x1711, 1}, {0x1712, 0x1715, 2}, {0x171F, 0x1731, 1}, {0x1732, 0x1734, 2}, + {0x1735, 0x1736, 4}, {0x1740, 0x1751, 1}, {0x1752, 0x1753, 2}, {0x1760, 0x176C, 1}, {0x176E, 0x1770, 1}, + {0x1772, 0x1773, 2}, {0x1780, 0x17B3, 1}, {0x17B4, 0x17D3, 2}, {0x17D4, 0x17D6, 4}, {0x17D7, 0x17D7, 1}, + {0x17D8, 0x17DA, 4}, {0x17DB, 0x17DB, 8}, {0x17DC, 0x17DC, 1}, {0x17DD, 0x17DD, 2}, {0x17E0, 0x17E9, 48}, + {0x17F0, 0x17F9, 16}, {0x1800, 0x180A, 4}, {0x180B, 0x180D, 2}, {0x180F, 0x180F, 2}, {0x1810, 0x1819, 48}, + {0x1820, 0x1878, 1}, {0x1880, 0x1884, 1}, {0x1885, 0x1886, 2}, {0x1887, 0x18A8, 1}, {0x18A9, 0x18A9, 2}, + {0x18AA, 0x18AA, 1}, {0x18B0, 0x18F5, 1}, {0x1900, 0x191E, 1}, {0x1920, 0x192B, 2}, {0x1930, 0x193B, 2}, + {0x1940, 0x1940, 8}, {0x1944, 0x1945, 4}, {0x1946, 0x194F, 48}, {0x1950, 0x196D, 1}, {0x1970, 0x1974, 1}, + {0x1980, 0x19AB, 1}, {0x19B0, 0x19C9, 1}, {0x19D0, 0x19D9, 48}, {0x19DA, 0x19DA, 16}, {0x19DE, 0x19FF, 8}, + {0x1A00, 0x1A16, 1}, {0x1A17, 0x1A1B, 2}, {0x1A1E, 0x1A1F, 4}, {0x1A20, 0x1A54, 1}, {0x1A55, 0x1A5E, 2}, + {0x1A60, 0x1A7C, 2}, {0x1A7F, 0x1A7F, 2}, {0x1A80, 0x1A89, 48}, {0x1A90, 0x1A99, 48}, {0x1AA0, 0x1AA6, 4}, + {0x1AA7, 0x1AA7, 1}, {0x1AA8, 0x1AAD, 4}, {0x1AB0, 0x1ADD, 2}, {0x1AE0, 0x1AEB, 2}, {0x1B00, 0x1B04, 2}, + {0x1B05, 0x1B33, 1}, {0x1B34, 0x1B44, 2}, {0x1B45, 0x1B4C, 1}, {0x1B4E, 0x1B4F, 4}, {0x1B50, 0x1B59, 48}, + {0x1B5A, 0x1B60, 4}, {0x1B61, 0x1B6A, 8}, {0x1B6B, 0x1B73, 2}, {0x1B74, 0x1B7C, 8}, {0x1B7D, 0x1B7F, 4}, + {0x1B80, 0x1B82, 2}, {0x1B83, 0x1BA0, 1}, {0x1BA1, 0x1BAD, 2}, {0x1BAE, 0x1BAF, 1}, {0x1BB0, 0x1BB9, 48}, + {0x1BBA, 0x1BE5, 1}, {0x1BE6, 0x1BF3, 2}, {0x1BFC, 0x1BFF, 4}, {0x1C00, 0x1C23, 1}, {0x1C24, 0x1C37, 2}, + {0x1C3B, 0x1C3F, 4}, {0x1C40, 0x1C49, 48}, {0x1C4D, 0x1C4F, 1}, {0x1C50, 0x1C59, 48}, {0x1C5A, 0x1C7D, 1}, + {0x1C7E, 0x1C7F, 4}, {0x1C80, 0x1C8A, 1}, {0x1C90, 0x1CBA, 1}, {0x1CBD, 0x1CBF, 1}, {0x1CC0, 0x1CC7, 4}, + {0x1CD0, 0x1CD2, 2}, {0x1CD3, 0x1CD3, 4}, {0x1CD4, 0x1CE8, 2}, {0x1CE9, 0x1CEC, 1}, {0x1CED, 0x1CED, 2}, + {0x1CEE, 0x1CF3, 1}, {0x1CF4, 0x1CF4, 2}, {0x1CF5, 0x1CF6, 1}, {0x1CF7, 0x1CF9, 2}, {0x1CFA, 0x1CFA, 1}, + {0x1D00, 0x1DBF, 1}, {0x1DC0, 0x1DFF, 2}, {0x1E00, 0x1F15, 1}, {0x1F18, 0x1F1D, 1}, {0x1F20, 0x1F45, 1}, + {0x1F48, 0x1F4D, 1}, {0x1F50, 0x1F57, 1}, {0x1F59, 0x1F59, 1}, {0x1F5B, 0x1F5B, 1}, {0x1F5D, 0x1F5D, 1}, + {0x1F5F, 0x1F7D, 1}, {0x1F80, 0x1FB4, 1}, {0x1FB6, 0x1FBC, 1}, {0x1FBD, 0x1FBD, 8}, {0x1FBE, 0x1FBE, 1}, + {0x1FBF, 0x1FC1, 8}, {0x1FC2, 0x1FC4, 1}, {0x1FC6, 0x1FCC, 1}, {0x1FCD, 0x1FCF, 8}, {0x1FD0, 0x1FD3, 1}, + {0x1FD6, 0x1FDB, 1}, {0x1FDD, 0x1FDF, 8}, {0x1FE0, 0x1FEC, 1}, {0x1FED, 0x1FEF, 8}, {0x1FF2, 0x1FF4, 1}, + {0x1FF6, 0x1FFC, 1}, {0x1FFD, 0x1FFE, 8}, {0x2000, 0x200A, 64}, {0x2010, 0x2027, 4}, {0x2028, 0x2029, 64}, + {0x202F, 0x202F, 64}, {0x2030, 0x2043, 4}, {0x2044, 0x2044, 8}, {0x2045, 0x2051, 4}, {0x2052, 0x2052, 8}, + {0x2053, 0x205E, 4}, {0x205F, 0x205F, 64}, {0x2070, 0x2070, 16}, {0x2071, 0x2071, 1}, {0x2074, 0x2079, 16}, + {0x207A, 0x207C, 8}, {0x207D, 0x207E, 4}, {0x207F, 0x207F, 1}, {0x2080, 0x2089, 16}, {0x208A, 0x208C, 8}, + {0x208D, 0x208E, 4}, {0x2090, 0x209C, 1}, {0x20A0, 0x20C1, 8}, {0x20D0, 0x20F0, 2}, {0x2100, 0x2101, 8}, + {0x2102, 0x2102, 1}, {0x2103, 0x2106, 8}, {0x2107, 0x2107, 1}, {0x2108, 0x2109, 8}, {0x210A, 0x2113, 1}, + {0x2114, 0x2114, 8}, {0x2115, 0x2115, 1}, {0x2116, 0x2118, 8}, {0x2119, 0x211D, 1}, {0x211E, 0x2123, 8}, + {0x2124, 0x2124, 1}, {0x2125, 0x2125, 8}, {0x2126, 0x2126, 1}, {0x2127, 0x2127, 8}, {0x2128, 0x2128, 1}, + {0x2129, 0x2129, 8}, {0x212A, 0x212D, 1}, {0x212E, 0x212E, 8}, {0x212F, 0x2139, 1}, {0x213A, 0x213B, 8}, + {0x213C, 0x213F, 1}, {0x2140, 0x2144, 8}, {0x2145, 0x2149, 1}, {0x214A, 0x214D, 8}, {0x214E, 0x214E, 1}, + {0x214F, 0x214F, 8}, {0x2150, 0x2182, 16}, {0x2183, 0x2184, 1}, {0x2185, 0x2189, 16}, {0x218A, 0x218B, 8}, + {0x2190, 0x2307, 8}, {0x2308, 0x230B, 4}, {0x230C, 0x2328, 8}, {0x2329, 0x232A, 4}, {0x232B, 0x2429, 8}, + {0x2440, 0x244A, 8}, {0x2460, 0x249B, 16}, {0x249C, 0x24E9, 8}, {0x24EA, 0x24FF, 16}, {0x2500, 0x2767, 8}, + {0x2768, 0x2775, 4}, {0x2776, 0x2793, 16}, {0x2794, 0x27C4, 8}, {0x27C5, 0x27C6, 4}, {0x27C7, 0x27E5, 8}, + {0x27E6, 0x27EF, 4}, {0x27F0, 0x2982, 8}, {0x2983, 0x2998, 4}, {0x2999, 0x29D7, 8}, {0x29D8, 0x29DB, 4}, + {0x29DC, 0x29FB, 8}, {0x29FC, 0x29FD, 4}, {0x29FE, 0x2B73, 8}, {0x2B76, 0x2BFF, 8}, {0x2C00, 0x2CE4, 1}, + {0x2CE5, 0x2CEA, 8}, {0x2CEB, 0x2CEE, 1}, {0x2CEF, 0x2CF1, 2}, {0x2CF2, 0x2CF3, 1}, {0x2CF9, 0x2CFC, 4}, + {0x2CFD, 0x2CFD, 16}, {0x2CFE, 0x2CFF, 4}, {0x2D00, 0x2D25, 1}, {0x2D27, 0x2D27, 1}, {0x2D2D, 0x2D2D, 1}, + {0x2D30, 0x2D67, 1}, {0x2D6F, 0x2D6F, 1}, {0x2D70, 0x2D70, 4}, {0x2D7F, 0x2D7F, 2}, {0x2D80, 0x2D96, 1}, + {0x2DA0, 0x2DA6, 1}, {0x2DA8, 0x2DAE, 1}, {0x2DB0, 0x2DB6, 1}, {0x2DB8, 0x2DBE, 1}, {0x2DC0, 0x2DC6, 1}, + {0x2DC8, 0x2DCE, 1}, {0x2DD0, 0x2DD6, 1}, {0x2DD8, 0x2DDE, 1}, {0x2DE0, 0x2DFF, 2}, {0x2E00, 0x2E2E, 4}, + {0x2E2F, 0x2E2F, 1}, {0x2E30, 0x2E4F, 4}, {0x2E50, 0x2E51, 8}, {0x2E52, 0x2E5D, 4}, {0x2E80, 0x2E99, 8}, + {0x2E9B, 0x2EF3, 8}, {0x2F00, 0x2FD5, 8}, {0x2FF0, 0x2FFF, 8}, {0x3000, 0x3000, 64}, {0x3001, 0x3003, 4}, + {0x3004, 0x3004, 8}, {0x3005, 0x3006, 1}, {0x3007, 0x3007, 16}, {0x3008, 0x3011, 4}, {0x3012, 0x3013, 8}, + {0x3014, 0x301F, 4}, {0x3020, 0x3020, 8}, {0x3021, 0x3029, 16}, {0x302A, 0x302F, 2}, {0x3030, 0x3030, 4}, + {0x3031, 0x3035, 1}, {0x3036, 0x3037, 8}, {0x3038, 0x303A, 16}, {0x303B, 0x303C, 1}, {0x303D, 0x303D, 4}, + {0x303E, 0x303F, 8}, {0x3041, 0x3096, 1}, {0x3099, 0x309A, 2}, {0x309B, 0x309C, 8}, {0x309D, 0x309F, 1}, + {0x30A0, 0x30A0, 4}, {0x30A1, 0x30FA, 1}, {0x30FB, 0x30FB, 4}, {0x30FC, 0x30FF, 1}, {0x3105, 0x312F, 1}, + {0x3131, 0x318E, 1}, {0x3190, 0x3191, 8}, {0x3192, 0x3195, 16}, {0x3196, 0x319F, 8}, {0x31A0, 0x31BF, 1}, + {0x31C0, 0x31E5, 8}, {0x31EF, 0x31EF, 8}, {0x31F0, 0x31FF, 1}, {0x3200, 0x321E, 8}, {0x3220, 0x3229, 16}, + {0x322A, 0x3247, 8}, {0x3248, 0x324F, 16}, {0x3250, 0x3250, 8}, {0x3251, 0x325F, 16}, {0x3260, 0x327F, 8}, + {0x3280, 0x3289, 16}, {0x328A, 0x32B0, 8}, {0x32B1, 0x32BF, 16}, {0x32C0, 0x33FF, 8}, {0x3400, 0x4DBF, 1}, + {0x4DC0, 0x4DFF, 8}, {0x4E00, 0xA48C, 1}, {0xA490, 0xA4C6, 8}, {0xA4D0, 0xA4FD, 1}, {0xA4FE, 0xA4FF, 4}, + {0xA500, 0xA60C, 1}, {0xA60D, 0xA60F, 4}, {0xA610, 0xA61F, 1}, {0xA620, 0xA629, 48}, {0xA62A, 0xA62B, 1}, + {0xA640, 0xA66E, 1}, {0xA66F, 0xA672, 2}, {0xA673, 0xA673, 4}, {0xA674, 0xA67D, 2}, {0xA67E, 0xA67E, 4}, + {0xA67F, 0xA69D, 1}, {0xA69E, 0xA69F, 2}, {0xA6A0, 0xA6E5, 1}, {0xA6E6, 0xA6EF, 16}, {0xA6F0, 0xA6F1, 2}, + {0xA6F2, 0xA6F7, 4}, {0xA700, 0xA716, 8}, {0xA717, 0xA71F, 1}, {0xA720, 0xA721, 8}, {0xA722, 0xA788, 1}, + {0xA789, 0xA78A, 8}, {0xA78B, 0xA7DC, 1}, {0xA7F1, 0xA801, 1}, {0xA802, 0xA802, 2}, {0xA803, 0xA805, 1}, + {0xA806, 0xA806, 2}, {0xA807, 0xA80A, 1}, {0xA80B, 0xA80B, 2}, {0xA80C, 0xA822, 1}, {0xA823, 0xA827, 2}, + {0xA828, 0xA82B, 8}, {0xA82C, 0xA82C, 2}, {0xA830, 0xA835, 16}, {0xA836, 0xA839, 8}, {0xA840, 0xA873, 1}, + {0xA874, 0xA877, 4}, {0xA880, 0xA881, 2}, {0xA882, 0xA8B3, 1}, {0xA8B4, 0xA8C5, 2}, {0xA8CE, 0xA8CF, 4}, + {0xA8D0, 0xA8D9, 48}, {0xA8E0, 0xA8F1, 2}, {0xA8F2, 0xA8F7, 1}, {0xA8F8, 0xA8FA, 4}, {0xA8FB, 0xA8FB, 1}, + {0xA8FC, 0xA8FC, 4}, {0xA8FD, 0xA8FE, 1}, {0xA8FF, 0xA8FF, 2}, {0xA900, 0xA909, 48}, {0xA90A, 0xA925, 1}, + {0xA926, 0xA92D, 2}, {0xA92E, 0xA92F, 4}, {0xA930, 0xA946, 1}, {0xA947, 0xA953, 2}, {0xA95F, 0xA95F, 4}, + {0xA960, 0xA97C, 1}, {0xA980, 0xA983, 2}, {0xA984, 0xA9B2, 1}, {0xA9B3, 0xA9C0, 2}, {0xA9C1, 0xA9CD, 4}, + {0xA9CF, 0xA9CF, 1}, {0xA9D0, 0xA9D9, 48}, {0xA9DE, 0xA9DF, 4}, {0xA9E0, 0xA9E4, 1}, {0xA9E5, 0xA9E5, 2}, + {0xA9E6, 0xA9EF, 1}, {0xA9F0, 0xA9F9, 48}, {0xA9FA, 0xA9FE, 1}, {0xAA00, 0xAA28, 1}, {0xAA29, 0xAA36, 2}, + {0xAA40, 0xAA42, 1}, {0xAA43, 0xAA43, 2}, {0xAA44, 0xAA4B, 1}, {0xAA4C, 0xAA4D, 2}, {0xAA50, 0xAA59, 48}, + {0xAA5C, 0xAA5F, 4}, {0xAA60, 0xAA76, 1}, {0xAA77, 0xAA79, 8}, {0xAA7A, 0xAA7A, 1}, {0xAA7B, 0xAA7D, 2}, + {0xAA7E, 0xAAAF, 1}, {0xAAB0, 0xAAB0, 2}, {0xAAB1, 0xAAB1, 1}, {0xAAB2, 0xAAB4, 2}, {0xAAB5, 0xAAB6, 1}, + {0xAAB7, 0xAAB8, 2}, {0xAAB9, 0xAABD, 1}, {0xAABE, 0xAABF, 2}, {0xAAC0, 0xAAC0, 1}, {0xAAC1, 0xAAC1, 2}, + {0xAAC2, 0xAAC2, 1}, {0xAADB, 0xAADD, 1}, {0xAADE, 0xAADF, 4}, {0xAAE0, 0xAAEA, 1}, {0xAAEB, 0xAAEF, 2}, + {0xAAF0, 0xAAF1, 4}, {0xAAF2, 0xAAF4, 1}, {0xAAF5, 0xAAF6, 2}, {0xAB01, 0xAB06, 1}, {0xAB09, 0xAB0E, 1}, + {0xAB11, 0xAB16, 1}, {0xAB20, 0xAB26, 1}, {0xAB28, 0xAB2E, 1}, {0xAB30, 0xAB5A, 1}, {0xAB5B, 0xAB5B, 8}, + {0xAB5C, 0xAB69, 1}, {0xAB6A, 0xAB6B, 8}, {0xAB70, 0xABE2, 1}, {0xABE3, 0xABEA, 2}, {0xABEB, 0xABEB, 4}, + {0xABEC, 0xABED, 2}, {0xABF0, 0xABF9, 48}, {0xAC00, 0xD7A3, 1}, {0xD7B0, 0xD7C6, 1}, {0xD7CB, 0xD7FB, 1}, + {0xF900, 0xFA6D, 1}, {0xFA70, 0xFAD9, 1}, {0xFB00, 0xFB06, 1}, {0xFB13, 0xFB17, 1}, {0xFB1D, 0xFB1D, 1}, + {0xFB1E, 0xFB1E, 2}, {0xFB1F, 0xFB28, 1}, {0xFB29, 0xFB29, 8}, {0xFB2A, 0xFB36, 1}, {0xFB38, 0xFB3C, 1}, + {0xFB3E, 0xFB3E, 1}, {0xFB40, 0xFB41, 1}, {0xFB43, 0xFB44, 1}, {0xFB46, 0xFBB1, 1}, {0xFBB2, 0xFBD2, 8}, + {0xFBD3, 0xFD3D, 1}, {0xFD3E, 0xFD3F, 4}, {0xFD40, 0xFD4F, 8}, {0xFD50, 0xFD8F, 1}, {0xFD90, 0xFD91, 8}, + {0xFD92, 0xFDC7, 1}, {0xFDC8, 0xFDCF, 8}, {0xFDF0, 0xFDFB, 1}, {0xFDFC, 0xFDFF, 8}, {0xFE00, 0xFE0F, 2}, + {0xFE10, 0xFE19, 4}, {0xFE20, 0xFE2F, 2}, {0xFE30, 0xFE52, 4}, {0xFE54, 0xFE61, 4}, {0xFE62, 0xFE62, 8}, + {0xFE63, 0xFE63, 4}, {0xFE64, 0xFE66, 8}, {0xFE68, 0xFE68, 4}, {0xFE69, 0xFE69, 8}, {0xFE6A, 0xFE6B, 4}, + {0xFE70, 0xFE74, 1}, {0xFE76, 0xFEFC, 1}, {0xFF01, 0xFF03, 4}, {0xFF04, 0xFF04, 8}, {0xFF05, 0xFF0A, 4}, + {0xFF0B, 0xFF0B, 8}, {0xFF0C, 0xFF0F, 4}, {0xFF10, 0xFF19, 48}, {0xFF1A, 0xFF1B, 4}, {0xFF1C, 0xFF1E, 8}, + {0xFF1F, 0xFF20, 4}, {0xFF21, 0xFF3A, 1}, {0xFF3B, 0xFF3D, 4}, {0xFF3E, 0xFF3E, 8}, {0xFF3F, 0xFF3F, 4}, + {0xFF40, 0xFF40, 8}, {0xFF41, 0xFF5A, 1}, {0xFF5B, 0xFF5B, 4}, {0xFF5C, 0xFF5C, 8}, {0xFF5D, 0xFF5D, 4}, + {0xFF5E, 0xFF5E, 8}, {0xFF5F, 0xFF65, 4}, {0xFF66, 0xFFBE, 1}, {0xFFC2, 0xFFC7, 1}, {0xFFCA, 0xFFCF, 1}, + {0xFFD2, 0xFFD7, 1}, {0xFFDA, 0xFFDC, 1}, {0xFFE0, 0xFFE6, 8}, {0xFFE8, 0xFFEE, 8}, {0xFFFC, 0xFFFD, 8}, + {0x10000, 0x1000B, 1}, {0x1000D, 0x10026, 1}, {0x10028, 0x1003A, 1}, {0x1003C, 0x1003D, 1}, {0x1003F, 0x1004D, 1}, + {0x10050, 0x1005D, 1}, {0x10080, 0x100FA, 1}, {0x10100, 0x10102, 4}, {0x10107, 0x10133, 16}, {0x10137, 0x1013F, 8}, + {0x10140, 0x10178, 16}, {0x10179, 0x10189, 8}, {0x1018A, 0x1018B, 16}, {0x1018C, 0x1018E, 8}, {0x10190, 0x1019C, 8}, + {0x101A0, 0x101A0, 8}, {0x101D0, 0x101FC, 8}, {0x101FD, 0x101FD, 2}, {0x10280, 0x1029C, 1}, {0x102A0, 0x102D0, 1}, + {0x102E0, 0x102E0, 2}, {0x102E1, 0x102FB, 16}, {0x10300, 0x1031F, 1}, {0x10320, 0x10323, 16}, {0x1032D, 0x10340, 1}, + {0x10341, 0x10341, 16}, {0x10342, 0x10349, 1}, {0x1034A, 0x1034A, 16}, {0x10350, 0x10375, 1}, {0x10376, 0x1037A, 2}, + {0x10380, 0x1039D, 1}, {0x1039F, 0x1039F, 4}, {0x103A0, 0x103C3, 1}, {0x103C8, 0x103CF, 1}, {0x103D0, 0x103D0, 4}, + {0x103D1, 0x103D5, 16}, {0x10400, 0x1049D, 1}, {0x104A0, 0x104A9, 48}, {0x104B0, 0x104D3, 1}, {0x104D8, 0x104FB, 1}, + {0x10500, 0x10527, 1}, {0x10530, 0x10563, 1}, {0x1056F, 0x1056F, 4}, {0x10570, 0x1057A, 1}, {0x1057C, 0x1058A, 1}, + {0x1058C, 0x10592, 1}, {0x10594, 0x10595, 1}, {0x10597, 0x105A1, 1}, {0x105A3, 0x105B1, 1}, {0x105B3, 0x105B9, 1}, + {0x105BB, 0x105BC, 1}, {0x105C0, 0x105F3, 1}, {0x10600, 0x10736, 1}, {0x10740, 0x10755, 1}, {0x10760, 0x10767, 1}, + {0x10780, 0x10785, 1}, {0x10787, 0x107B0, 1}, {0x107B2, 0x107BA, 1}, {0x10800, 0x10805, 1}, {0x10808, 0x10808, 1}, + {0x1080A, 0x10835, 1}, {0x10837, 0x10838, 1}, {0x1083C, 0x1083C, 1}, {0x1083F, 0x10855, 1}, {0x10857, 0x10857, 4}, + {0x10858, 0x1085F, 16}, {0x10860, 0x10876, 1}, {0x10877, 0x10878, 8}, {0x10879, 0x1087F, 16}, {0x10880, 0x1089E, 1}, + {0x108A7, 0x108AF, 16}, {0x108E0, 0x108F2, 1}, {0x108F4, 0x108F5, 1}, {0x108FB, 0x108FF, 16}, {0x10900, 0x10915, 1}, + {0x10916, 0x1091B, 16}, {0x1091F, 0x1091F, 4}, {0x10920, 0x10939, 1}, {0x1093F, 0x1093F, 4}, {0x10940, 0x10959, 1}, + {0x10980, 0x109B7, 1}, {0x109BC, 0x109BD, 16}, {0x109BE, 0x109BF, 1}, {0x109C0, 0x109CF, 16}, {0x109D2, 0x109FF, 16}, + {0x10A00, 0x10A00, 1}, {0x10A01, 0x10A03, 2}, {0x10A05, 0x10A06, 2}, {0x10A0C, 0x10A0F, 2}, {0x10A10, 0x10A13, 1}, + {0x10A15, 0x10A17, 1}, {0x10A19, 0x10A35, 1}, {0x10A38, 0x10A3A, 2}, {0x10A3F, 0x10A3F, 2}, {0x10A40, 0x10A48, 16}, + {0x10A50, 0x10A58, 4}, {0x10A60, 0x10A7C, 1}, {0x10A7D, 0x10A7E, 16}, {0x10A7F, 0x10A7F, 4}, {0x10A80, 0x10A9C, 1}, + {0x10A9D, 0x10A9F, 16}, {0x10AC0, 0x10AC7, 1}, {0x10AC8, 0x10AC8, 8}, {0x10AC9, 0x10AE4, 1}, {0x10AE5, 0x10AE6, 2}, + {0x10AEB, 0x10AEF, 16}, {0x10AF0, 0x10AF6, 4}, {0x10B00, 0x10B35, 1}, {0x10B39, 0x10B3F, 4}, {0x10B40, 0x10B55, 1}, + {0x10B58, 0x10B5F, 16}, {0x10B60, 0x10B72, 1}, {0x10B78, 0x10B7F, 16}, {0x10B80, 0x10B91, 1}, {0x10B99, 0x10B9C, 4}, + {0x10BA9, 0x10BAF, 16}, {0x10C00, 0x10C48, 1}, {0x10C80, 0x10CB2, 1}, {0x10CC0, 0x10CF2, 1}, {0x10CFA, 0x10CFF, 16}, + {0x10D00, 0x10D23, 1}, {0x10D24, 0x10D27, 2}, {0x10D30, 0x10D39, 48}, {0x10D40, 0x10D49, 48}, {0x10D4A, 0x10D65, 1}, + {0x10D69, 0x10D6D, 2}, {0x10D6E, 0x10D6E, 4}, {0x10D6F, 0x10D85, 1}, {0x10D8E, 0x10D8F, 8}, {0x10E60, 0x10E7E, 16}, + {0x10E80, 0x10EA9, 1}, {0x10EAB, 0x10EAC, 2}, {0x10EAD, 0x10EAD, 4}, {0x10EB0, 0x10EB1, 1}, {0x10EC2, 0x10EC7, 1}, + {0x10ED0, 0x10ED0, 4}, {0x10ED1, 0x10ED8, 8}, {0x10EFA, 0x10EFF, 2}, {0x10F00, 0x10F1C, 1}, {0x10F1D, 0x10F26, 16}, + {0x10F27, 0x10F27, 1}, {0x10F30, 0x10F45, 1}, {0x10F46, 0x10F50, 2}, {0x10F51, 0x10F54, 16}, {0x10F55, 0x10F59, 4}, + {0x10F70, 0x10F81, 1}, {0x10F82, 0x10F85, 2}, {0x10F86, 0x10F89, 4}, {0x10FB0, 0x10FC4, 1}, {0x10FC5, 0x10FCB, 16}, + {0x10FE0, 0x10FF6, 1}, {0x11000, 0x11002, 2}, {0x11003, 0x11037, 1}, {0x11038, 0x11046, 2}, {0x11047, 0x1104D, 4}, + {0x11052, 0x11065, 16}, {0x11066, 0x1106F, 48}, {0x11070, 0x11070, 2}, {0x11071, 0x11072, 1}, {0x11073, 0x11074, 2}, + {0x11075, 0x11075, 1}, {0x1107F, 0x11082, 2}, {0x11083, 0x110AF, 1}, {0x110B0, 0x110BA, 2}, {0x110BB, 0x110BC, 4}, + {0x110BE, 0x110C1, 4}, {0x110C2, 0x110C2, 2}, {0x110D0, 0x110E8, 1}, {0x110F0, 0x110F9, 48}, {0x11100, 0x11102, 2}, + {0x11103, 0x11126, 1}, {0x11127, 0x11134, 2}, {0x11136, 0x1113F, 48}, {0x11140, 0x11143, 4}, {0x11144, 0x11144, 1}, + {0x11145, 0x11146, 2}, {0x11147, 0x11147, 1}, {0x11150, 0x11172, 1}, {0x11173, 0x11173, 2}, {0x11174, 0x11175, 4}, + {0x11176, 0x11176, 1}, {0x11180, 0x11182, 2}, {0x11183, 0x111B2, 1}, {0x111B3, 0x111C0, 2}, {0x111C1, 0x111C4, 1}, + {0x111C5, 0x111C8, 4}, {0x111C9, 0x111CC, 2}, {0x111CD, 0x111CD, 4}, {0x111CE, 0x111CF, 2}, {0x111D0, 0x111D9, 48}, + {0x111DA, 0x111DA, 1}, {0x111DB, 0x111DB, 4}, {0x111DC, 0x111DC, 1}, {0x111DD, 0x111DF, 4}, {0x111E1, 0x111F4, 16}, + {0x11200, 0x11211, 1}, {0x11213, 0x1122B, 1}, {0x1122C, 0x11237, 2}, {0x11238, 0x1123D, 4}, {0x1123E, 0x1123E, 2}, + {0x1123F, 0x11240, 1}, {0x11241, 0x11241, 2}, {0x11280, 0x11286, 1}, {0x11288, 0x11288, 1}, {0x1128A, 0x1128D, 1}, + {0x1128F, 0x1129D, 1}, {0x1129F, 0x112A8, 1}, {0x112A9, 0x112A9, 4}, {0x112B0, 0x112DE, 1}, {0x112DF, 0x112EA, 2}, + {0x112F0, 0x112F9, 48}, {0x11300, 0x11303, 2}, {0x11305, 0x1130C, 1}, {0x1130F, 0x11310, 1}, {0x11313, 0x11328, 1}, + {0x1132A, 0x11330, 1}, {0x11332, 0x11333, 1}, {0x11335, 0x11339, 1}, {0x1133B, 0x1133C, 2}, {0x1133D, 0x1133D, 1}, + {0x1133E, 0x11344, 2}, {0x11347, 0x11348, 2}, {0x1134B, 0x1134D, 2}, {0x11350, 0x11350, 1}, {0x11357, 0x11357, 2}, + {0x1135D, 0x11361, 1}, {0x11362, 0x11363, 2}, {0x11366, 0x1136C, 2}, {0x11370, 0x11374, 2}, {0x11380, 0x11389, 1}, + {0x1138B, 0x1138B, 1}, {0x1138E, 0x1138E, 1}, {0x11390, 0x113B5, 1}, {0x113B7, 0x113B7, 1}, {0x113B8, 0x113C0, 2}, + {0x113C2, 0x113C2, 2}, {0x113C5, 0x113C5, 2}, {0x113C7, 0x113CA, 2}, {0x113CC, 0x113D0, 2}, {0x113D1, 0x113D1, 1}, + {0x113D2, 0x113D2, 2}, {0x113D3, 0x113D3, 1}, {0x113D4, 0x113D5, 4}, {0x113D7, 0x113D8, 4}, {0x113E1, 0x113E2, 2}, + {0x11400, 0x11434, 1}, {0x11435, 0x11446, 2}, {0x11447, 0x1144A, 1}, {0x1144B, 0x1144F, 4}, {0x11450, 0x11459, 48}, + {0x1145A, 0x1145B, 4}, {0x1145D, 0x1145D, 4}, {0x1145E, 0x1145E, 2}, {0x1145F, 0x11461, 1}, {0x11480, 0x114AF, 1}, + {0x114B0, 0x114C3, 2}, {0x114C4, 0x114C5, 1}, {0x114C6, 0x114C6, 4}, {0x114C7, 0x114C7, 1}, {0x114D0, 0x114D9, 48}, + {0x11580, 0x115AE, 1}, {0x115AF, 0x115B5, 2}, {0x115B8, 0x115C0, 2}, {0x115C1, 0x115D7, 4}, {0x115D8, 0x115DB, 1}, + {0x115DC, 0x115DD, 2}, {0x11600, 0x1162F, 1}, {0x11630, 0x11640, 2}, {0x11641, 0x11643, 4}, {0x11644, 0x11644, 1}, + {0x11650, 0x11659, 48}, {0x11660, 0x1166C, 4}, {0x11680, 0x116AA, 1}, {0x116AB, 0x116B7, 2}, {0x116B8, 0x116B8, 1}, + {0x116B9, 0x116B9, 4}, {0x116C0, 0x116C9, 48}, {0x116D0, 0x116E3, 48}, {0x11700, 0x1171A, 1}, {0x1171D, 0x1172B, 2}, + {0x11730, 0x11739, 48}, {0x1173A, 0x1173B, 16}, {0x1173C, 0x1173E, 4}, {0x1173F, 0x1173F, 8}, {0x11740, 0x11746, 1}, + {0x11800, 0x1182B, 1}, {0x1182C, 0x1183A, 2}, {0x1183B, 0x1183B, 4}, {0x118A0, 0x118DF, 1}, {0x118E0, 0x118E9, 48}, + {0x118EA, 0x118F2, 16}, {0x118FF, 0x11906, 1}, {0x11909, 0x11909, 1}, {0x1190C, 0x11913, 1}, {0x11915, 0x11916, 1}, + {0x11918, 0x1192F, 1}, {0x11930, 0x11935, 2}, {0x11937, 0x11938, 2}, {0x1193B, 0x1193E, 2}, {0x1193F, 0x1193F, 1}, + {0x11940, 0x11940, 2}, {0x11941, 0x11941, 1}, {0x11942, 0x11943, 2}, {0x11944, 0x11946, 4}, {0x11950, 0x11959, 48}, + {0x119A0, 0x119A7, 1}, {0x119AA, 0x119D0, 1}, {0x119D1, 0x119D7, 2}, {0x119DA, 0x119E0, 2}, {0x119E1, 0x119E1, 1}, + {0x119E2, 0x119E2, 4}, {0x119E3, 0x119E3, 1}, {0x119E4, 0x119E4, 2}, {0x11A00, 0x11A00, 1}, {0x11A01, 0x11A0A, 2}, + {0x11A0B, 0x11A32, 1}, {0x11A33, 0x11A39, 2}, {0x11A3A, 0x11A3A, 1}, {0x11A3B, 0x11A3E, 2}, {0x11A3F, 0x11A46, 4}, + {0x11A47, 0x11A47, 2}, {0x11A50, 0x11A50, 1}, {0x11A51, 0x11A5B, 2}, {0x11A5C, 0x11A89, 1}, {0x11A8A, 0x11A99, 2}, + {0x11A9A, 0x11A9C, 4}, {0x11A9D, 0x11A9D, 1}, {0x11A9E, 0x11AA2, 4}, {0x11AB0, 0x11AF8, 1}, {0x11B00, 0x11B09, 4}, + {0x11B60, 0x11B67, 2}, {0x11BC0, 0x11BE0, 1}, {0x11BE1, 0x11BE1, 4}, {0x11BF0, 0x11BF9, 48}, {0x11C00, 0x11C08, 1}, + {0x11C0A, 0x11C2E, 1}, {0x11C2F, 0x11C36, 2}, {0x11C38, 0x11C3F, 2}, {0x11C40, 0x11C40, 1}, {0x11C41, 0x11C45, 4}, + {0x11C50, 0x11C59, 48}, {0x11C5A, 0x11C6C, 16}, {0x11C70, 0x11C71, 4}, {0x11C72, 0x11C8F, 1}, {0x11C92, 0x11CA7, 2}, + {0x11CA9, 0x11CB6, 2}, {0x11D00, 0x11D06, 1}, {0x11D08, 0x11D09, 1}, {0x11D0B, 0x11D30, 1}, {0x11D31, 0x11D36, 2}, + {0x11D3A, 0x11D3A, 2}, {0x11D3C, 0x11D3D, 2}, {0x11D3F, 0x11D45, 2}, {0x11D46, 0x11D46, 1}, {0x11D47, 0x11D47, 2}, + {0x11D50, 0x11D59, 48}, {0x11D60, 0x11D65, 1}, {0x11D67, 0x11D68, 1}, {0x11D6A, 0x11D89, 1}, {0x11D8A, 0x11D8E, 2}, + {0x11D90, 0x11D91, 2}, {0x11D93, 0x11D97, 2}, {0x11D98, 0x11D98, 1}, {0x11DA0, 0x11DA9, 48}, {0x11DB0, 0x11DDB, 1}, + {0x11DE0, 0x11DE9, 48}, {0x11EE0, 0x11EF2, 1}, {0x11EF3, 0x11EF6, 2}, {0x11EF7, 0x11EF8, 4}, {0x11F00, 0x11F01, 2}, + {0x11F02, 0x11F02, 1}, {0x11F03, 0x11F03, 2}, {0x11F04, 0x11F10, 1}, {0x11F12, 0x11F33, 1}, {0x11F34, 0x11F3A, 2}, + {0x11F3E, 0x11F42, 2}, {0x11F43, 0x11F4F, 4}, {0x11F50, 0x11F59, 48}, {0x11F5A, 0x11F5A, 2}, {0x11FB0, 0x11FB0, 1}, + {0x11FC0, 0x11FD4, 16}, {0x11FD5, 0x11FF1, 8}, {0x11FFF, 0x11FFF, 4}, {0x12000, 0x12399, 1}, {0x12400, 0x1246E, 16}, + {0x12470, 0x12474, 4}, {0x12480, 0x12543, 1}, {0x12F90, 0x12FF0, 1}, {0x12FF1, 0x12FF2, 4}, {0x13000, 0x1342F, 1}, + {0x13440, 0x13440, 2}, {0x13441, 0x13446, 1}, {0x13447, 0x13455, 2}, {0x13460, 0x143FA, 1}, {0x14400, 0x14646, 1}, + {0x16100, 0x1611D, 1}, {0x1611E, 0x1612F, 2}, {0x16130, 0x16139, 48}, {0x16800, 0x16A38, 1}, {0x16A40, 0x16A5E, 1}, + {0x16A60, 0x16A69, 48}, {0x16A6E, 0x16A6F, 4}, {0x16A70, 0x16ABE, 1}, {0x16AC0, 0x16AC9, 48}, {0x16AD0, 0x16AED, 1}, + {0x16AF0, 0x16AF4, 2}, {0x16AF5, 0x16AF5, 4}, {0x16B00, 0x16B2F, 1}, {0x16B30, 0x16B36, 2}, {0x16B37, 0x16B3B, 4}, + {0x16B3C, 0x16B3F, 8}, {0x16B40, 0x16B43, 1}, {0x16B44, 0x16B44, 4}, {0x16B45, 0x16B45, 8}, {0x16B50, 0x16B59, 48}, + {0x16B5B, 0x16B61, 16}, {0x16B63, 0x16B77, 1}, {0x16B7D, 0x16B8F, 1}, {0x16D40, 0x16D6C, 1}, {0x16D6D, 0x16D6F, 4}, + {0x16D70, 0x16D79, 48}, {0x16E40, 0x16E7F, 1}, {0x16E80, 0x16E96, 16}, {0x16E97, 0x16E9A, 4}, {0x16EA0, 0x16EB8, 1}, + {0x16EBB, 0x16ED3, 1}, {0x16F00, 0x16F4A, 1}, {0x16F4F, 0x16F4F, 2}, {0x16F50, 0x16F50, 1}, {0x16F51, 0x16F87, 2}, + {0x16F8F, 0x16F92, 2}, {0x16F93, 0x16F9F, 1}, {0x16FE0, 0x16FE1, 1}, {0x16FE2, 0x16FE2, 4}, {0x16FE3, 0x16FE3, 1}, + {0x16FE4, 0x16FE4, 2}, {0x16FF0, 0x16FF1, 2}, {0x16FF2, 0x16FF3, 1}, {0x16FF4, 0x16FF6, 16}, {0x17000, 0x18CD5, 1}, + {0x18CFF, 0x18D1E, 1}, {0x18D80, 0x18DF2, 1}, {0x1AFF0, 0x1AFF3, 1}, {0x1AFF5, 0x1AFFB, 1}, {0x1AFFD, 0x1AFFE, 1}, + {0x1B000, 0x1B122, 1}, {0x1B132, 0x1B132, 1}, {0x1B150, 0x1B152, 1}, {0x1B155, 0x1B155, 1}, {0x1B164, 0x1B167, 1}, + {0x1B170, 0x1B2FB, 1}, {0x1BC00, 0x1BC6A, 1}, {0x1BC70, 0x1BC7C, 1}, {0x1BC80, 0x1BC88, 1}, {0x1BC90, 0x1BC99, 1}, + {0x1BC9C, 0x1BC9C, 8}, {0x1BC9D, 0x1BC9E, 2}, {0x1BC9F, 0x1BC9F, 4}, {0x1CC00, 0x1CCEF, 8}, {0x1CCF0, 0x1CCF9, 48}, + {0x1CCFA, 0x1CCFC, 8}, {0x1CD00, 0x1CEB3, 8}, {0x1CEBA, 0x1CED0, 8}, {0x1CEE0, 0x1CEF0, 8}, {0x1CF00, 0x1CF2D, 2}, + {0x1CF30, 0x1CF46, 2}, {0x1CF50, 0x1CFC3, 8}, {0x1D000, 0x1D0F5, 8}, {0x1D100, 0x1D126, 8}, {0x1D129, 0x1D164, 8}, + {0x1D165, 0x1D169, 2}, {0x1D16A, 0x1D16C, 8}, {0x1D16D, 0x1D172, 2}, {0x1D17B, 0x1D182, 2}, {0x1D183, 0x1D184, 8}, + {0x1D185, 0x1D18B, 2}, {0x1D18C, 0x1D1A9, 8}, {0x1D1AA, 0x1D1AD, 2}, {0x1D1AE, 0x1D1EA, 8}, {0x1D200, 0x1D241, 8}, + {0x1D242, 0x1D244, 2}, {0x1D245, 0x1D245, 8}, {0x1D2C0, 0x1D2D3, 16}, {0x1D2E0, 0x1D2F3, 16}, {0x1D300, 0x1D356, 8}, + {0x1D360, 0x1D378, 16}, {0x1D400, 0x1D454, 1}, {0x1D456, 0x1D49C, 1}, {0x1D49E, 0x1D49F, 1}, {0x1D4A2, 0x1D4A2, 1}, + {0x1D4A5, 0x1D4A6, 1}, {0x1D4A9, 0x1D4AC, 1}, {0x1D4AE, 0x1D4B9, 1}, {0x1D4BB, 0x1D4BB, 1}, {0x1D4BD, 0x1D4C3, 1}, + {0x1D4C5, 0x1D505, 1}, {0x1D507, 0x1D50A, 1}, {0x1D50D, 0x1D514, 1}, {0x1D516, 0x1D51C, 1}, {0x1D51E, 0x1D539, 1}, + {0x1D53B, 0x1D53E, 1}, {0x1D540, 0x1D544, 1}, {0x1D546, 0x1D546, 1}, {0x1D54A, 0x1D550, 1}, {0x1D552, 0x1D6A5, 1}, + {0x1D6A8, 0x1D6C0, 1}, {0x1D6C1, 0x1D6C1, 8}, {0x1D6C2, 0x1D6DA, 1}, {0x1D6DB, 0x1D6DB, 8}, {0x1D6DC, 0x1D6FA, 1}, + {0x1D6FB, 0x1D6FB, 8}, {0x1D6FC, 0x1D714, 1}, {0x1D715, 0x1D715, 8}, {0x1D716, 0x1D734, 1}, {0x1D735, 0x1D735, 8}, + {0x1D736, 0x1D74E, 1}, {0x1D74F, 0x1D74F, 8}, {0x1D750, 0x1D76E, 1}, {0x1D76F, 0x1D76F, 8}, {0x1D770, 0x1D788, 1}, + {0x1D789, 0x1D789, 8}, {0x1D78A, 0x1D7A8, 1}, {0x1D7A9, 0x1D7A9, 8}, {0x1D7AA, 0x1D7C2, 1}, {0x1D7C3, 0x1D7C3, 8}, + {0x1D7C4, 0x1D7CB, 1}, {0x1D7CE, 0x1D7FF, 48}, {0x1D800, 0x1D9FF, 8}, {0x1DA00, 0x1DA36, 2}, {0x1DA37, 0x1DA3A, 8}, + {0x1DA3B, 0x1DA6C, 2}, {0x1DA6D, 0x1DA74, 8}, {0x1DA75, 0x1DA75, 2}, {0x1DA76, 0x1DA83, 8}, {0x1DA84, 0x1DA84, 2}, + {0x1DA85, 0x1DA86, 8}, {0x1DA87, 0x1DA8B, 4}, {0x1DA9B, 0x1DA9F, 2}, {0x1DAA1, 0x1DAAF, 2}, {0x1DF00, 0x1DF1E, 1}, + {0x1DF25, 0x1DF2A, 1}, {0x1E000, 0x1E006, 2}, {0x1E008, 0x1E018, 2}, {0x1E01B, 0x1E021, 2}, {0x1E023, 0x1E024, 2}, + {0x1E026, 0x1E02A, 2}, {0x1E030, 0x1E06D, 1}, {0x1E08F, 0x1E08F, 2}, {0x1E100, 0x1E12C, 1}, {0x1E130, 0x1E136, 2}, + {0x1E137, 0x1E13D, 1}, {0x1E140, 0x1E149, 48}, {0x1E14E, 0x1E14E, 1}, {0x1E14F, 0x1E14F, 8}, {0x1E290, 0x1E2AD, 1}, + {0x1E2AE, 0x1E2AE, 2}, {0x1E2C0, 0x1E2EB, 1}, {0x1E2EC, 0x1E2EF, 2}, {0x1E2F0, 0x1E2F9, 48}, {0x1E2FF, 0x1E2FF, 8}, + {0x1E4D0, 0x1E4EB, 1}, {0x1E4EC, 0x1E4EF, 2}, {0x1E4F0, 0x1E4F9, 48}, {0x1E5D0, 0x1E5ED, 1}, {0x1E5EE, 0x1E5EF, 2}, + {0x1E5F0, 0x1E5F0, 1}, {0x1E5F1, 0x1E5FA, 48}, {0x1E5FF, 0x1E5FF, 4}, {0x1E6C0, 0x1E6DE, 1}, {0x1E6E0, 0x1E6E2, 1}, + {0x1E6E3, 0x1E6E3, 2}, {0x1E6E4, 0x1E6E5, 1}, {0x1E6E6, 0x1E6E6, 2}, {0x1E6E7, 0x1E6ED, 1}, {0x1E6EE, 0x1E6EF, 2}, + {0x1E6F0, 0x1E6F4, 1}, {0x1E6F5, 0x1E6F5, 2}, {0x1E6FE, 0x1E6FF, 1}, {0x1E7E0, 0x1E7E6, 1}, {0x1E7E8, 0x1E7EB, 1}, + {0x1E7ED, 0x1E7EE, 1}, {0x1E7F0, 0x1E7FE, 1}, {0x1E800, 0x1E8C4, 1}, {0x1E8C7, 0x1E8CF, 16}, {0x1E8D0, 0x1E8D6, 2}, + {0x1E900, 0x1E943, 1}, {0x1E944, 0x1E94A, 2}, {0x1E94B, 0x1E94B, 1}, {0x1E950, 0x1E959, 48}, {0x1E95E, 0x1E95F, 4}, + {0x1EC71, 0x1ECAB, 16}, {0x1ECAC, 0x1ECAC, 8}, {0x1ECAD, 0x1ECAF, 16}, {0x1ECB0, 0x1ECB0, 8}, {0x1ECB1, 0x1ECB4, 16}, + {0x1ED01, 0x1ED2D, 16}, {0x1ED2E, 0x1ED2E, 8}, {0x1ED2F, 0x1ED3D, 16}, {0x1EE00, 0x1EE03, 1}, {0x1EE05, 0x1EE1F, 1}, + {0x1EE21, 0x1EE22, 1}, {0x1EE24, 0x1EE24, 1}, {0x1EE27, 0x1EE27, 1}, {0x1EE29, 0x1EE32, 1}, {0x1EE34, 0x1EE37, 1}, + {0x1EE39, 0x1EE39, 1}, {0x1EE3B, 0x1EE3B, 1}, {0x1EE42, 0x1EE42, 1}, {0x1EE47, 0x1EE47, 1}, {0x1EE49, 0x1EE49, 1}, + {0x1EE4B, 0x1EE4B, 1}, {0x1EE4D, 0x1EE4F, 1}, {0x1EE51, 0x1EE52, 1}, {0x1EE54, 0x1EE54, 1}, {0x1EE57, 0x1EE57, 1}, + {0x1EE59, 0x1EE59, 1}, {0x1EE5B, 0x1EE5B, 1}, {0x1EE5D, 0x1EE5D, 1}, {0x1EE5F, 0x1EE5F, 1}, {0x1EE61, 0x1EE62, 1}, + {0x1EE64, 0x1EE64, 1}, {0x1EE67, 0x1EE6A, 1}, {0x1EE6C, 0x1EE72, 1}, {0x1EE74, 0x1EE77, 1}, {0x1EE79, 0x1EE7C, 1}, + {0x1EE7E, 0x1EE7E, 1}, {0x1EE80, 0x1EE89, 1}, {0x1EE8B, 0x1EE9B, 1}, {0x1EEA1, 0x1EEA3, 1}, {0x1EEA5, 0x1EEA9, 1}, + {0x1EEAB, 0x1EEBB, 1}, {0x1EEF0, 0x1EEF1, 8}, {0x1F000, 0x1F02B, 8}, {0x1F030, 0x1F093, 8}, {0x1F0A0, 0x1F0AE, 8}, + {0x1F0B1, 0x1F0BF, 8}, {0x1F0C1, 0x1F0CF, 8}, {0x1F0D1, 0x1F0F5, 8}, {0x1F100, 0x1F10C, 16}, {0x1F10D, 0x1F1AD, 8}, + {0x1F1E6, 0x1F202, 8}, {0x1F210, 0x1F23B, 8}, {0x1F240, 0x1F248, 8}, {0x1F250, 0x1F251, 8}, {0x1F260, 0x1F265, 8}, + {0x1F300, 0x1F6D8, 8}, {0x1F6DC, 0x1F6EC, 8}, {0x1F6F0, 0x1F6FC, 8}, {0x1F700, 0x1F7D9, 8}, {0x1F7E0, 0x1F7EB, 8}, + {0x1F7F0, 0x1F7F0, 8}, {0x1F800, 0x1F80B, 8}, {0x1F810, 0x1F847, 8}, {0x1F850, 0x1F859, 8}, {0x1F860, 0x1F887, 8}, + {0x1F890, 0x1F8AD, 8}, {0x1F8B0, 0x1F8BB, 8}, {0x1F8C0, 0x1F8C1, 8}, {0x1F8D0, 0x1F8D8, 8}, {0x1F900, 0x1FA57, 8}, + {0x1FA60, 0x1FA6D, 8}, {0x1FA70, 0x1FA7C, 8}, {0x1FA80, 0x1FA8A, 8}, {0x1FA8E, 0x1FAC6, 8}, {0x1FAC8, 0x1FAC8, 8}, + {0x1FACD, 0x1FADC, 8}, {0x1FADF, 0x1FAEA, 8}, {0x1FAEF, 0x1FAF8, 8}, {0x1FB00, 0x1FB92, 8}, {0x1FB94, 0x1FBEF, 8}, + {0x1FBF0, 0x1FBF9, 48}, {0x1FBFA, 0x1FBFA, 8}, {0x20000, 0x2A6DF, 1}, {0x2A700, 0x2B81D, 1}, {0x2B820, 0x2CEAD, 1}, + {0x2CEB0, 0x2EBE0, 1}, {0x2EBF0, 0x2EE5D, 1}, {0x2F800, 0x2FA1D, 1}, {0x30000, 0x3134A, 1}, {0x31350, 0x33479, 1}, + {0xE0100, 0xE01EF, 2}, +}; +inline uint8_t unicodeFlags(uint32_t cp) { + size_t l = 0, r = std::size(unicode_ranges); + while (l < r) { + auto m = (l + r) / 2; + if (unicode_ranges[m].last < cp) + l = m + 1; + else + r = m; + } + return l < std::size(unicode_ranges) && unicode_ranges[l].first <= cp ? unicode_ranges[l].flags : 0; +} +} // namespace mllm::models::spark2_5 diff --git a/mllm/nn/layers/GroupedQueryAttention.cpp b/mllm/nn/layers/GroupedQueryAttention.cpp index 0e629c091..cdd46e9ab 100644 --- a/mllm/nn/layers/GroupedQueryAttention.cpp +++ b/mllm/nn/layers/GroupedQueryAttention.cpp @@ -8,7 +8,8 @@ namespace mllm::nn { GroupedQueryAttention::GroupedQueryAttention() : Layer(OpTypes::kGroupedQueryAttention, aops::GroupedQueryAttentionOpOptions{}) {} -GroupedQueryAttention::GroupedQueryAttention(aops::GroupedQueryAttentionImplementation implementation) - : Layer(OpTypes::kGroupedQueryAttention, aops::GroupedQueryAttentionOpOptions{.implementation = implementation}) {} +GroupedQueryAttention::GroupedQueryAttention(aops::GroupedQueryAttentionImplementation implementation, int32_t sliding_window) + : Layer(OpTypes::kGroupedQueryAttention, + aops::GroupedQueryAttentionOpOptions{.implementation = implementation, .sliding_window = sliding_window}) {} } // namespace mllm::nn diff --git a/mllm/nn/layers/GroupedQueryAttention.hpp b/mllm/nn/layers/GroupedQueryAttention.hpp index 99eca8769..16faeb160 100644 --- a/mllm/nn/layers/GroupedQueryAttention.hpp +++ b/mllm/nn/layers/GroupedQueryAttention.hpp @@ -11,7 +11,7 @@ namespace mllm::nn { class GroupedQueryAttention : public Layer { public: GroupedQueryAttention(); - explicit GroupedQueryAttention(aops::GroupedQueryAttentionImplementation implementation); + explicit GroupedQueryAttention(aops::GroupedQueryAttentionImplementation implementation, int32_t sliding_window = 0); MLLM_LAYER_ANY_INPUTS_1_OUTPUTS_FORWARD }; diff --git a/tests/models/CMakeLists.txt b/tests/models/CMakeLists.txt index e023c0366..2dc77f432 100644 --- a/tests/models/CMakeLists.txt +++ b/tests/models/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(qwen3_5) add_subdirectory(minicpm5) +add_subdirectory(spark2_5) diff --git a/tests/models/spark2_5/CMakeLists.txt b/tests/models/spark2_5/CMakeLists.txt new file mode 100644 index 000000000..3e63c0d40 --- /dev/null +++ b/tests/models/spark2_5/CMakeLists.txt @@ -0,0 +1,6 @@ +add_executable(Mllm-Test-Spark25 SparkTest.cpp) +target_link_libraries(Mllm-Test-Spark25 PRIVATE gtest_main MllmRT MllmCPUBackend) +target_include_directories(Mllm-Test-Spark25 PRIVATE ${MLLM_INCLUDE_DIR}) +target_compile_definitions(Mllm-Test-Spark25 PRIVATE SPARK_EXAMPLE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/../../../examples/spark2_5") +add_test(NAME Spark25Focused COMMAND Mllm-Test-Spark25) +set_tests_properties(Spark25Focused PROPERTIES LABELS spark25) diff --git a/tests/models/spark2_5/SparkTest.cpp b/tests/models/spark2_5/SparkTest.cpp new file mode 100644 index 000000000..15b8b536b --- /dev/null +++ b/tests/models/spark2_5/SparkTest.cpp @@ -0,0 +1,101 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. +#include +#include +#include +#include "mllm/models/spark2_5/modeling_spark2_5.hpp" +#include "mllm/models/spark2_5/tokenization_spark2_5.hpp" +using namespace mllm; +using namespace mllm::models::spark2_5; +class SparkTest : public testing::Test { + protected: + static void SetUpTestSuite() { initializeContext(); } +}; +static std::string exampleDir() { + auto* p = std::getenv("MLLM_SPARK_EXAMPLE_DIR"); + return p ? p : SPARK_EXAMPLE_DIR; +} +TEST_F(SparkTest, OfficialConfigAndTemplate) { + SparkConfig c(exampleDir() + "/config_1.7B_fp32.json"); + EXPECT_EQ(c.full_layers, 7); + EXPECT_EQ(c.rotary_dim, 64); + EXPECT_EQ(c.head_dim, 256); + EXPECT_EQ(c.sliding_window, 512); + EXPECT_EQ(SparkTokenizer::applyChatTemplate({"Hi", "", false}), + "<｜start▁of▁sentence｜><|System|>\nyou are a helpful " + "assistant.<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|User|>Hi<｜end▁of▁sentence｜><｜start▁of▁sentence｜><|Bot|" + ">"); + EXPECT_THROW(SparkTokenizer::applyChatTemplate({"", "", true}), std::invalid_argument); +} +TEST_F(SparkTest, PretokenizerUnicodeAndDigits) { + EXPECT_EQ(sparkPieces(L"12345"), (std::vector{L"1", L"2", L"3", L"4", L"5"})); + EXPECT_EQ(sparkPieces(L"中文hello"), (std::vector{L"中文", L"hello"})); + EXPECT_EQ(sparkPieces(L"e\u0301"), (std::vector{L"e\u0301"})); + EXPECT_EQ(sparkPieces(L" hello"), (std::vector{L" hello"})); +} +TEST_F(SparkTest, OfficialTokenizerParity) { + auto* root = std::getenv("MLLM_SPARK_TOKENIZER_FIXTURES"); + if (!root) GTEST_SKIP() << "Official checkpoint fixture is external"; + const auto j = nlohmann::json::parse(std::ifstream(std::string(root) + "/tokenizer_cases.json")); + SparkTokenizer t(std::string(root) + "/tokenizer.json"); + for (const auto& c : j) { + const auto text = c.at("text").get(); + EXPECT_EQ(t.encode(text), c.at("ids").get>()) << text; + std::string decoded; + for (auto id : t.encode(text)) decoded += t.detokenizeBytes(id); + EXPECT_EQ(decoded, text); + } +} +TEST_F(SparkTest, SmallModelChunkAndReset) { + SparkConfig c(exampleDir() + "/config_1.7B_fp32.json"); + c.vocab_size = 32; + c.hidden_size = 16; + c.intermediate_size = 24; + c.head_dim = 8; + c.rotary_dim = 2; + c.num_attention_heads = 2; + c.num_key_value_heads = 1; + c.num_hidden_layers = 4; + c.full_layers = 1; + c.sliding_window = 4; + c.max_cache_length = 32; + c.layer_types = {"sliding_attention", "sliding_attention", "sliding_attention", "full_attention"}; + auto params = ParameterFile::create(); + auto add = [&](std::string name, Tensor::shape_t shape, bool norm = false) { + auto t = Tensor::empty(shape, kFloat32, kCPU).alloc(); + for (size_t i = 0; i < t.numel(); ++i) t.ptr()[i] = norm ? 1.0F : 0.07F * std::sin(float(i + 1)); + params->push(name, t); + }; + add("model.embedding.weight", {32, 16}); + add("lm_head.weight", {32, 16}); + add("model.norm.weight", {16}, true); + for (int l = 0; l < 4; ++l) { + std::string p = "model.layers." + std::to_string(l) + "."; + add(p + "input_layernorm.weight", {16}, true); + add(p + "post_attention_layernorm.weight", {16}, true); + add(p + "self_attn.q_k_v_proj.weight", {32, 16}); + add(p + "self_attn.g_proj.weight", {2, 16}); + add(p + "self_attn.out_proj.weight", {16, 16}); + add(p + "mlp.gate_proj.weight", {24, 16}); + add(p + "mlp.up_proj.weight", {24, 16}); + add(p + "mlp.down_proj.weight", {16, 24}); + } + SparkForCausalLM m(c); + m.load(params); + m.all_logits = true; + auto tokens = Tensor::empty({1, 13}, kInt64, kCPU).alloc(); + for (int i = 0; i < 13; ++i) tokens.ptr()[i] = i; + auto full = m.forward({{"sequence", tokens}}, {}).at("sequence").clone(); + for (int chunk : {1, 3, 7}) { + m.resetState(); + for (int i = 0; i < 13; i += chunk) { + int end = std::min(i + chunk, 13); + auto part = m.forward({{"sequence", tokens[{kAll, {i, end}}].contiguous()}}, {}).at("sequence"); + for (int t = i; t < end; ++t) + for (int d = 0; d < 32; ++d) EXPECT_NEAR(part.ptr()[(t - i) * 32 + d], full.ptr()[t * 32 + d], 2e-5F); + } + } + m.resetState(); + auto again = m.forward({{"sequence", tokens}}, {}).at("sequence"); + for (size_t i = 0; i < full.numel(); ++i) EXPECT_FLOAT_EQ(full.ptr()[i], again.ptr()[i]); +} diff --git a/tests/nn/CMakeLists.txt b/tests/nn/CMakeLists.txt index 4d84b9399..c4626ee34 100644 --- a/tests/nn/CMakeLists.txt +++ b/tests/nn/CMakeLists.txt @@ -31,3 +31,15 @@ add_test(NAME GatedDeltaRuleFocused COMMAND Mllm-Test-Nn-GatedDeltaRule) add_test(NAME GroupedQueryAttentionFocused COMMAND Mllm-Test-Nn-GroupedQueryAttention) add_test(NAME KVHeadStaticCacheFocused COMMAND Mllm-Test-Nn-KVHeadStaticCache) set_tests_properties(GroupedQueryAttentionFocused KVHeadStaticCacheFocused PROPERTIES LABELS nn-op) + +add_executable(Mllm-Test-Nn-GELU GELUTest.cpp) +target_link_libraries(Mllm-Test-Nn-GELU PRIVATE gtest_main MllmRT MllmCPUBackend) +target_include_directories(Mllm-Test-Nn-GELU PRIVATE ${MLLM_INCLUDE_DIR}) +add_test(NAME GELUFocused COMMAND Mllm-Test-Nn-GELU) +set_tests_properties(GELUFocused PROPERTIES LABELS nn-op) + +add_executable(Mllm-Test-Nn-Sigmoid SigmoidTest.cpp) +target_link_libraries(Mllm-Test-Nn-Sigmoid PRIVATE gtest_main MllmRT MllmCPUBackend) +target_include_directories(Mllm-Test-Nn-Sigmoid PRIVATE ${MLLM_INCLUDE_DIR}) +add_test(NAME SigmoidFocused COMMAND Mllm-Test-Nn-Sigmoid) +set_tests_properties(SigmoidFocused PROPERTIES LABELS nn-op) diff --git a/tests/nn/GELUTest.cpp b/tests/nn/GELUTest.cpp new file mode 100644 index 000000000..b89758cbb --- /dev/null +++ b/tests/nn/GELUTest.cpp @@ -0,0 +1,57 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. +#include +#include +#include "mllm/mllm.hpp" +#include "mllm/nn/layers/GELU.hpp" +#include "mllm/compile/ir/Trace.hpp" +#include "mllm/compile/ir/linalg/Op.hpp" +#include "mllm/compile/jit/binary/LinalgIRSerialization.hpp" +#include "mllm/compile/jit/interpreter/AopsFromJson.hpp" +class GELUTest : public testing::Test { + protected: + static void SetUpTestSuite() { mllm::initializeContext(); } +}; +class ExactGELUModule : public mllm::nn::Module { + public: + ExactGELUModule() : Module("exact_gelu") { + op = reg("gelu", mllm::aops::GELUOpOptions{.approximate = false}); + } + std::vector forward(const std::vector& x, const std::vector&) override { + return {op(x[0])}; + } + mllm::nn::GELU op; +}; +TEST_F(GELUTest, ExactErfAndTail) { + ExactGELUModule module; + auto& gelu = module.op; + auto x = mllm::Tensor::empty({137}, mllm::kFloat32, mllm::kCPU).alloc(); + for (int i = 0; i < 137; ++i) x.ptr()[i] = (i - 68) * 0.1F; + auto y = gelu(x); + for (int i = 0; i < 137; ++i) { + double v = x.ptr()[i]; + EXPECT_NEAR(y.ptr()[i], 0.5 * v * std::erfc(-v / std::sqrt(2.)), 2e-6); + } + EXPECT_THROW(gelu(mllm::Tensor::zeros({2}, mllm::kFloat16, mllm::kCPU)), std::invalid_argument); +} +static mllm::ir::linalg::GELUOp::ptr_t findGelu(const mllm::ir::node_ptr_t& n) { + if (n->isa_()) return n->cast_(); + if (n->isa_()) + for (auto& r : n->cast_()->regions()) + for (auto& o : r->ops()) + if (auto f = findGelu(o)) return f; + return nullptr; +} +TEST_F(GELUTest, ExactOptionTraceRoundTrip) { + ExactGELUModule m; + auto ir = mllm::ir::trace(m, mllm::Tensor::empty({4}, mllm::kFloat32, mllm::kCPU)); + auto op = findGelu(ir->topLevelOp()); + ASSERT_NE(op, nullptr); + auto options = mllm::jit::binary::dumpLinalgIROptions(op); + EXPECT_EQ(options.at("approximate"), false); + auto restored = + mllm::jit::interpreter::aopsFromJson(nlohmann::json{{"op_type", "GELU"}, {"backend", "CPU"}, {"op_options", options}}); + EXPECT_FALSE(std::static_pointer_cast(restored)->options().approximate); + auto legacy = mllm::jit::interpreter::aopsFromJson(nlohmann::json{{"op_type", "GELU"}, {"backend", "CPU"}}); + EXPECT_TRUE(std::static_pointer_cast(legacy)->options().approximate); +} diff --git a/tests/nn/GroupedQueryAttentionTest.cpp b/tests/nn/GroupedQueryAttentionTest.cpp index af66bb718..9b5dc70ad 100644 --- a/tests/nn/GroupedQueryAttentionTest.cpp +++ b/tests/nn/GroupedQueryAttentionTest.cpp @@ -17,6 +17,7 @@ #include "mllm/compile/jit/interpreter/AopsFromJson.hpp" #include "mllm/core/OpTypes.hpp" #include "mllm/nn/Functional.hpp" +#include "mllm/nn/layers/GroupedQueryAttention.hpp" #include "mllm/nn/llm_components/GroupedQueryAttention.hpp" namespace { @@ -39,14 +40,19 @@ class GroupedQueryAttentionDecodeTraceModule final : public mllm::nn::Module { class GroupedQueryAttentionTraceModule final : public mllm::nn::Module { public: - GroupedQueryAttentionTraceModule() : Module("gqa_trace") {} + GroupedQueryAttentionTraceModule(mllm::aops::GroupedQueryAttentionImplementation implementation = + mllm::aops::GroupedQueryAttentionImplementation::kDirectStrided, + int window = 512) + : Module("gqa_trace") { + op = reg("attn", implementation, window); + } + mllm::nn::GroupedQueryAttention op; std::vector forward(const std::vector& inputs, const std::vector& args) override { - return {mllm::nn::functional::groupedQueryAttention(inputs[0], inputs[1], inputs[2])}; + return {op(inputs[0], inputs[1], inputs[2])}; } }; - mllm::ir::linalg::GroupedQueryAttentionOp::ptr_t findGroupedQueryAttentionOp(const mllm::ir::node_ptr_t& node) { if (node->isa_()) { return node->cast_(); @@ -68,7 +74,7 @@ Tensor sequential(const Tensor::shape_t& shape, float scale) { return tensor; } -Tensor gqaReference(const Tensor& query, const Tensor& key, const Tensor& value) { +Tensor gqaReference(const Tensor& query, const Tensor& key, const Tensor& value, int32_t window = 0) { const auto q_shape = query.shape(); const auto k_shape = key.shape(); const auto v_shape = value.shape(); @@ -82,9 +88,9 @@ Tensor gqaReference(const Tensor& query, const Tensor& key, const Tensor& value) const int32_t kv_head = query_head / groups; for (int32_t query_index = 0; query_index < q_shape[2]; ++query_index) { const int32_t allowed_keys = context_offset + query_index + 1; - std::vector probabilities(static_cast(allowed_keys)); + std::vector probabilities(static_cast(allowed_keys), -std::numeric_limits::infinity()); float maximum = -std::numeric_limits::infinity(); - for (int32_t key_index = 0; key_index < allowed_keys; ++key_index) { + for (int32_t key_index = (window ? std::max(0, allowed_keys - window) : 0); key_index < allowed_keys; ++key_index) { float score = 0.0F; for (int32_t dim = 0; dim < q_shape[3]; ++dim) { score += *query.cptrAt({batch, query_head, query_index, dim}) @@ -100,7 +106,7 @@ Tensor gqaReference(const Tensor& query, const Tensor& key, const Tensor& value) } for (int32_t value_dim = 0; value_dim < v_shape[3]; ++value_dim) { float result = 0.0F; - for (int32_t key_index = 0; key_index < allowed_keys; ++key_index) { + for (int32_t key_index = (window ? std::max(0, allowed_keys - window) : 0); key_index < allowed_keys; ++key_index) { result += probabilities[static_cast(key_index)] / denominator * *value.cptrAt({batch, kv_head, key_index, value_dim}); } @@ -231,7 +237,7 @@ TEST_F(GroupedQueryAttentionTest, DirectStridedMatchesLegacyReductionAtLfm25Geom } } -TEST_F(GroupedQueryAttentionTest, DirectStridedOpTraceAndSerializationRoundTrip) { +TEST_F(GroupedQueryAttentionTest, SlidingOpTraceAndSerializationRoundTrip) { GroupedQueryAttentionTraceModule module; auto ir_ctx = mllm::ir::trace(module, Tensor::empty({1, 4, 2, 5}, mllm::kFloat32, mllm::kCPU), Tensor::empty({1, 2, 4, 5}, mllm::kFloat32, mllm::kCPU), @@ -242,6 +248,7 @@ TEST_F(GroupedQueryAttentionTest, DirectStridedOpTraceAndSerializationRoundTrip) const auto options = mllm::jit::binary::dumpLinalgIROptions(ir_op); EXPECT_EQ(options.at("implementation"), "DirectStrided"); + EXPECT_EQ(options.at("sliding_window"), 512); const auto restored = mllm::jit::interpreter::aopsFromJson( nlohmann::json{{"op_type", "GroupedQueryAttention"}, {"backend", "CPU"}, {"op_options", options}}); ASSERT_NE(restored, nullptr); @@ -361,3 +368,35 @@ TEST_F(GroupedQueryAttentionTest, SupportsTransposedNonContiguousHeadViews) { expectNear(actual, expected); } } // namespace + +TEST_F(GroupedQueryAttentionTest, SlidingWindowBoundaryAndChunks) { + for (int n : {511, 512, 513, 1025}) { + auto q = sequential({1, 4, n, 8}, 0.023F), k = sequential({1, 2, n, 8}, 0.031F), v = sequential({1, 2, n, 8}, 0.017F); + GroupedQueryAttentionTraceModule module; + auto& op = module.op; + auto actual = op(q, k, v); + auto expected = gqaReference(q, k, v, 512); + for (size_t i = 0; i < actual.numel(); ++i) EXPECT_NEAR(actual.ptr()[i], expected.ptr()[i], 2e-5F); + int pos = 0; + while (pos < n) { + int end = std::min(n, pos + 137), begin = std::max(0, pos - 511); + auto part = op(q[{mllm::kAll, mllm::kAll, {pos, end}, mllm::kAll}], k[{mllm::kAll, mllm::kAll, {begin, end}, mllm::kAll}], + v[{mllm::kAll, mllm::kAll, {begin, end}, mllm::kAll}]); + for (int h = 0; h < 4; ++h) + for (int t = pos; t < end; ++t) + for (int d = 0; d < 8; ++d) + EXPECT_NEAR(part.ptr()[(h * (end - pos) + t - pos) * 8 + d], actual.ptr()[(h * n + t) * 8 + d], + 2e-5F); + pos = end; + } + } +} +TEST_F(GroupedQueryAttentionTest, InvalidWindow) { + auto q = sequential({1, 2, 1, 8}, 0.1F); + GroupedQueryAttentionTraceModule bad_module(mllm::aops::GroupedQueryAttentionImplementation::kDirectStrided, -1); + auto& bad = bad_module.op; + EXPECT_THROW(bad(q, q, q), std::invalid_argument); + GroupedQueryAttentionTraceModule decode_module(mllm::aops::GroupedQueryAttentionImplementation::kDecodeNativeKV, 512); + auto& decode = decode_module.op; + EXPECT_THROW(decode(q, q, q), std::invalid_argument); +} diff --git a/tests/nn/SigmoidTest.cpp b/tests/nn/SigmoidTest.cpp new file mode 100644 index 000000000..0ae9fcb19 --- /dev/null +++ b/tests/nn/SigmoidTest.cpp @@ -0,0 +1,57 @@ +// Copyright (c) MLLM Team. +// Licensed under the MIT License. +#include +#include +#include "mllm/mllm.hpp" +#include "mllm/nn/layers/Sigmoid.hpp" +#include "mllm/compile/ir/Trace.hpp" +#include "mllm/compile/ir/linalg/Op.hpp" +#include "mllm/compile/jit/binary/LinalgIRSerialization.hpp" +#include "mllm/compile/jit/interpreter/AopsFromJson.hpp" +class SigmoidTest : public testing::Test { + protected: + static void SetUpTestSuite() { mllm::initializeContext(); } +}; +class ExactSigmoidModule : public mllm::nn::Module { + public: + ExactSigmoidModule() : Module("exact_sigmoid") { + op = reg("sigmoid", mllm::aops::SigmoidOpOptions{.approximate = false}); + } + std::vector forward(const std::vector& x, const std::vector&) override { + return {op(x[0])}; + } + mllm::nn::Sigmoid op; +}; +TEST_F(SigmoidTest, AccurateLogisticAndTail) { + ExactSigmoidModule module; + auto& sigmoid = module.op; + auto x = mllm::Tensor::empty({137}, mllm::kFloat32, mllm::kCPU).alloc(); + for (int i = 0; i < 137; ++i) x.ptr()[i] = (i - 68) * 0.3F; + auto y = sigmoid(x); + for (int i = 0; i < 137; ++i) { + double v = x.ptr()[i]; + EXPECT_NEAR(y.ptr()[i], 1.0 / (1.0 + std::exp(-v)), 1e-7); + } + EXPECT_THROW(sigmoid(mllm::Tensor::zeros({2}, mllm::kFloat16, mllm::kCPU)), std::invalid_argument); +} +static mllm::ir::linalg::SigmoidOp::ptr_t findSigmoid(const mllm::ir::node_ptr_t& n) { + if (n->isa_()) return n->cast_(); + if (n->isa_()) + for (auto& r : n->cast_()->regions()) + for (auto& o : r->ops()) + if (auto f = findSigmoid(o)) return f; + return nullptr; +} +TEST_F(SigmoidTest, ExactOptionTraceRoundTrip) { + ExactSigmoidModule m; + auto ir = mllm::ir::trace(m, mllm::Tensor::empty({4}, mllm::kFloat32, mllm::kCPU)); + auto op = findSigmoid(ir->topLevelOp()); + ASSERT_NE(op, nullptr); + auto options = mllm::jit::binary::dumpLinalgIROptions(op); + EXPECT_EQ(options.at("approximate"), false); + auto restored = + mllm::jit::interpreter::aopsFromJson(nlohmann::json{{"op_type", "Sigmoid"}, {"backend", "CPU"}, {"op_options", options}}); + EXPECT_FALSE(std::static_pointer_cast(restored)->options().approximate); + auto legacy = mllm::jit::interpreter::aopsFromJson(nlohmann::json{{"op_type", "Sigmoid"}, {"backend", "CPU"}}); + EXPECT_TRUE(std::static_pointer_cast(legacy)->options().approximate); +}