Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -4348,8 +4348,7 @@ version = "0.0.0"
dependencies = [
"expect-test",
"memchr",
"unicode-ident",
"unicode-properties",
"rustc_unicode",
]

[[package]]
Expand Down Expand Up @@ -4380,10 +4379,10 @@ dependencies = [
"rustc_symbol_mangling",
"rustc_target",
"rustc_trait_selection",
"rustc_unicode",
"smallvec",
"thin-vec",
"tracing",
"unicode-security",
]

[[package]]
Expand Down Expand Up @@ -4633,11 +4632,9 @@ dependencies = [
"rustc_macros",
"rustc_session",
"rustc_span",
"rustc_unicode",
"thin-vec",
"tracing",
"unicode-normalization",
"unicode-properties",
"unicode-width 0.2.2",
]

[[package]]
Expand Down Expand Up @@ -4870,11 +4867,11 @@ dependencies = [
"rustc_index",
"rustc_macros",
"rustc_serialize",
"rustc_unicode",
"scoped-tls",
"sha1",
"sha2",
"tracing",
"unicode-width 0.2.2",
]

[[package]]
Expand Down Expand Up @@ -5077,6 +5074,18 @@ dependencies = [
"synstructure",
]

[[package]]
name = "rustc_unicode"
version = "0.0.0"
dependencies = [
"unicode-ident",
"unicode-normalization",
"unicode-properties",
"unicode-script",
"unicode-security",
"unicode-width 0.2.2",
]

[[package]]
name = "rustc_version"
version = "0.4.1"
Expand Down Expand Up @@ -6230,9 +6239,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"

[[package]]
name = "unicode-script"
version = "0.5.7"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fb421b350c9aff471779e262955939f565ec18b86c15364e6bdf0d662ca7c1f"
checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee"

[[package]]
name = "unicode-security"
Expand Down
5 changes: 2 additions & 3 deletions compiler/rustc_lexer/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,10 @@ Rust lexer used by rustc. No stability guarantees are provided.
# This will be used when publishing this crate as `rustc-ap-rustc_lexer`.
[lib]

# Note that this crate purposefully does not depend on other rustc crates
# Note that this crate purposefully does not depend on other rustc crates other than `rustc-unicode`

@clubby789 clubby789 Nov 1, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if this is appropriate, and how better to resolve this

View changes since the review

@fmease fmease Nov 1, 2026 •

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well, the reason for this is the fact that rust-analyzer wants to depend on this crate (or rather their republish, ra-ap-rustc_lexer IIUC). Not sure if your change would break their process..

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it should be fine as long as rustc_unicode doesn't use unstable features.

[dependencies]
memchr = "2.7.6"
unicode-properties = { version = "0.1.4", default-features = false, features = ["emoji"] }
unicode-ident = "1.0.22"
rustc_unicode = { path = "../rustc_unicode" }

[dev-dependencies]
expect-test = "1.4.0"
19 changes: 1 addition & 18 deletions compiler/rustc_lexer/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,26 +32,9 @@ use std::str::Chars;

use LiteralKind::*;
use TokenKind::*;
pub use unicode_ident::UNICODE_VERSION;
use rustc_unicode::{unicode_ident, unicode_properties};
use unicode_properties::UnicodeEmoji;

// Make sure that the Unicode version of the dependencies is the same.
const _: () = {
let properties = unicode_properties::UNICODE_VERSION;
let ident = unicode_ident::UNICODE_VERSION;

if properties.0 != ident.0 as u64
|| properties.1 != ident.1 as u64
|| properties.2 != ident.2 as u64
{
panic!(
"unicode-properties and unicode-ident must use the same Unicode version, \
`unicode_properties::UNICODE_VERSION` and `unicode_ident::UNICODE_VERSION` are \
different."
);
}
};

/// Parsed token.
/// It doesn't contain information about data that has been parsed,
/// only the type of the token and its size.
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_lint/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ rustc_structures = { path = "../rustc_structures" }
rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
rustc_target = { path = "../rustc_target" }
rustc_trait_selection = { path = "../rustc_trait_selection" }
rustc_unicode = { path = "../rustc_unicode" }
smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
thin-vec = "0.2.19"
tracing = "0.1"
unicode-security = "0.1.0"
# tidy-alphabetical-end
1 change: 1 addition & 0 deletions compiler/rustc_lint/src/non_ascii_idents.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use rustc_data_structures::fx::FxIndexMap;
use rustc_data_structures::unord::UnordMap;
use rustc_lint_defs::{declare_lint, declare_lint_pass};
use rustc_span::{OrdSpan, Symbol};
use rustc_unicode::unicode_security;
use unicode_security::general_security_profile::IdentifierType;

use crate::diagnostics::{
Expand Down
4 changes: 1 addition & 3 deletions compiler/rustc_parse/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@ rustc_lint_defs = { path = "../rustc_lint_defs" }
rustc_macros = { path = "../rustc_macros" }
rustc_session = { path = "../rustc_session" }
rustc_span = { path = "../rustc_span" }
rustc_unicode = { path = "../rustc_unicode" }
thin-vec = "0.2.21"
tracing = "0.1"
unicode-normalization = "0.1.25"
unicode-properties = { version = "0.1.4", default-features = false, features = ["emoji"] }
unicode-width = "0.2.2"
# tidy-alphabetical-end

[dev-dependencies]
Expand Down
4 changes: 2 additions & 2 deletions compiler/rustc_parse/src/lexer/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ use rustc_literal_escaper::{EscapeError, Mode, check_for_errors};
use rustc_session::parse::ParseSess;
use rustc_span::edition::Edition;
use rustc_span::{BytePos, Pos, Span, Symbol, sym};
use rustc_unicode::unicode_properties::emoji::UnicodeEmoji;
use tracing::debug;
use unicode_properties::emoji::UnicodeEmoji;

use crate::lexer::diagnostics::TokenTreeDiagInfo;
use crate::lexer::unicode_chars::UNICODE_ARRAY;
Expand Down Expand Up @@ -1301,7 +1301,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
}

pub fn nfc_normalize(string: &str) -> Symbol {
use unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick};
use rustc_unicode::unicode_normalization::{IsNormalized, UnicodeNormalization, is_nfc_quick};
match is_nfc_quick(string.chars()) {
IsNormalized::Yes => Symbol::intern(string),
_ => {
Expand Down
1 change: 1 addition & 0 deletions compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use std::ops::Range;
use rustc_errors::{Applicability, Diag, DiagCtxtHandle, ErrorGuaranteed};
use rustc_literal_escaper::{EscapeError, Mode};
use rustc_span::{BytePos, Span};
use rustc_unicode::{unicode_normalization, unicode_width};
use tracing::debug;

use crate::diagnostics::{
Expand Down
39 changes: 0 additions & 39 deletions compiler/rustc_parse/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ use rustc_ast::token;
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_ast_pretty::pprust;
use rustc_errors::{Diag, FatalError, PResult, pluralize};
pub use rustc_lexer::UNICODE_VERSION;
use rustc_session::parse::ParseSess;
use rustc_span::edit_distance::find_best_match_for_name;
use rustc_span::source_map::SourceMap;
Expand All @@ -36,44 +35,6 @@ pub mod lexer;

mod diagnostics;

// Make sure that the Unicode version of the dependencies is the same.
const _: () = {
let rustc_lexer = rustc_lexer::UNICODE_VERSION;
let rustc_span = rustc_span::UNICODE_VERSION;
let normalization = unicode_normalization::UNICODE_VERSION;
let width = unicode_width::UNICODE_VERSION;

if rustc_lexer.0 != rustc_span.0
|| rustc_lexer.1 != rustc_span.1
|| rustc_lexer.2 != rustc_span.2
{
panic!(
"rustc_lexer and rustc_span must use the same Unicode version, \
`rustc_lexer::UNICODE_VERSION` and `rustc_span::UNICODE_VERSION` are \
different."
);
}

if rustc_lexer.0 != normalization.0
|| rustc_lexer.1 != normalization.1
|| rustc_lexer.2 != normalization.2
{
panic!(
"rustc_lexer and unicode-normalization must use the same Unicode version, \
`rustc_lexer::UNICODE_VERSION` and `unicode_normalization::UNICODE_VERSION` are \
different."
);
}

if rustc_lexer.0 != width.0 || rustc_lexer.1 != width.1 || rustc_lexer.2 != width.2 {
panic!(
"rustc_lexer and unicode-width must use the same Unicode version, \
`rustc_lexer::UNICODE_VERSION` and `unicode_width::UNICODE_VERSION` are \
different."
);
}
};

// Unwrap the result if `Ok`, otherwise emit the diagnostics and abort.
pub fn unwrap_or_emit_fatal<T>(expr: Result<T, Vec<Diag<'_>>>) -> T {
match expr {
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_span/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ rustc_hashes = { path = "../rustc_hashes" }
rustc_index = { path = "../rustc_index" }
rustc_macros = { path = "../rustc_macros" }
rustc_serialize = { path = "../rustc_serialize" }
rustc_unicode = { path = "../rustc_unicode" }
scoped-tls = "1.0"
sha1 = "0.10.0"
sha2 = "0.10.1"
tracing = "0.1"
unicode-width = "0.2.2"
# tidy-alphabetical-end
2 changes: 1 addition & 1 deletion compiler/rustc_span/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ use rustc_serialize::opaque::mem_encoder::MemEncoder;
use rustc_serialize::opaque::{FileEncoder, MemDecoder};
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
use tracing::debug;
pub use unicode_width::UNICODE_VERSION;

mod caching_source_map_view;
pub mod source_map;
Expand Down Expand Up @@ -90,6 +89,7 @@ use rustc_data_structures::stable_hash::{StableHash, StableHasher};
use rustc_data_structures::sync::{FreezeLock, FreezeWriteGuard, Lock};
use rustc_data_structures::unord::UnordMap;
use rustc_hashes::{Hash64, Hash128};
use rustc_unicode::unicode_width;
use sha1::Sha1;
use sha2::Sha256;
use span_encoding::InlineCtxt;
Expand Down
14 changes: 14 additions & 0 deletions compiler/rustc_unicode/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
[package]
name = "rustc_unicode"
version = "0.0.0"
edition = "2024"

[dependencies]
# tidy-alphabetical-start
unicode-ident = "=1.0.24"
unicode-normalization = "=0.1.25"
unicode-properties = { version = "=0.1.4", default-features = false, features = ["emoji"] }
unicode-script = "=0.5.8"
unicode-security = "=0.1.2"
unicode-width = "=0.2.2"
# tidy-alphabetical-end
38 changes: 38 additions & 0 deletions compiler/rustc_unicode/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//! Unified crate for re-exporting Unicode crates for the compiler.
//! Used to avoid having multiple different Unicode versions in use.

// If this changes, update https://github.com/rust-lang/reference/blob/HEAD/src/identifiers.md.
const UNICODE_VERSION: (u8, u8, u8) = (17, 0, 0);

macro_rules! reexport_and_check {
($($krate:ident),+ $(,)?) => {
$(
pub use $krate;
const _: () = {
let (major, minor, patch) = crate::UNICODE_VERSION;
let (major_, minor_, patch_) = $krate::UNICODE_VERSION;
if major != major_ as u8 || minor != minor_ as u8 || patch != patch_ as u8 {
panic!(
concat!(
stringify!($krate),
" does not have the correct Unicode version (`rustc-unicode::UNICODE_VERSION`)"
)
)
}
};
)+
}
}
Comment on lines +4 to +25

@tgross35 tgross35 Nov 2, 2026 •

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Presumably this should check char::UNICODE_VERSION as well, since that is relied upon pretty much everywhere. Cc #164841 (comment).

I assume some cfg(bootstrap) may be needed for that since we can't bump it independently.

View changes since the review

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since Unicode 18 is in use there, we won't be able to add that check until #165289 lands. Depending on the order of things getting merged I'll open up a followup PR, or update this PR to Unicode 18.


reexport_and_check!(
// tidy-alphabetical-start
unicode_ident,
unicode_normalization,
unicode_properties,
// FIXME: Update to version supporting Unicode 17
// unicode_security,
unicode_script,
unicode_width,
// tidy-alphabetical-end
);
pub use unicode_security;
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@ expression: bench
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Bench] test::CrateRustdoc
targets: [x86_64-unknown-linux-gnu]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ expression: build compiler
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Build] compile::Assemble
targets: [x86_64-unknown-linux-gnu]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ expression: check
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Check] check::Rustdoc
targets: [x86_64-unknown-linux-gnu]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,4 +82,5 @@ expression: check compiler
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ expression: check compiletest --include-default-paths
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Check] check::Rustdoc
targets: [x86_64-unknown-linux-gnu]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,4 +97,5 @@ expression: clippy
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ expression: fix
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Fix] check::Rustdoc
targets: [x86_64-unknown-linux-gnu]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ expression: test
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Test] test::CrateRustdoc
targets: [x86_64-unknown-linux-gnu]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ expression: test --skip=coverage
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Test] test::CrateRustdoc
targets: [x86_64-unknown-linux-gnu]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ expression: test --skip=coverage-map
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Test] test::CrateRustdoc
targets: [x86_64-unknown-linux-gnu]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ expression: test --skip=coverage-run
- Set({compiler/rustc_ty_walk})
- Set({compiler/rustc_type_ir})
- Set({compiler/rustc_type_ir_macros})
- Set({compiler/rustc_unicode})
- Set({compiler/rustc_windows_rc})
[Test] test::CrateRustdoc
targets: [x86_64-unknown-linux-gnu]
Expand Down
Loading
Loading