From e777e04c3c4e57e274211eff1544c0cf06ca9d24 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Tue, 21 Jul 2026 18:28:05 +0800 Subject: [PATCH 1/2] feat(load): reject managed TsFile directories --- .../apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 22 +++++++++++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 3 ++ .../iotdb/db/i18n/DataNodeQueryMessages.java | 3 ++ .../statement/crud/LoadTsFileStatement.java | 37 +++++++++++++++++-- .../crud/LoadTsFileStatementTest.java | 31 ++++++++++++++++ 5 files changed, 93 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index 1eda0f0622651..7fc079e6d4a6f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -887,6 +887,28 @@ public void testLoadWithRelativePathName() throws Exception { } } + @Test + public void testLoadDataNodeInternalDataDirectoryIsRejectedWithoutLeakingPath() throws Exception { + final DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + final File dataDir = new File(dataNodeWrapper.getDataPath()); + + try (final Connection connection = + EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper); + final Statement statement = connection.createStatement()) { + try { + statement.execute(String.format("load \"%s\"", dataDir.getAbsolutePath())); + Assert.fail("Expected LOAD from the DataNode internal data directory to be rejected."); + } catch (final SQLException e) { + Assert.assertTrue( + e.getMessage(), + e.getMessage() + .contains( + "Cannot load files because the specified directory contains IoTDB data.")); + Assert.assertFalse(e.getMessage(), e.getMessage().contains(dataDir.getAbsolutePath())); + } + } + } + @Test public void testLoadWithMods() throws Exception { final long writtenPoint1; diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index b540ebb25404c..acb1382646690 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3013,6 +3013,9 @@ private DataNodeQueryMessages() {} "Can not find %s on this machine, notice that load can only handle files on this machine."; public static final String QUERY_EXCEPTION_LOAD_TSFILE_SOURCE_PATH_S_IS_OUTSIDE_ALLOWED_DIRECTORIES_85A6019F = "Load TsFile source path %s is outside allowed directories %s."; + public static final String + QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D = + "Cannot load files because the specified directory contains IoTDB data."; public static final String QUERY_EXCEPTION_FAILED_TO_RESOLVE_CANONICAL_PATH_FOR_LOAD_TSFILE_SOURCE_09CC9AC6 = "Failed to resolve canonical path for Load TsFile source %s: %s"; public static final String QUERY_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0 = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 0e73bd1dd967e..7fe8dcc7781f4 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3633,6 +3633,9 @@ private DataNodeQueryMessages() {} public static final String QUERY_EXCEPTION_LOAD_TSFILE_SOURCE_PATH_S_IS_OUTSIDE_ALLOWED_DIRECTORIES_85A6019F = "加载 TsFile 的源路径 %s 位于允许目录 %s 之外。"; + public static final String + QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D = + "指定目录包含 IoTDB 数据，无法加载文件。"; public static final String QUERY_EXCEPTION_FAILED_TO_RESOLVE_CANONICAL_PATH_FOR_LOAD_TSFILE_SOURCE_09CC9AC6 = "无法解析 load TsFile source %s 的 canonical path：%s"; public static final String QUERY_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0 = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java index 0d632ef87bf7e..31a4697da4d7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java @@ -48,6 +48,8 @@ import java.util.Map; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; +import static org.apache.iotdb.commons.conf.IoTDBConstant.SEQUENCE_FOLDER_NAME; +import static org.apache.iotdb.commons.conf.IoTDBConstant.UNSEQUENCE_FOLDER_NAME; import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.ASYNC_LOAD_KEY; import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.CONVERT_ON_TYPE_MISMATCH_KEY; import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.DATABASE_LEVEL_KEY; @@ -109,6 +111,8 @@ public static List processTsFile(final File file) throws FileNotFoundExcep public static List processTsFile(final File file, final boolean validateSourcePath) throws FileNotFoundException { + final Path[] internalTsFileDirCanonicalPaths = getInternalTsFileDirCanonicalPaths(); + validateNotLoadingInternalTsFile(file, internalTsFileDirCanonicalPaths); if (validateSourcePath) { validateLoadSourcePath(file); } @@ -124,7 +128,7 @@ public static List processTsFile(final File file, final boolean validateSo .QUERY_EXCEPTION_CAN_NOT_FIND_S_ON_THIS_MACHINE_NOTICE_THAT_LOAD_CAN_ONLY_B7886C0E, file.getPath())); } - tsFiles.addAll(findAllTsFile(file, validateSourcePath)); + tsFiles.addAll(findAllTsFile(file, validateSourcePath, internalTsFileDirCanonicalPaths)); } sortTsFiles(tsFiles); return tsFiles; @@ -146,7 +150,8 @@ protected LoadTsFileStatement() { this.statementType = StatementType.MULTI_BATCH_INSERT; } - private static List findAllTsFile(File file, boolean validateSourcePath) + private static List findAllTsFile( + File file, boolean validateSourcePath, Path[] internalTsFileDirCanonicalPaths) throws FileNotFoundException { final File[] files = file.listFiles(); if (files == null) { @@ -155,13 +160,14 @@ private static List findAllTsFile(File file, boolean validateSourcePath) final List tsFiles = new ArrayList<>(); for (File nowFile : files) { + validateNotLoadingInternalTsFile(nowFile, internalTsFileDirCanonicalPaths); if (validateSourcePath) { validateLoadSourcePath(nowFile); } if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) { tsFiles.add(nowFile); } else if (nowFile.isDirectory()) { - tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath)); + tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath, internalTsFileDirCanonicalPaths)); } } return tsFiles; @@ -195,6 +201,31 @@ private static void validateLoadSourcePath(final File file) throws FileNotFoundE Arrays.toString(allowedDirs))); } + private static Path[] getInternalTsFileDirCanonicalPaths() throws FileNotFoundException { + final String[] localDataDirs = IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs(); + final Path[] internalTsFileDirCanonicalPaths = new Path[localDataDirs.length * 2]; + for (int i = 0; i < localDataDirs.length; i++) { + internalTsFileDirCanonicalPaths[i * 2] = + canonicalPath(new File(localDataDirs[i], SEQUENCE_FOLDER_NAME)); + internalTsFileDirCanonicalPaths[i * 2 + 1] = + canonicalPath(new File(localDataDirs[i], UNSEQUENCE_FOLDER_NAME)); + } + return internalTsFileDirCanonicalPaths; + } + + private static void validateNotLoadingInternalTsFile( + final File file, final Path[] internalTsFileDirCanonicalPaths) throws FileNotFoundException { + final Path sourcePath = canonicalPath(file); + for (final Path internalTsFileDirCanonicalPath : internalTsFileDirCanonicalPaths) { + if (sourcePath.startsWith(internalTsFileDirCanonicalPath) + || internalTsFileDirCanonicalPath.startsWith(sourcePath)) { + throw new FileNotFoundException( + DataNodeQueryMessages + .QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D); + } + } + } + private static Path canonicalPath(final File file) throws FileNotFoundException { try { return file.getCanonicalFile().toPath(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java index bfebf51d28191..c3ded839d1a44 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java @@ -109,6 +109,37 @@ public void testLoadSourcePathCheckCanBeDisabled() throws Exception { } } + @Test + public void testLoadInternalTsFileIsRejectedWithoutLeakingPath() throws Exception { + final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + final String[][] originalTierDataDirs = config.getTierDataDirs(); + final boolean originalCheckEnabled = config.isLoadTsFileSourcePathCheckEnabled(); + final Path dataDir = Files.createTempDirectory("load-tsfile-internal-data"); + final Path internalTsFile = + Files.createDirectories(dataDir.resolve("sequence").resolve("root.db")).resolve("a.tsfile"); + Files.createFile(internalTsFile); + + try { + config.setTierDataDirs(new String[][] {{dataDir.toString()}}); + config.setLoadTsFileSourcePathCheckEnabled(false); + + try { + new LoadTsFileStatement(dataDir.toString()); + Assert.fail("Expected internal IoTDB data directory to be rejected."); + } catch (final FileNotFoundException e) { + Assert.assertEquals( + "Cannot load files because the specified directory contains IoTDB data.", + e.getMessage()); + Assert.assertFalse(e.getMessage().contains(dataDir.toString())); + Assert.assertFalse(e.getMessage().contains(internalTsFile.toString())); + } + } finally { + config.setTierDataDirs(originalTierDataDirs); + config.setLoadTsFileSourcePathCheckEnabled(originalCheckEnabled); + deleteRecursively(dataDir); + } + } + private static void assertLoadSourcePathRejected(final Path sourcePath) { try { new LoadTsFileStatement(sourcePath.toString()); From fdccefc505a2a55a5d4356161ffc2c2742da6f01 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Mon, 27 Jul 2026 10:52:00 +0800 Subject: [PATCH 2/2] perf(load): cache internal TsFile directories --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 22 +++++++++++++++++++ .../statement/crud/LoadTsFileStatement.java | 17 ++------------ 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 7f151e450d2ce..4f616bce44d39 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -310,6 +310,8 @@ public class IoTDBConfig { private CanonicalPaths loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs); + private volatile CanonicalPaths internalTsFileDirCanonicalPaths = new CanonicalPaths(new Path[0]); + private boolean loadTsFileSourcePathCheckEnabled = false; /** Strategy of multiple directories. */ @@ -1429,6 +1431,7 @@ private void formulateFolders() { queryDir = addDataHomeDir(queryDir); sortTmpDir = addDataHomeDir(sortTmpDir); formulateDataDirs(tierDataDirs); + formulateInternalTsFileDirs(tierDataDirs); } private void formulateDataDirs(String[][] tierDataDirs) { @@ -1480,6 +1483,7 @@ void reloadDataDirs(String[][] newTierDataDirs) throws LoadConfigurationExceptio } } this.tierDataDirs = newTierDataDirs; + formulateInternalTsFileDirs(newTierDataDirs); reloadSystemMetrics(); } @@ -1556,6 +1560,10 @@ public String[] getLocalDataDirs() { .toArray(String[]::new); } + public Path[] getInternalTsFileDirCanonicalPaths() throws FileNotFoundException { + return internalTsFileDirCanonicalPaths.getPaths(); + } + public String[][] getTierDataDirs() { return tierDataDirs; } @@ -1564,6 +1572,7 @@ public String[][] getTierDataDirs() { public void setTierDataDirs(String[][] tierDataDirs) { formulateDataDirs(tierDataDirs); this.tierDataDirs = tierDataDirs; + formulateInternalTsFileDirs(tierDataDirs); // TODO(szywilliam): rewrite the logic here when ratis supports complete snapshot semantic setRatisDataRegionSnapshotDir( tierDataDirs[0][0] + File.separator + IoTDBConstant.SNAPSHOT_FOLDER_NAME); @@ -1663,6 +1672,19 @@ public void formulateLoadTsFileDirs(String[][] tierDataDirs) { this.loadTsFileDirCanonicalPaths = canonicalPaths(newLoadTsFileDirs); } + private void formulateInternalTsFileDirs(final String[][] tierDataDirs) { + final List internalTsFileDirs = new ArrayList<>(); + for (final String[] tierDataDir : tierDataDirs) { + for (final String dataDir : tierDataDir) { + if (FSUtils.isLocal(dataDir)) { + internalTsFileDirs.add(dataDir + File.separator + IoTDBConstant.SEQUENCE_FOLDER_NAME); + internalTsFileDirs.add(dataDir + File.separator + IoTDBConstant.UNSEQUENCE_FOLDER_NAME); + } + } + } + internalTsFileDirCanonicalPaths = canonicalPaths(internalTsFileDirs.toArray(new String[0])); + } + private static CanonicalPaths canonicalPaths(final String[] dirs) { final Path[] paths = new Path[dirs.length]; for (int i = 0; i < dirs.length; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java index 31a4697da4d7f..595241f513abf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java @@ -48,8 +48,6 @@ import java.util.Map; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; -import static org.apache.iotdb.commons.conf.IoTDBConstant.SEQUENCE_FOLDER_NAME; -import static org.apache.iotdb.commons.conf.IoTDBConstant.UNSEQUENCE_FOLDER_NAME; import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.ASYNC_LOAD_KEY; import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.CONVERT_ON_TYPE_MISMATCH_KEY; import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.DATABASE_LEVEL_KEY; @@ -111,7 +109,8 @@ public static List processTsFile(final File file) throws FileNotFoundExcep public static List processTsFile(final File file, final boolean validateSourcePath) throws FileNotFoundException { - final Path[] internalTsFileDirCanonicalPaths = getInternalTsFileDirCanonicalPaths(); + final Path[] internalTsFileDirCanonicalPaths = + IoTDBDescriptor.getInstance().getConfig().getInternalTsFileDirCanonicalPaths(); validateNotLoadingInternalTsFile(file, internalTsFileDirCanonicalPaths); if (validateSourcePath) { validateLoadSourcePath(file); @@ -201,18 +200,6 @@ private static void validateLoadSourcePath(final File file) throws FileNotFoundE Arrays.toString(allowedDirs))); } - private static Path[] getInternalTsFileDirCanonicalPaths() throws FileNotFoundException { - final String[] localDataDirs = IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs(); - final Path[] internalTsFileDirCanonicalPaths = new Path[localDataDirs.length * 2]; - for (int i = 0; i < localDataDirs.length; i++) { - internalTsFileDirCanonicalPaths[i * 2] = - canonicalPath(new File(localDataDirs[i], SEQUENCE_FOLDER_NAME)); - internalTsFileDirCanonicalPaths[i * 2 + 1] = - canonicalPath(new File(localDataDirs[i], UNSEQUENCE_FOLDER_NAME)); - } - return internalTsFileDirCanonicalPaths; - } - private static void validateNotLoadingInternalTsFile( final File file, final Path[] internalTsFileDirCanonicalPaths) throws FileNotFoundException { final Path sourcePath = canonicalPath(file);