diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java index 1eda0f0622651..7fc079e6d4a6f 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java @@ -887,6 +887,28 @@ public void testLoadWithRelativePathName() throws Exception { } } + @Test + public void testLoadDataNodeInternalDataDirectoryIsRejectedWithoutLeakingPath() throws Exception { + final DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0); + final File dataDir = new File(dataNodeWrapper.getDataPath()); + + try (final Connection connection = + EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper); + final Statement statement = connection.createStatement()) { + try { + statement.execute(String.format("load \"%s\"", dataDir.getAbsolutePath())); + Assert.fail("Expected LOAD from the DataNode internal data directory to be rejected."); + } catch (final SQLException e) { + Assert.assertTrue( + e.getMessage(), + e.getMessage() + .contains( + "Cannot load files because the specified directory contains IoTDB data.")); + Assert.assertFalse(e.getMessage(), e.getMessage().contains(dataDir.getAbsolutePath())); + } + } + } + @Test public void testLoadWithMods() throws Exception { final long writtenPoint1; diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index b540ebb25404c..acb1382646690 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3013,6 +3013,9 @@ private DataNodeQueryMessages() {} "Can not find %s on this machine, notice that load can only handle files on this machine."; public static final String QUERY_EXCEPTION_LOAD_TSFILE_SOURCE_PATH_S_IS_OUTSIDE_ALLOWED_DIRECTORIES_85A6019F = "Load TsFile source path %s is outside allowed directories %s."; + public static final String + QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D = + "Cannot load files because the specified directory contains IoTDB data."; public static final String QUERY_EXCEPTION_FAILED_TO_RESOLVE_CANONICAL_PATH_FOR_LOAD_TSFILE_SOURCE_09CC9AC6 = "Failed to resolve canonical path for Load TsFile source %s: %s"; public static final String QUERY_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0 = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java index 0e73bd1dd967e..7fe8dcc7781f4 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java @@ -3633,6 +3633,9 @@ private DataNodeQueryMessages() {} public static final String QUERY_EXCEPTION_LOAD_TSFILE_SOURCE_PATH_S_IS_OUTSIDE_ALLOWED_DIRECTORIES_85A6019F = "加载 TsFile 的源路径 %s 位于允许目录 %s 之外。"; + public static final String + QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D = + "指定目录包含 IoTDB 数据，无法加载文件。"; public static final String QUERY_EXCEPTION_FAILED_TO_RESOLVE_CANONICAL_PATH_FOR_LOAD_TSFILE_SOURCE_09CC9AC6 = "无法解析 load TsFile source %s 的 canonical path：%s"; public static final String QUERY_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0 = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 7f151e450d2ce..4f616bce44d39 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -310,6 +310,8 @@ public class IoTDBConfig { private CanonicalPaths loadTsFileAllowedDirCanonicalPaths = canonicalPaths(loadTsFileAllowedDirs); + private volatile CanonicalPaths internalTsFileDirCanonicalPaths = new CanonicalPaths(new Path[0]); + private boolean loadTsFileSourcePathCheckEnabled = false; /** Strategy of multiple directories. */ @@ -1429,6 +1431,7 @@ private void formulateFolders() { queryDir = addDataHomeDir(queryDir); sortTmpDir = addDataHomeDir(sortTmpDir); formulateDataDirs(tierDataDirs); + formulateInternalTsFileDirs(tierDataDirs); } private void formulateDataDirs(String[][] tierDataDirs) { @@ -1480,6 +1483,7 @@ void reloadDataDirs(String[][] newTierDataDirs) throws LoadConfigurationExceptio } } this.tierDataDirs = newTierDataDirs; + formulateInternalTsFileDirs(newTierDataDirs); reloadSystemMetrics(); } @@ -1556,6 +1560,10 @@ public String[] getLocalDataDirs() { .toArray(String[]::new); } + public Path[] getInternalTsFileDirCanonicalPaths() throws FileNotFoundException { + return internalTsFileDirCanonicalPaths.getPaths(); + } + public String[][] getTierDataDirs() { return tierDataDirs; } @@ -1564,6 +1572,7 @@ public String[][] getTierDataDirs() { public void setTierDataDirs(String[][] tierDataDirs) { formulateDataDirs(tierDataDirs); this.tierDataDirs = tierDataDirs; + formulateInternalTsFileDirs(tierDataDirs); // TODO(szywilliam): rewrite the logic here when ratis supports complete snapshot semantic setRatisDataRegionSnapshotDir( tierDataDirs[0][0] + File.separator + IoTDBConstant.SNAPSHOT_FOLDER_NAME); @@ -1663,6 +1672,19 @@ public void formulateLoadTsFileDirs(String[][] tierDataDirs) { this.loadTsFileDirCanonicalPaths = canonicalPaths(newLoadTsFileDirs); } + private void formulateInternalTsFileDirs(final String[][] tierDataDirs) { + final List internalTsFileDirs = new ArrayList<>(); + for (final String[] tierDataDir : tierDataDirs) { + for (final String dataDir : tierDataDir) { + if (FSUtils.isLocal(dataDir)) { + internalTsFileDirs.add(dataDir + File.separator + IoTDBConstant.SEQUENCE_FOLDER_NAME); + internalTsFileDirs.add(dataDir + File.separator + IoTDBConstant.UNSEQUENCE_FOLDER_NAME); + } + } + } + internalTsFileDirCanonicalPaths = canonicalPaths(internalTsFileDirs.toArray(new String[0])); + } + private static CanonicalPaths canonicalPaths(final String[] dirs) { final Path[] paths = new Path[dirs.length]; for (int i = 0; i < dirs.length; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java index 0d632ef87bf7e..595241f513abf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java @@ -109,6 +109,9 @@ public static List processTsFile(final File file) throws FileNotFoundExcep public static List processTsFile(final File file, final boolean validateSourcePath) throws FileNotFoundException { + final Path[] internalTsFileDirCanonicalPaths = + IoTDBDescriptor.getInstance().getConfig().getInternalTsFileDirCanonicalPaths(); + validateNotLoadingInternalTsFile(file, internalTsFileDirCanonicalPaths); if (validateSourcePath) { validateLoadSourcePath(file); } @@ -124,7 +127,7 @@ public static List processTsFile(final File file, final boolean validateSo .QUERY_EXCEPTION_CAN_NOT_FIND_S_ON_THIS_MACHINE_NOTICE_THAT_LOAD_CAN_ONLY_B7886C0E, file.getPath())); } - tsFiles.addAll(findAllTsFile(file, validateSourcePath)); + tsFiles.addAll(findAllTsFile(file, validateSourcePath, internalTsFileDirCanonicalPaths)); } sortTsFiles(tsFiles); return tsFiles; @@ -146,7 +149,8 @@ protected LoadTsFileStatement() { this.statementType = StatementType.MULTI_BATCH_INSERT; } - private static List findAllTsFile(File file, boolean validateSourcePath) + private static List findAllTsFile( + File file, boolean validateSourcePath, Path[] internalTsFileDirCanonicalPaths) throws FileNotFoundException { final File[] files = file.listFiles(); if (files == null) { @@ -155,13 +159,14 @@ private static List findAllTsFile(File file, boolean validateSourcePath) final List tsFiles = new ArrayList<>(); for (File nowFile : files) { + validateNotLoadingInternalTsFile(nowFile, internalTsFileDirCanonicalPaths); if (validateSourcePath) { validateLoadSourcePath(nowFile); } if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) { tsFiles.add(nowFile); } else if (nowFile.isDirectory()) { - tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath)); + tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath, internalTsFileDirCanonicalPaths)); } } return tsFiles; @@ -195,6 +200,19 @@ private static void validateLoadSourcePath(final File file) throws FileNotFoundE Arrays.toString(allowedDirs))); } + private static void validateNotLoadingInternalTsFile( + final File file, final Path[] internalTsFileDirCanonicalPaths) throws FileNotFoundException { + final Path sourcePath = canonicalPath(file); + for (final Path internalTsFileDirCanonicalPath : internalTsFileDirCanonicalPaths) { + if (sourcePath.startsWith(internalTsFileDirCanonicalPath) + || internalTsFileDirCanonicalPath.startsWith(sourcePath)) { + throw new FileNotFoundException( + DataNodeQueryMessages + .QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D); + } + } + } + private static Path canonicalPath(final File file) throws FileNotFoundException { try { return file.getCanonicalFile().toPath(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java index bfebf51d28191..c3ded839d1a44 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java @@ -109,6 +109,37 @@ public void testLoadSourcePathCheckCanBeDisabled() throws Exception { } } + @Test + public void testLoadInternalTsFileIsRejectedWithoutLeakingPath() throws Exception { + final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + final String[][] originalTierDataDirs = config.getTierDataDirs(); + final boolean originalCheckEnabled = config.isLoadTsFileSourcePathCheckEnabled(); + final Path dataDir = Files.createTempDirectory("load-tsfile-internal-data"); + final Path internalTsFile = + Files.createDirectories(dataDir.resolve("sequence").resolve("root.db")).resolve("a.tsfile"); + Files.createFile(internalTsFile); + + try { + config.setTierDataDirs(new String[][] {{dataDir.toString()}}); + config.setLoadTsFileSourcePathCheckEnabled(false); + + try { + new LoadTsFileStatement(dataDir.toString()); + Assert.fail("Expected internal IoTDB data directory to be rejected."); + } catch (final FileNotFoundException e) { + Assert.assertEquals( + "Cannot load files because the specified directory contains IoTDB data.", + e.getMessage()); + Assert.assertFalse(e.getMessage().contains(dataDir.toString())); + Assert.assertFalse(e.getMessage().contains(internalTsFile.toString())); + } + } finally { + config.setTierDataDirs(originalTierDataDirs); + config.setLoadTsFileSourcePathCheckEnabled(originalCheckEnabled); + deleteRecursively(dataDir); + } + } + private static void assertLoadSourcePathRejected(final Path sourcePath) { try { new LoadTsFileStatement(sourcePath.toString());