From 37b2b0dbd358b8aec678e5970f98ee662b0713f2 Mon Sep 17 00:00:00 2001 From: HTHou Date: Mon, 19 Jan 2026 18:42:08 +0800 Subject: [PATCH 1/3] cherry-pick Fix query error after insert an all null aligned tablet and flush (#14777) --- .../session/it/IoTDBSessionInsertNullIT.java | 68 +++++++++++++++++++ .../dataregion/flush/MemTableFlushTask.java | 2 +- .../memtable/AlignedWritableMemChunk.java | 23 ++++++- .../AlignedWritableMemChunkGroup.java | 5 ++ .../memtable/IWritableMemChunkGroup.java | 2 + .../memtable/WritableMemChunkGroup.java | 5 ++ 6 files changed, 103 insertions(+), 2 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java index 503b61f2d7cd7..9c6b228763c40 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java @@ -359,6 +359,74 @@ public void insertAlignedRecordsOfOneDeviceNullTest() { } } + @Test + public void insertTabletNullTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + prepareData(session); + + String deviceId = "root.sg1.clsu.d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.INT32)), + 3); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, null); + tablet.addValue("s2", 0, null); + tablet.addTimestamp(1, 400); + tablet.addValue("s1", 1, null); + tablet.addValue("s2", 1, null); + tablet.addTimestamp(2, 500); + tablet.addValue("s1", 2, null); + tablet.addValue("s2", 2, null); + session.insertTablet(tablet); + long nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + session.executeNonQueryStatement("flush"); + nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void insertAlignedTabletNullTest() { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + prepareData(session); + + String deviceId = "root.sg1.clsu.aligned_d1"; + Tablet tablet = + new Tablet( + deviceId, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.BOOLEAN), + new MeasurementSchema("s2", TSDataType.INT32)), + 3); + tablet.addTimestamp(0, 300); + tablet.addValue("s1", 0, null); + tablet.addValue("s2", 0, null); + tablet.addTimestamp(1, 400); + tablet.addValue("s1", 1, null); + tablet.addValue("s2", 1, null); + tablet.addTimestamp(2, 500); + tablet.addValue("s1", 2, null); + tablet.addValue("s2", 2, null); + session.insertAlignedTablet(tablet); + long nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + session.executeNonQueryStatement("flush"); + nums = queryCountRecords(session, "select count(s1) from " + deviceId); + assertEquals(0, nums); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + @Test public void insertTabletNullMeasurementTest() { try (ISession session = EnvFactory.getEnv().getSessionConnection()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 61abb78c24f9d..48efe55c223d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -159,7 +159,7 @@ public void syncFlushMemTable() throws ExecutionException, InterruptedException for (IDeviceID deviceID : deviceIDList) { final Map value = memTableMap.get(deviceID).getMemChunkMap(); // skip the empty device/chunk group - if (memTableMap.get(deviceID).count() == 0 || value.isEmpty()) { + if (memTableMap.get(deviceID).isEmpty() || value.isEmpty()) { continue; } encodingTaskQueue.put(new StartFlushGroupIOTask(deviceID)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index 50ae45b432c7e..2fca3ca398a51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -593,7 +593,28 @@ public long getLastPoint() { @Override public boolean isEmpty() { - return rowCount() == 0; + if (rowCount() == 0) { + return true; + } + if (measurementIndexMap.isEmpty()) { + return true; + } + + if (list.rowCount() > 0) { + BitMap allValueColDeletedMap = list.getAllValueColDeletedMap(); + if (allValueColDeletedMap == null || !allValueColDeletedMap.isAllMarked()) { + return false; + } + } + for (AlignedTVList alignedTvList : sortedList) { + if (alignedTvList.rowCount() > 0) { + BitMap allValueColDeletedMap = alignedTvList.getAllValueColDeletedMap(); + if (allValueColDeletedMap == null || !allValueColDeletedMap.isAllMarked()) { + return false; + } + } + } + return true; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java index bc2031c069a8a..6a9571fbc6ca1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java @@ -102,6 +102,11 @@ public Map getMemChunkMap() { return Collections.singletonMap("", memChunk); } + @Override + public boolean isEmpty() { + return memChunk.isEmpty(); + } + @SuppressWarnings("squid:S3776") @Override public int delete( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java index 79baafe86fda3..1738a8b92542e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java @@ -48,6 +48,8 @@ void writeTablet( Map getMemChunkMap(); + boolean isEmpty(); + int delete( PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index 313fcd71540ac..226c2fd398098 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -109,6 +109,11 @@ public Map getMemChunkMap() { return memChunkMap; } + @Override + public boolean isEmpty() { + return memChunkMap.isEmpty() || count() == 0; + } + @SuppressWarnings("squid:S3776") @Override public int delete( From 12de4a1fd282d3e48260f46c764e79da98586c62 Mon Sep 17 00:00:00 2001 From: Haonan Date: Sat, 22 Mar 2025 00:22:59 +0800 Subject: [PATCH 2/3] Fix broken file genaterated after insert null to aligned timeseries (#15163) --- .../session/it/IoTDBSessionInsertNullIT.java | 28 +++++++++++++++++++ .../dataregion/memtable/AbstractMemTable.java | 5 ++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java index 9c6b228763c40..439f05b83aec3 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionInsertNullIT.java @@ -21,6 +21,7 @@ import org.apache.iotdb.isession.ISession; import org.apache.iotdb.isession.SessionDataSet; import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; @@ -43,11 +44,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; @RunWith(IoTDBTestRunner.class) @@ -421,6 +424,31 @@ public void insertAlignedTabletNullTest() { session.executeNonQueryStatement("flush"); nums = queryCountRecords(session, "select count(s1) from " + deviceId); assertEquals(0, nums); + for (DataNodeWrapper dn : EnvFactory.getEnv().getDataNodeWrapperList()) { + File dir = + new File( + dn.getDataDir() + + File.separator + + "datanode" + + File.separator + + "data" + + File.separator + + "sequence" + + File.separator + + "root.sg1" + + File.separator + + "1" + + File.separator + + "0"); + if (dir.exists() && dir.isDirectory()) { + File[] files = dir.listFiles(); + if (files != null) { + for (File file : files) { + assertFalse(file.getName().endsWith("broken")); + } + } + } + } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 9d1078a471eb7..eb49d5325b0b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -972,9 +972,8 @@ protected void deserialize(DataInputStream stream, boolean multiTvListMemChunk) public Map getMaxTime() { Map latestTimeForEachDevice = new HashMap<>(); for (Entry entry : memTableMap.entrySet()) { - long maxTime = entry.getValue().getMaxTime(); - if (entry.getValue().count() > 0) { - latestTimeForEachDevice.put(entry.getKey(), maxTime); + if (entry.getValue().count() > 0 && !entry.getValue().isEmpty()) { + latestTimeForEachDevice.put(entry.getKey(), entry.getValue().getMaxTime()); } } return latestTimeForEachDevice; From ebcd844481a056a2e5b64aee3edfa5bd430b5bee Mon Sep 17 00:00:00 2001 From: HTHou Date: Mon, 19 Jan 2026 20:03:44 +0800 Subject: [PATCH 3/3] fix compile --- .../org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index 3df21cb42a697..bf4189d599b36 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -145,6 +145,10 @@ public String getSystemPropertiesPath() { return workDirFilePath("data/datanode/system/schema", IoTDBStartCheck.PROPERTIES_FILE_NAME); } + public String getDataDir() { + return getNodePath() + File.separator + "data"; + } + @Override protected MppJVMConfig initVMConfig() { return MppJVMConfig.builder()