Skip to content

Bug fix for delete when deleting from flushing memtable #1882

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
HTHou merged 1 commit into from
Oct 28, 2020
Merged

Bug fix for delete when deleting from flushing memtable #1882

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
Expand Down Expand Up @@ -203,7 +203,7 @@ public boolean isEmpty() {
@Override
public ReadOnlyMemChunk query(String deviceId, String measurement, TSDataType dataType,
TSEncoding encoding, Map<String, String> props, long timeLowerBound)
throws IOException, QueryProcessException {
throws IOException, QueryProcessException, MetadataException {
if (!checkPath(deviceId, measurement)) {
return null;
}
Expand All @@ -216,13 +216,13 @@ public ReadOnlyMemChunk query(String deviceId, String measurement, TSDataType da
}

private List<TimeRange> constructDeletionList(String deviceId, String measurement,
long timeLowerBound) {
long timeLowerBound) throws MetadataException {
List<TimeRange> deletionList = new ArrayList<>();
deletionList.add(new TimeRange(Long.MIN_VALUE, timeLowerBound));
for (Modification modification : modifications) {
if (modification instanceof Deletion) {
Deletion deletion = (Deletion) modification;
if (deletion.getDevice().equals(deviceId) && deletion.getMeasurement().equals(measurement)
if (deletion.getPath().matchFullPath(new PartialPath(deviceId, measurement))
&& deletion.getEndTime() > timeLowerBound) {
long lowerBound = Math.max(deletion.getStartTime(), timeLowerBound);
deletionList.add(new TimeRange(lowerBound, deletion.getEndTime()));
Expand Down
3 changes: 2 additions & 1 deletion server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
Expand Down Expand Up @@ -77,7 +78,7 @@ void insertTablet(InsertTabletPlan insertTabletPlan, int start, int end)

ReadOnlyMemChunk query(String deviceId, String measurement, TSDataType dataType,
TSEncoding encoding, Map<String, String> props, long timeLowerBound)
throws IOException, QueryProcessException;
throws IOException, QueryProcessException, MetadataException;

/**
* putBack all the memory resources.
Expand Down
9 changes: 5 additions & 4 deletions server/src/test/java/org/apache/iotdb/db/engine/memtable/PrimitiveMemTableTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.Collections;
import java.util.Random;
import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.utils.MathUtils;
import org.apache.iotdb.db.utils.datastructure.TVList;
Expand Down Expand Up @@ -66,7 +67,7 @@ public void memSeriesSortIteratorTest() throws IOException {
}

@Test
public void simpleTest() throws IOException, QueryProcessException {
public void simpleTest() throws IOException, QueryProcessException, MetadataException {
IMemTable memTable = new PrimitiveMemTable();
int count = 10;
String deviceId = "d1";
Expand Down Expand Up @@ -96,7 +97,7 @@ public void simpleTest() throws IOException, QueryProcessException {
}

private void write(IMemTable memTable, String deviceId, String sensorId, TSDataType dataType,
TSEncoding encoding, int size) throws IOException, QueryProcessException {
TSEncoding encoding, int size) throws IOException, QueryProcessException, MetadataException {
TimeValuePair[] ret = genTimeValuePair(size, dataType);

for (TimeValuePair aRet : ret) {
Expand Down Expand Up @@ -134,15 +135,15 @@ private void write(IMemTable memTable, String deviceId, String sensorId, TSDataT
}

@Test
public void testFloatType() throws IOException, QueryProcessException {
public void testFloatType() throws IOException, QueryProcessException, MetadataException {
IMemTable memTable = new PrimitiveMemTable();
String deviceId = "d1";
int size = 100;
write(memTable, deviceId, "s1", TSDataType.FLOAT, TSEncoding.RLE, size);
}

@Test
public void testAllType() throws IOException, QueryProcessException {
public void testAllType() throws IOException, QueryProcessException, MetadataException {
IMemTable memTable = new PrimitiveMemTable();
int count = 10;
String deviceId = "d1";
Expand Down
29 changes: 29 additions & 0 deletions server/src/test/java/org/apache/iotdb/db/integration/IoTDBDeletionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,35 @@ public void testRangeDelete() throws SQLException {
cleanData();
}

@Test
public void testDelFlushingMemtable() throws SQLException {
long size = IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold();
// Adjust memstable threshold size to make it flush automatically
IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(10000);
try (Connection connection = DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
Statement statement = connection.createStatement()) {

for (int i = 1; i <= 10000; i++) {
statement.execute(
String.format(insertTemplate, i, i, i, (double) i, "\'" + i + "\'",
i % 2 == 0));
}

statement.execute("DELETE FROM root.vehicle.d0.s0 WHERE time > 1500 and time <= 9000");
try (ResultSet set = statement.executeQuery("SELECT s0 FROM root.vehicle.d0")) {
int cnt = 0;
while (set.next()) {
cnt++;
}
assertEquals(2500, cnt);
}
cleanData();
}
IoTDBDescriptor.getInstance().getConfig().setMemtableSizeThreshold(size);
}

private static void prepareSeries() {
try (Connection connection = DriverManager
.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
Expand Down