Skip to content

Commit

Permalink
Merge branch 'master' into fix-segment-cache-races
Browse files Browse the repository at this point in the history
  • Loading branch information
@gianm
gianm committed Apr 25, 2023
2 parents dc1496a + 89e7948 commit 0deae93
Show file tree
Hide file tree
Showing 28 changed files with 2,180 additions and 1,910 deletions.
2 changes: 1 addition & 1 deletion docs/multi-stage-query/reference.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ The following table describes error codes you may encounter in the `multiStageQu
| <a name="error_InsertCannotBeEmpty">`InsertCannotBeEmpty`</a> | An INSERT or REPLACE query did not generate any output rows in a situation where output rows are required for success. This can happen for INSERT or REPLACE queries with `PARTITIONED BY` set to something other than `ALL` or `ALL TIME`. | `dataSource` |
| <a name="error_InsertCannotOrderByDescending">`InsertCannotOrderByDescending`</a> | An INSERT query contained a `CLUSTERED BY` expression in descending order. Druid's segment generation code only supports ascending order. | `columnName` |
| <a name="error_InsertLockPreempted">`InsertLockPreempted`</a> | An INSERT or REPLACE query was canceled by a higher-priority ingestion job, such as a real-time ingestion task. | |
| <a name="error_InsertTimeNull">`InsertTimeNull`</a> | An INSERT or REPLACE query encountered a null timestamp in the `__time` field.<br /><br />This can happen due to using an expression like `TIME_PARSE(timestamp) AS __time` with a timestamp that cannot be parsed. ([`TIME_PARSE`](../querying/sql-scalar.md#date-and-time-functions) returns null when it cannot parse a timestamp.) In this case, try parsing your timestamps using a different function or pattern. Or, if your timestamps may genuinely be null, consider using [`COALESCE`](../querying/sql-scalar.md#other-scalar-functions) to provide a default value. One option is [`CURRENT_TIMESTAMP`](../querying/sql-scalar.md#date-and-time-functions), which represents the start time of the job.<br /><br />This error code only occurs when [`druid.generic.useDefaultValueForNull = false`](../configuration/index.md#sql-compatible-null-handling). When this property is `true` (the default setting), null timestamps are instead replaced by `1970-01-01 00:00:00`.|
| <a name="error_InsertTimeNull">`InsertTimeNull`</a> | An INSERT or REPLACE query encountered a null timestamp in the `__time` field.<br /><br />This can happen due to using an expression like `TIME_PARSE(timestamp) AS __time` with a timestamp that cannot be parsed. ([`TIME_PARSE`](../querying/sql-scalar.md#date-and-time-functions) returns null when it cannot parse a timestamp.) In this case, try parsing your timestamps using a different function or pattern. Or, if your timestamps may genuinely be null, consider using [`COALESCE`](../querying/sql-scalar.md#other-scalar-functions) to provide a default value. One option is [`CURRENT_TIMESTAMP`](../querying/sql-scalar.md#date-and-time-functions), which represents the start time of the job.|
| <a name="error_InsertTimeOutOfBounds">`InsertTimeOutOfBounds`</a> | A REPLACE query generated a timestamp outside the bounds of the TIMESTAMP parameter for your OVERWRITE WHERE clause.<br /> <br />To avoid this error, verify that the you specified is valid. | `interval`: time chunk interval corresponding to the out-of-bounds timestamp |
| <a name="error_InvalidNullByte">`InvalidNullByte`</a> | A string column included a null byte. Null bytes in strings are not permitted. | `column`: The column that included the null byte |
| <a name="error_QueryNotSupported">`QueryNotSupported`</a> | QueryKit could not translate the provided native query to a multi-stage query.<br /> <br />This can happen if the query uses features that aren't supported, like GROUPING SETS. | |
Expand Down
5 changes: 5 additions & 0 deletions extensions-core/multi-stage-query/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,11 @@
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>pl.pragmatists</groupId>
<artifactId>JUnitParams</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-processing</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion .../multi-stage-query/src/main/java/org/apache/druid/msq/querykit/InputNumberDataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public boolean isGlobal()
@Override
public boolean isConcrete()
{
return false;
return true;
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion ...query/src/main/java/org/apache/druid/msq/querykit/common/SortMergeJoinFrameProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ public boolean hasMark()
*/
public boolean hasCompletelyNonNullMark()
{
return hasMark() && !holders.get(markFrame).comparisonWidget.isPartiallyNullKey(markRow);
return hasMark() && holders.get(markFrame).comparisonWidget.isCompletelyNonNullKey(markRow);
}

/**
Expand Down
42 changes: 6 additions & 36 deletions extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQFaultsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,8 @@

package org.apache.druid.msq.exec;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.indexing.common.actions.SegmentAllocateAction;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.msq.indexing.error.InsertCannotAllocateSegmentFault;
Expand All @@ -41,7 +37,6 @@
import org.apache.druid.msq.test.MSQTestFileUtils;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.timeline.SegmentId;
import org.junit.Test;
import org.mockito.Mockito;

Expand Down Expand Up @@ -149,37 +144,12 @@ public void testInsertTimeNullFault()
+ "PARTITIONED BY DAY\n"
+ "CLUSTERED BY dim1";

if (NullHandling.sqlCompatible()) {
testIngestQuery()
.setSql(sql)
.setExpectedDataSource(expectedDataSource)
.setExpectedRowSignature(rowSignature)
.setExpectedMSQFault(InsertTimeNullFault.instance())
.verifyResults();
} else {
testIngestQuery()
.setSql(sql)
.setExpectedDataSource(expectedDataSource)
.setExpectedRowSignature(rowSignature)
.setExpectedSegment(
ImmutableSet.of(
SegmentId.of(expectedDataSource, Intervals.of("0001-01-01T/P1D"), "test", 0),
SegmentId.of(expectedDataSource, Intervals.of("0002-01-01T/P1D"), "test", 0),
SegmentId.of(expectedDataSource, Intervals.of("1970-01-01T/P1D"), "test", 0)
)
)
.setExpectedResultRows(
ImmutableList.of(
new Object[]{DateTimes.of("0001-01-01").getMillis(), "1"},
new Object[]{DateTimes.of("0002-01-01").getMillis(), "2"},
new Object[]{DateTimes.of("1970-01-01").getMillis(), null},
new Object[]{DateTimes.of("1970-01-01").getMillis(), "10.1"},
new Object[]{DateTimes.of("1970-01-01").getMillis(), "abc"},
new Object[]{DateTimes.of("1970-01-01").getMillis(), "def"}
)
)
.verifyResults();
}
testIngestQuery()
.setSql(sql)
.setExpectedDataSource(expectedDataSource)
.setExpectedRowSignature(rowSignature)
.setExpectedMSQFault(InsertTimeNullFault.instance())
.verifyResults();
}

@Test
Expand Down
74 changes: 37 additions & 37 deletions ...y/src/test/java/org/apache/druid/msq/querykit/common/SortMergeJoinFrameProcessorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,35 +202,35 @@ public void testLeftJoinEmptyRightSide() throws Exception
);

final List<List<Object>> expectedRows = Arrays.asList(
Arrays.asList("Agama mossambica", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Apamea abruzzorum", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Atractus flammigerus", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Rallicula", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Talk:Oswald Tilghman", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Peremptory norm", "AU", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Didier Leclair", "CA", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Les Argonautes", "CA", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Sarah Michelle Gellar", "CA", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Golpe de Estado en Chile de 1973", "CL", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Diskussion:Sebastian Schulz", "DE", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Gabinete Ministerial de Rafael Correa", "EC", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Saison 9 de Secret Story", "FR", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Glasgow", "GB", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Giusy Ferreri discography", "IT", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Roma-Bangkok", "IT", null, null, NullHandling.defaultLongValue()),
Arrays.asList("青野武", "JP", null, null, NullHandling.defaultLongValue()),
Arrays.asList("유희왕 GX", "KR", null, null, NullHandling.defaultLongValue()),
Arrays.asList("History of Fourems", "MMMM", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Mathis Bolly", "MX", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Orange Soda", "MatchNothing", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Алиса в Зазеркалье", "NO", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Cream Soda", "SU", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Wendigo", "SV", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Carlo Curti", "US", null, null, NullHandling.defaultLongValue()),
Arrays.asList("DirecTV", "US", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Old Anatolian Turkish", "US", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Otjiwarongo Airport", "US", null, null, NullHandling.defaultLongValue()),
Arrays.asList("President of India", "US", null, null, NullHandling.defaultLongValue())
Arrays.asList("Agama mossambica", null, null, null, null),
Arrays.asList("Apamea abruzzorum", null, null, null, null),
Arrays.asList("Atractus flammigerus", null, null, null, null),
Arrays.asList("Rallicula", null, null, null, null),
Arrays.asList("Talk:Oswald Tilghman", null, null, null, null),
Arrays.asList("Peremptory norm", "AU", null, null, null),
Arrays.asList("Didier Leclair", "CA", null, null, null),
Arrays.asList("Les Argonautes", "CA", null, null, null),
Arrays.asList("Sarah Michelle Gellar", "CA", null, null, null),
Arrays.asList("Golpe de Estado en Chile de 1973", "CL", null, null, null),
Arrays.asList("Diskussion:Sebastian Schulz", "DE", null, null, null),
Arrays.asList("Gabinete Ministerial de Rafael Correa", "EC", null, null, null),
Arrays.asList("Saison 9 de Secret Story", "FR", null, null, null),
Arrays.asList("Glasgow", "GB", null, null, null),
Arrays.asList("Giusy Ferreri discography", "IT", null, null, null),
Arrays.asList("Roma-Bangkok", "IT", null, null, null),
Arrays.asList("青野武", "JP", null, null, null),
Arrays.asList("유희왕 GX", "KR", null, null, null),
Arrays.asList("History of Fourems", "MMMM", null, null, null),
Arrays.asList("Mathis Bolly", "MX", null, null, null),
Arrays.asList("Orange Soda", "MatchNothing", null, null, null),
Arrays.asList("Алиса в Зазеркалье", "NO", null, null, null),
Arrays.asList("Cream Soda", "SU", null, null, null),
Arrays.asList("Wendigo", "SV", null, null, null),
Arrays.asList("Carlo Curti", "US", null, null, null),
Arrays.asList("DirecTV", "US", null, null, null),
Arrays.asList("Old Anatolian Turkish", "US", null, null, null),
Arrays.asList("Otjiwarongo Airport", "US", null, null, null),
Arrays.asList("President of India", "US", null, null, null)
);

assertResult(processor, outputChannel.readable(), joinSignature, expectedRows);
Expand Down Expand Up @@ -317,11 +317,11 @@ public void testLeftJoinCountryIsoCode() throws Exception
);

final List<List<Object>> expectedRows = Arrays.asList(
Arrays.asList("Agama mossambica", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Apamea abruzzorum", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Atractus flammigerus", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Rallicula", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Talk:Oswald Tilghman", null, null, null, NullHandling.defaultLongValue()),
Arrays.asList("Agama mossambica", null, null, null, null),
Arrays.asList("Apamea abruzzorum", null, null, null, null),
Arrays.asList("Atractus flammigerus", null, null, null, null),
Arrays.asList("Rallicula", null, null, null, null),
Arrays.asList("Talk:Oswald Tilghman", null, null, null, null),
Arrays.asList("Peremptory norm", "AU", "AU", "Australia", 0L),
Arrays.asList("Didier Leclair", "CA", "CA", "Canada", 1L),
Arrays.asList("Les Argonautes", "CA", "CA", "Canada", 1L),
Expand All @@ -337,7 +337,7 @@ public void testLeftJoinCountryIsoCode() throws Exception
Arrays.asList("유희왕 GX", "KR", "KR", "Republic of Korea", 9L),
Arrays.asList("History of Fourems", "MMMM", "MMMM", "Fourems", 205L),
Arrays.asList("Mathis Bolly", "MX", "MX", "Mexico", 10L),
Arrays.asList("Orange Soda", "MatchNothing", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Orange Soda", "MatchNothing", null, null, null),
Arrays.asList("Алиса в Зазеркалье", "NO", "NO", "Norway", 11L),
Arrays.asList("Cream Soda", "SU", "SU", "States United", 15L),
Arrays.asList("Wendigo", "SV", "SV", "El Salvador", 12L),
Expand Down Expand Up @@ -778,7 +778,7 @@ public void testLeftJoinCountryNumber() throws Exception
Arrays.asList("Otjiwarongo Airport", "US", "US", "United States", 13L),
Arrays.asList("President of India", "US", "US", "United States", 13L),
Arrays.asList("Cream Soda", "SU", "SU", "States United", 15L),
Arrays.asList("Orange Soda", "MatchNothing", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Orange Soda", "MatchNothing", null, null, null),
Arrays.asList("History of Fourems", "MMMM", "MMMM", "Fourems", 205L)
);

Expand Down Expand Up @@ -871,7 +871,7 @@ public void testRightJoinCountryNumber() throws Exception
Arrays.asList("Otjiwarongo Airport", "US", "US", "United States", 13L),
Arrays.asList("President of India", "US", "US", "United States", 13L),
Arrays.asList("Cream Soda", "SU", "SU", "States United", 15L),
Arrays.asList("Orange Soda", "MatchNothing", null, null, NullHandling.defaultLongValue()),
Arrays.asList("Orange Soda", "MatchNothing", null, null, null),
Arrays.asList("History of Fourems", "MMMM", "MMMM", "Fourems", 205L)
);

Expand Down
1 change: 1 addition & 0 deletions ...core/multi-stage-query/src/test/java/org/apache/druid/msq/test/CalciteMSQTestsHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ private static Supplier<ResourceHolder<Segment>> getSupplierForSegment(SegmentId
.buildMMappedIndex();
break;
case DATASOURCE3:
case CalciteTests.BROADCAST_DATASOURCE:
index = IndexBuilder
.create()
.tmpDir(new File(temporaryFolder.newFolder(), "3"))
Expand Down
Loading

0 comments on commit 0deae93

Please sign in to comment.