kafka index service error task

[qianmoQ]

0.16.0 version

I was running the kafkaIndexService, and I configured taskCount to be 1 to generate multiple Tasks at a time, resulting in a large number of Tasks failing

{
    "type": "kafka",
    "dataSchema": {
        "dataSource": "testData",
        "granularitySpec": {
            "queryGranularity": "HOUR",
            "segmentGranularity": "HOUR",
            "type": "uniform"
        },
        "metricsSpec": [
            {
                "type": "count",
                "name": "count"
            },
            {
                "type": "doubleSum",
                "name": "doubleSum_eve_b_point",
                "fieldName": "eve_b_point"
            },
            {
                "type": "doubleMin",
                "name": "doubleMin_eve_b_point",
                "fieldName": "eve_b_point"
            },
            {
                "type": "doubleMax",
                "name": "doubleMax_eve_b_point",
                "fieldName": "eve_b_point"
            },
            {
                "type": "doubleSum",
                "name": "doubleSum_visit",
                "fieldName": "is_visit"
            },
            {
                "type": "doubleSum",
                "name": "doubleSum_bounce",
                "fieldName": "is_bounce"
            },
            {
                "type": "thetaSketch",
                "name": "thetaSketch_userid",
                "fieldName": "sid",
                "isInputThetaSketch": false,
                "size": 8388608
            },
            {
                "type": "tupleBitsetSketch",
                "name": "tupleBitsetSketch_HourOfDay_userid",
                "fieldName": "sid",
                "isInputThetaSketch": false,
                "size": 8388608,
                "unit": "HourOfDay"
            },
            {
                "type": "tupleBitsetSketch",
                "name": "tupleBitsetSketch_DayOfWeek_userid",
                "fieldName": "sid",
                "isInputThetaSketch": false,
                "size": 8388608,
                "unit": "DayOfWeek"
            },
            {
                "type": "tupleBitsetSketch",
                "name": "tupleBitsetSketch_DayOfMonth_userid",
                "fieldName": "sid",
                "isInputThetaSketch": false,
                "size": 8388608,
                "unit": "DayOfMonth"
            },
            {
                "type": "tupleDoubleSketch",
                "name": "tupleDoubleSketch_userid",
                "fieldName": "sid",
                "isInputThetaSketch": false,
                "size": 8388608,
                "maxCount": 48
            }
        ],
        "parser": {
            "parseSpec": {
                "dimensionsSpec": {
                    "dimensionExclusions": [
                        "eve_b_point"
                    ],
                    "dimensions": [
                        "eve_b_second",
                        "eve_b_cv",
                        "event",
                        "eve_b_hour",
                        "eve_b_topic",
                        "eve_b_create_time",
                        "eve_b_minute",
                        "eve_b_ym",
                        "eve_b_stat",
                        "eve_b_md_etype",
                        "eve_b_eid",
                        "eve_b_year"
                    ]
                },
                "format": "json",
                "timestampSpec": {
                    "column": "timestamp",
                    "format": "auto"
                }
            },
            "type": "string"
        }
    },
    "tuningConfig": {
        "type": "kafka",
        "maxRowsPerSegment": 100000000,
        "workerThreads": 2
    },
    "ioConfig": {
        "topic": "inke_olap_online_transferred",
        "consumerProperties": {
            "bootstrap.servers": "cxxxxx"
        },
        "useEarliestOffset": false,
        "taskCount": 1,
        "replicas": 1,
        "taskDuration": "PT15M"
    }
}

error log:

org.apache.druid.java.util.common.ISE: Failed to publish segments because of [java.lang.RuntimeException: Aborting transaction!].
	at org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver.lambda$publishInBackground$8(BaseAppenderatorDriver.java:605) ~[druid-server-0.16.0-incubating.jar:0.16.0-incubating]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_131]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_131]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_131]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
2019-09-28T16:19:11,696 ERROR [publish-0] org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner - Error while publishing segments for sequenceNumber[SequenceMetadata{sequenceId=0, sequenceName='index_kafka_WD452H2WNH18GGN6HRFRSRSG2SDW544S_9c2fb61e50b2b72_0', assignments=[], startOffsets={2=20646665, 6=21142905}, exclusiveStartPartitions=[], endOffsets={2=20653603, 6=21149229}, sentinel=false, checkpointed=true}]
org.apache.druid.java.util.common.ISE: Failed to publish segments because of [java.lang.RuntimeException: Aborting transaction!].
	at org.apache.druid.segment.realtime.appenderator.BaseAppenderatorDriver.lambda$publishInBackground$8(BaseAppenderatorDriver.java:605) ~[druid-server-0.16.0-incubating.jar:0.16.0-incubating]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_131]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [?:1.8.0_131]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [?:1.8.0_131]
	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
2019-09-28T16:19:11,700 INFO [task-runner-0-priority-0] org.apache.druid.segment.realtime.appenderator.AppenderatorImpl - Shutting down immediately...

[vogievetsky]

If you configured taskCount to be 1 it should only generate 1 task at a time.
What does your task view look like in the newer console?

[qianmoQ]

Multiple tasks were generated, but all were in FAILED state

[stale]

This issue has been marked as stale due to 280 days of inactivity. It will be closed in 4 weeks if no further activity occurs. If this issue is still relevant, please simply write any comment. Even if closed, you can still revive the issue at any time or discuss it on the dev@druid.apache.org list. Thank you for your contributions.

[stale]

This issue has been closed due to lack of activity. If you think that is incorrect, or the issue requires additional review, you can revive the issue at any time.