[bug] MySQL -> ElasticSearch 源表过大DataX 批量导入直接卡死

[baisui1981]

CREATE TABLE `item` (
  `id` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '主键',
  `entity_id` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '实体id',
  `platform_id` int(10) NOT NULL,
  `category_id` int(11) NOT NULL COMMENT '类目id',
  `shop_category_id` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '店铺分类id',
  `name` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '名称',
  `code` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `inner_code` varchar(100) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `original_price` bigint(20) NOT NULL COMMENT '原价、吊牌价',
  `main_picture` varchar(500) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '主图',
  `server` varchar(200) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `detail` text COLLATE utf8mb4_unicode_ci,
  `unit_id` bigint(20) NOT NULL COMMENT '单位',
  `tag` varchar(500) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '标记',
  `label` varchar(500) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '商品标签',
  `sort_code` int(11) NOT NULL COMMENT '排序码',
  `props` varchar(1000) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '属性kv对',
  `status` tinyint(4) NOT NULL COMMENT '状态',
  `ext` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '扩展字段',
  `is_valid` tinyint(4) NOT NULL COMMENT '是否有效:0.失效 1有效',
  `create_time` bigint(20) NOT NULL COMMENT '创建时间',
  `op_time` bigint(20) NOT NULL COMMENT '修改时间',
  `last_ver` int(11) NOT NULL COMMENT '版本号',
  `spell` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '拼写对应的属性',
  `spell2` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '拼写对应的属性2',
  `biz_type` tinyint(3) DEFAULT '0' COMMENT '提供给业务方标示商品类型',
  `type` tinyint(3) NOT NULL DEFAULT '0' COMMENT '0-普通商品;1-预约商品',
  `outer_id` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '外部ID',
  `outer_ext` varchar(1000) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '外部扩展字段',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC

使用如下语句多次，将原表数量达到3000w，执行dataX 导入，直接卡死，尝试在java 启动命令上设置大的jvm堆内存也无效果：

java  -Xmx2G -Xms2G -Ddata.dir=/tmp/tis-uber/data -Denv_props=false -Dlog.dir=/tmp/tis-uber/logs -Druntime=daily -Dlogback.configurationFile=logback-datax.xml -DexecTimeStamp=1683374841695 -DnotFetchFromCenterRepository=true -classpath /tmp/tis-uber/tis-assemble/lib/*:/tmp/tis-uber/data/libs/plugins/tis-datax-local-executor/WEB-INF/lib/*:/tmp/tis-uber/tis-assemble/conf:/tmp/tis-uber/web-start/lib/* com.qlangtech.tis.datax.DataxExecutor 13 item_0.json/item/item mysql_es 127.0.0.1:56432 local -1 ap 1 1683374841695

insert ignore into item
select replace( @ID := uuid(),'-','' ) as id,
       `entity_id` ,
       `platform_id` ,
       `category_id` ,
       `shop_category_id` ,
       `name` ,
       `code` ,
       `inner_code` ,
       `original_price` ,
       `main_picture` ,
       `server` ,
       `detail` ,
       `unit_id` ,
       `tag` ,
       `label` ,
       `sort_code` ,
       `props` ,
       `status` ,
       `ext` ,
       `is_valid` ,
       `create_time` ,
       `op_time` ,
       `last_ver` ,
       `spell` ,
       `spell2` ,
       `biz_type` ,
       `type` ,
       `outer_id` ,
       `outer_ext`
from item

[baisui1981]

修复了