# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # The below commented values will effect as default settings # Uncomment and override them if necessary kylin.metadata.url=kylin_metadata@jdbc,url=jdbc:mysql://ip-xxx-xx-x-xx:3306/kylin,username=user,password=password,maxActive=10,maxIdle=10 # kylin.env.hdfs-working-dir=s3a://bucket-name/kylin4_local_cache/ ## Connect to a remote zookeeper with the url, should set kylin.env.zookeeper-is-local to false kylin.env.zookeeper-connect-string=ip-xxx-xx-x-xx ## Disable retry kylin.engine.max-retry-time=1 ## Build Engine Resource kylin.engine.spark-conf.spark.executor.cores=2 kylin.engine.spark-conf.spark.executor.instances=8 kylin.engine.spark-conf.spark.executor.memory=7GB kylin.engine.spark-conf.spark.executor.memoryOverhead=1GB kylin.engine.spark-conf.spark.sql.hive.metastore.version=2.3.7 kylin.engine.spark-conf.spark.sql.hive.metastore.jars=/usr/lib/hive/lib/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop-hdfs/*:/usr/lib/hadoop-yarn/*:/usr/lib/hadoop-mapreduce/* kylin.engine.spark-conf.spark.hadoop.fs.s3a.endpoint=s3.cn-northwest-1.amazonaws.com.cn kylin.engine.spark-conf.spark.hadoop.fs.s3a.connection.ssl.enabled=false kylin.engine.spark-conf.spark.hadoop.fs.s3a.access.key=xxx kylin.engine.spark-conf.spark.hadoop.fs.s3a.secret.key=xxx kylin.query.spark-conf.spark.sql.hive.metastore.version=2.3.7 kylin.query.spark-conf.spark.sql.hive.metastore.jars=/usr/lib/hive/lib/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop-hdfs/*:/usr/lib/hadoop-yarn/*:/usr/lib/hadoop-mapreduce/* ## Query Engine Resource kylin.query.spark-conf.spark.master=yarn kylin.query.spark-conf.spark.driver.cores=1 kylin.query.spark-conf.spark.driver.memory=8GB kylin.query.spark-conf.spark.driver.memoryOverhead=1G kylin.query.spark-conf.spark.executor.cores=2 kylin.query.spark-conf.spark.executor.instances=30 kylin.query.spark-conf.spark.executor.memory=8G kylin.query.spark-conf.spark.executor.memoryOverhead=2G kylin.query.spark-conf.spark.sql.parquet.filterPushdown=false kylin.query.spark-conf.spark.sql.files.maxPartitionBytes=20MB ## Disable canary kylin.canary.sparder-context-canary-enabled=false ## Shard setting kylin.storage.columnar.shard-size-mb=50 kylin.storage.columnar.shard-rowcount=600000 kylin.storage.columnar.shard-countdistinct-rowcount=300000 kylin.storage.columnar.repartition-threshold-size-mb=50 kylin.engine.spark-conf.spark.hadoop.parquet.block.size=268435456 kylin.query.spark-conf.spark.executor.extraJavaOptions=-Dhdp.version=current -Dlog4j.configuration=spark-executor-log4j.properties -Dlog4j.debug -Dkylin.hdfs.working.dir=${kylin.env.hdfs-working-dir} -Dkylin.metadata.identifier=${kylin.metadata.url.identifier} -Dkylin.spark.category=sparder -Dkylin.spark.identifier={{APP_ID}} -Dalluxio.user.client.cache.dir=/mnt/alluxio-cache/alluxio-cache-{{APP_ID}}-{{EXECUTOR_ID}} kylin.query.spark-conf.spark.driver.extraJavaOptions=-Dhdp.version=current -Dalluxio.user.client.cache.dir=/mnt/alluxio-cache/alluxio-cache-driver kylin.query.spark-conf.spark.locality.wait=0 kylin.query.spark-conf.spark.sql.sources.ignoreDataLocality=true kylin.query.spark-conf.spark.extraListeners=org.apache.kylin.softaffinity.scheduler.SoftAffinityListener kylin.query.spark-conf.spark.hadoop.spark.kylin.local-cache.enabled=true kylin.query.spark-conf.spark.kylin.soft-affinity.enabled=true kylin.query.spark-conf.spark.kylin.soft-affinity.replications.num=2 kylin.query.spark-conf.spark.hadoop.io.file.buffer.size=524288 kylin.query.spark-conf.spark.hadoop.fs.s3a.impl=org.apache.kylin.cache.fs.kylin.KylinCacheFileSystem kylin.query.spark-conf.spark.hadoop.fs.s3a.experimental.input.fadvise=random kylin.query.spark-conf.spark.hadoop.parquet.enable.summary-metadata=false kylin.query.spark-conf.spark.sql.parquet.mergeSchema=false kylin.query.spark-conf.spark.sql.hive.metastorePartitionPruning=true # LOCAL or BUFF kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.store.type=LOCAL kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.async.restore.enabled=true kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.async.write.enabled=true kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.async.write.threads=6 kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.size=20GB kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.page.size=1MB kylin.query.spark-conf.spark.hadoop.alluxio.user.client.cache.local.store.file.buckets=1000 kylin.query.spark-conf.spark.hadoop.alluxio.user.update.file.accesstime.disabled=true kylin.query.spark-conf.spark.hadoop.fs.s3a.endpoint=s3.cn-northwest-1.amazonaws.com.cn kylin.query.spark-conf.spark.hadoop.fs.s3a.connection.ssl.enabled=false kylin.query.spark-conf.spark.hadoop.fs.s3a.access.key=xxx kylin.query.spark-conf.spark.hadoop.fs.s3a.secret.key=xxx kylin.query.cache-enabled=false kylin.query.auto-sparder-context-enabled=true