[Bug] ClassNotFoundException: org.apache.spark.internal.Logging$class

[wangzhen-wudangshan]

Search before asking

• I had searched in the issues and found no similar issues.

What happened

ClassNotFoundException: org.apache.spark.internal.Logging$class

SeaTunnel Version

2.2.0-beta

SeaTunnel Config

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

######
###### This config file is a demonstration of batch processing in SeaTunnel config
######

env {
  # You can set flink configuration here
  execution.parallelism = 1
  job.name="test_tidb_source_to_hive"
}

source {
  # This is a example input plugin **only for test and demonstrate the feature input plugin**
  jdbc{
        driver = "com.mysql.jdbc.Driver"
        url = "jdbc:mysql://10.20.116.65:40000/flink_db?serverTimezone=UTC&useSSL=false&useUnicode=true&characterEncoding=UTF-8"
        username = "flink"
        password = "Gj1_dkaPwyVcm"
        query = "select * from t_records_data"
        fetch_size = 500
        result_table_name = "tidb_result"
     }

  # If you would like to get more information about how to configure seatunnel and see full list of input plugins,
  # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
}

transform {
  #Sql {
   # sql = "select * from tidb_result"
  #}

  # If you would like to get more information about how to configure seatunnel and see full list of filter plugins,
  # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
}

sink {
  # choose stdout output plugin to output data to console
  Hive {
    source_table_name = "tidb_result"
    #result_table_name = "seatunnel.hive_test"
    save_mode = "overwrite"
    sink_columns = "application_id,create_flow_count,total_bitps,create_time,time"
    metastore_uri = "thrift://hdpapcuat01v:9083"
    partition_by = ["time"]
  }
  # If you would like to get more information about how to configure seatunnel and see full list of output plugins,
  # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
}

Running Command

./start-seatunnel-spark.sh --master yarn --deploy-mode cluster --config ../config/spark_hive.batch.conf

Error Exception

22/10/14 17:57:11 INFO Client:
         client token: N/A
         diagnostics: User class threw exception: java.util.ServiceConfigurationError: org.apache.seatunnel.spark.BaseSparkSink: Provider org.apache.seatunnel.spark.hive.sink.Hive could not be instantiated
        at java.util.ServiceLoader.fail(ServiceLoader.java:232)
        at java.util.ServiceLoader.access$100(ServiceLoader.java:185)
        at java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384)
        at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)
        at java.util.ServiceLoader$1.next(ServiceLoader.java:480)
        at org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery.loadPluginInstance(AbstractPluginDiscovery.java:128)
        at org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery.createPluginInstance(AbstractPluginDiscovery.java:99)
        at org.apache.seatunnel.core.spark.config.SparkExecutionContext.lambda$getSinks$2(SparkExecutionContext.java:95)
        at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
        at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
        at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
        at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
        at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
        at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
        at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
        at org.apache.seatunnel.core.spark.config.SparkExecutionContext.getSinks(SparkExecutionContext.java:98)
        at org.apache.seatunnel.core.spark.command.SparkTaskExecuteCommand.execute(SparkTaskExecuteCommand.java:57)
        at org.apache.seatunnel.core.base.Seatunnel.run(Seatunnel.java:40)
        at org.apache.seatunnel.core.spark.SeatunnelSpark.main(SeatunnelSpark.java:33)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:728)
Caused by: java.lang.NoClassDefFoundError: org/apache/spark/internal/Logging$class
        at org.apache.seatunnel.spark.hive.sink.Hive.<init>(Hive.scala:29)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380)
        ... 21 more
Caused by: java.lang.ClassNotFoundException: org.apache.spark.internal.Logging$class
        at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
        ... 28 more

         ApplicationMaster host: hdpapcuat01v
         ApplicationMaster RPC port: 6651
         queue: default
         start time: 1665741406406
         final status: FAILED
         tracking URL: http://hdpapcuat02v:8088/proxy/application_1647855649472_0183/
         user: hdfs
22/10/14 17:57:11 ERROR Client: Application diagnostics message: User class threw exception: java.util.ServiceConfigurationError: org.apache.seatunnel.spark.BaseSparkSink: Provider org.apache.seatunnel.spark.hive.sink.Hive could not be instantiated
        at java.util.ServiceLoader.fail(ServiceLoader.java:232)
        at java.util.ServiceLoader.access$100(ServiceLoader.java:185)
        at java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:384)
        at java.util.ServiceLoader$LazyIterator.next(ServiceLoader.java:404)
        at java.util.ServiceLoader$1.next(ServiceLoader.java:480)
        at org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery.loadPluginInstance(AbstractPluginDiscovery.java:128)
        at org.apache.seatunnel.plugin.discovery.AbstractPluginDiscovery.createPluginInstance(AbstractPluginDiscovery.java:99)
        at org.apache.seatunnel.core.spark.config.SparkExecutionContext.lambda$getSinks$2(SparkExecutionContext.java:95)
        at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)
        at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
        at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:482)
        at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:472)
        at java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:708)
        at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
        at java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:499)
        at org.apache.seatunnel.core.spark.config.SparkExecutionContext.getSinks(SparkExecutionContext.java:98)
        at org.apache.seatunnel.core.spark.command.SparkTaskExecuteCommand.execute(SparkTaskExecuteCommand.java:57)
        at org.apache.seatunnel.core.base.Seatunnel.run(Seatunnel.java:40)
        at org.apache.seatunnel.core.spark.SeatunnelSpark.main(SeatunnelSpark.java:33)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:728)
Caused by: java.lang.NoClassDefFoundError: org/apache/spark/internal/Logging$class
        at org.apache.seatunnel.spark.hive.sink.Hive.<init>(Hive.scala:29)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at java.lang.Class.newInstance(Class.java:442)
        at java.util.ServiceLoader$LazyIterator.nextService(ServiceLoader.java:380)
        ... 21 more
Caused by: java.lang.ClassNotFoundException: org.apache.spark.internal.Logging$class
        at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
        at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
        ... 28 more

Exception in thread "main" org.apache.spark.SparkException: Application application_1647855649472_0183 finished with failed status
        at org.apache.spark.deploy.yarn.Client.run(Client.scala:1192)
        at org.apache.spark.deploy.yarn.YarnClusterApplication.start(Client.scala:1583)
        at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:928)
        at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
        at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
        at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
        at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1007)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1016)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

Flink or Spark Version

flink: 1.13.5
spark: 3.0.0

Java or Scala Version

jdk: 1.8
sacla: 2.12

Screenshots

No response

Are you willing to submit PR?

• Yes I am willing to submit a PR!

Code of Conduct

• I agree to follow this project's Code of Conduct

[Hisoka-X]

We not support spark3 yet, please track #2574