记录一次spark连接mysql遇到的问题
Posted zzzzMing -大数据技术
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了记录一次spark连接mysql遇到的问题相关的知识，希望对你有一定的参考价值。
在使用spark连接mysql的过程中报错了，错误如下
08:51:32.495 [main] ERROR - Error loading factory org.apache.calcite.jdbc.CalciteJdbc41Factory
java.lang.NoClassDefFoundError: org/apache/calcite/linq4j/QueryProvider
at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144]
at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144]
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144]
at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144]
at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144]
at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144]
at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144]
at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144]
at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144]
at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144]
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144]
at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144]
at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144]
at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144]
at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144]
at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144]
at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144]
at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144]
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144]
at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144]
at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144]
at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144]
at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144]
at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144]
at java.lang.Class.forName0(Native Method) ~[?:1.8.0_144]
at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_144]
at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:115) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating]
at org.apache.calcite.avatica.UnregisteredDriver.createFactory(UnregisteredDriver.java:74) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating]
at org.apache.calcite.avatica.UnregisteredDriver.<init>(UnregisteredDriver.java:55) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating]
at org.apache.calcite.jdbc.Driver.<init>(Driver.java:53) [calcite-core-1.2.0-incubating.jar:1.2.0-incubating]
at org.apache.calcite.jdbc.Driver.<clinit>(Driver.java:49) [calcite-core-1.2.0-incubating.jar:1.2.0-incubating]
at java.lang.Class.forName0(Native Method) [?:1.8.0_144]
at java.lang.Class.forName(Class.java:348) [?:1.8.0_144]
at java.sql.DriverManager.isDriverAllowed(DriverManager.java:556) [?:1.8.0_144]
at java.sql.DriverManager.isDriverAllowed(DriverManager.java:548) [?:1.8.0_144]
at java.sql.DriverManager.getDrivers(DriverManager.java:446) [?:1.8.0_144]
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:52) [spark-sql_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:50) [spark-sql_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:58) [spark-sql_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation.<init>(JDBCRelation.scala:113) [spark-sql_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:45) [spark-sql_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:330) [spark-sql_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:152) [spark-sql_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:125) [spark-sql_2.11-2.1.0.jar:2.1.0]
at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:45) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:50) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:52) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:54) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:56) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:58) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:60) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:62) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw$$iw.<init>(<console>:64) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw$$iw.<init>(<console>:66) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw$$iw.<init>(<console>:68) [scala-library-2.11.8.jar:?]
at $line31.$read$$iw.<init>(<console>:70) [scala-library-2.11.8.jar:?]
at $line31.$read.<init>(<console>:72) [scala-library-2.11.8.jar:?]
at $line31.$read$.<init>(<console>:76) [scala-library-2.11.8.jar:?]
at $line31.$read$.<clinit>(<console>) [scala-library-2.11.8.jar:?]
at $line31.$eval$.$print$lzycompute(<console>:7) [scala-library-2.11.8.jar:?]
at $line31.$eval$.$print(<console>:6) [scala-library-2.11.8.jar:?]
at $line31.$eval.$print(<console>) [scala-library-2.11.8.jar:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_144]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_144]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_144]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_144]
at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637) [scala-compiler-2.11.8.jar:?]
at scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31) [scala-reflect-2.11.8.jar:?]
at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19) [scala-reflect-2.11.8.jar:?]
at scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:825) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:825) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) [scala-compiler-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) [scala-compiler-2.11.8.jar:?]
at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97) [scala-reflect-2.11.8.jar:?]
at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) [scala-compiler-2.11.8.jar:?]
at org.apache.spark.repl.Main$.doMain(Main.scala:68) [spark-repl_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.repl.Main$.main(Main.scala:51) [spark-repl_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.repl.Main.main(Main.scala) [spark-repl_2.11-2.1.0.jar:2.1.0]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_144]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_144]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_144]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_144]
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) [spark-core_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) [spark-core_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) [spark-core_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) [spark-core_2.11-2.1.0.jar:2.1.0]
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) [spark-core_2.11-2.1.0.jar:2.1.0]
Caused by: java.lang.ClassNotFoundException: org.apache.calcite.linq4j.QueryProvider
at java.net.URLClassLoader.findClass(URLClassLoader.java:381) ~[?:1.8.0_144]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144]
... 110 more
java.lang.RuntimeException: Error loading factory org.apache.calcite.jdbc.CalciteJdbc41Factory
at org.apache.calcite.avatica.UnregisteredDriver.handle(UnregisteredDriver.java:132)
at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:126)
at org.apache.calcite.avatica.UnregisteredDriver.createFactory(UnregisteredDriver.java:74)
at org.apache.calcite.avatica.UnregisteredDriver.<init>(UnregisteredDriver.java:55)
at org.apache.calcite.jdbc.Driver.<init>(Driver.java:53)
at org.apache.calcite.jdbc.Driver.<clinit>(Driver.java:49)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at java.sql.DriverManager.isDriverAllowed(DriverManager.java:556)
at java.sql.DriverManager.isDriverAllowed(DriverManager.java:548)
at java.sql.DriverManager.getDrivers(DriverManager.java:446)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:52)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:50)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:58)
at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation.<init>(JDBCRelation.scala:113)
at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:45)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:330)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:152)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:125)
... 54 elided
Caused by: java.lang.NoClassDefFoundError: org/apache/calcite/linq4j/QueryProvider
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at java.lang.ClassLoader.defineClass1(Native Method)
at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
at java.security.AccessController.doPrivileged(Native Method)
at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:264)
at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:115)
... 71 more
Caused by: java.lang.ClassNotFoundException: org.apache.calcite.linq4j.QueryProvider
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 110 more
连接Mysql的代码是没问题的，但是居然会报这个错误，感觉很奇怪，而且去网上搜还搜不到相关资料，难道只有我碰到这个错误？
一开始以为是mysql的连接jar包有问题，但下载到本地用java连接又可以连接得上Mysql。
这时候注意到了错误ClassNotFoundException，一般这个错误要么是包冲突，要么是包缺失，包冲突就比较恶心了，所以先排查看看会不会是包缺失的问题，在Idea中键入两个shift，在搜素框中搜素org.apache.calcite.jdbc.CalciteJdbc41Factory。找到了这个东西，再去依赖包中查看相关jar包，发现calcite相关的有三个jar包，而在spark的jar包路径下calcite相关jar包只有两个，OK，在mven上下载缺失jar包，问题解决。
以上是关于记录一次spark连接mysql遇到的问题的主要内容，如果未能解决你的问题，请参考以下文章