Spark教程——（11）Spark程序本地执行和集群执行的差异
Posted ratels
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了Spark教程——（11）Spark程序本地执行和集群执行的差异相关的知识，希望对你有一定的参考价值。
本地执行Spark SQL程序：
package com.fc //import common.util.{phoenixConnectMode, timeUtil} import org.apache.spark.sql.SQLContext import org.apache.spark.sql.functions.col import org.apache.spark.{SparkConf, SparkContext} /* 每天执行 */ object costDay { def main(args: Array[String]): Unit = { val conf = new SparkConf() .setAppName("fdsf") .setMaster("local") val sc = new SparkContext(conf) val sqlContext = new SQLContext(sc) // val df = sqlContext.load( // "org.apache.phoenix.spark" // , Map("table" -> "ASSET_NORMAL" // , "zkUrl" -> "node3,node4,node5:2181") // ) val tableName = "ASSET_NORMAL" val columes = Array( "ID", "ASSET_ID", "ASSET_NAME", "ASSET_FIRST_DEGREE_ID", "ASSET_FIRST_DEGREE_NAME", "ASSET_SECOND_DEGREE_ID", "ASSET_SECOND_DEGREE_NAME", "GB_DEGREE_ID", "GB_DEGREE_NAME", "ASSET_USE_FIRST_DEGREE_ID", "ASSET_USE_FIRST_DEGREE_NAME", "ASSET_USE_SECOND_DEGREE_ID", "ASSET_USE_SECOND_DEGREE_NAME", "MANAGEMENT_TYPE_ID", "MANAGEMENT_TYPE_NAME", "ASSET_MODEL", "FACTORY_NUMBER", "ASSET_COUNTRY_ID", "ASSET_COUNTRY_NAME", "MANUFACTURER", "SUPPLIER", "SUPPLIER_TEL", "ORIGINAL_VALUE", "USE_DEPARTMENT_ID", "USE_DEPARTMENT_NAME", "USER_ID", "USER_NAME", "ASSET_LOCATION_OF_PARK_ID", "ASSET_LOCATION_OF_PARK_NAME", "ASSET_LOCATION_OF_BUILDING_ID", "ASSET_LOCATION_OF_BUILDING_NAME", "ASSET_LOCATION_OF_ROOM_ID", "ASSET_LOCATION_OF_ROOM_NUMBER", "PRODUCTION_DATE", "ACCEPTANCE_DATE", "REQUISITION_DATE", "PERFORMANCE_INDEX", "ASSET_STATE_ID", "ASSET_STATE_NAME", "INSPECTION_TYPE_ID", "INSPECTION_TYPE_NAME", "SEAL_DATE", "SEAL_CAUSE", "COST_ITEM_ID", "COST_ITEM_NAME", "ITEM_COMMENTS", "UNSEAL_DATE", "SCRAP_DATE", "PURCHASE_NUMBER", "WARRANTY_PERIOD", "DEPRECIABLE_LIVES_ID", "DEPRECIABLE_LIVES_NAME", "MEASUREMENT_UNITS_ID", "MEASUREMENT_UNITS_NAME", "ANNEX", "REMARK", "ACCOUNTING_TYPE_ID", "ACCOUNTING_TYPE_NAME", "SYSTEM_TYPE_ID", "SYSTEM_TYPE_NAME", "ASSET_ID_PARENT", "CLASSIFIED_LEVEL_ID", "CLASSIFIED_LEVEL_NAME", "ASSET_PICTURE", "MILITARY_SPECIAL_CODE", "CHECK_CYCLE_ID", "CHECK_CYCLE_NAME", "CHECK_DATE", "CHECK_EFFECTIVE_DATE", "CHECK_MODE_ID", "CHECK_MODE_NAME", "CHECK_DEPARTMENT_ID", "CHECK_DEPARTMENT_NAME", "RENT_STATUS_ID", "RENT_STATUS_NAME", "STORAGE_TIME", "UPDATE_USER", "UPDATE_TIME", "IS_ON_PROCESS", "IS_DELETED", "FIRST_DEPARTMENT_ID", "FIRST_DEPARTMENT_NAME", "SECOND_DEPARTMENT_ID", "SECOND_DEPARTMENT_NAME", "CREATE_USER", "CREATE_TIME" ) val df = phoenixConnectMode.getMode1(sqlContext, tableName, columes) .filter(col("USE_DEPARTMENT_ID") isNotNull) df.registerTempTable("asset_normal") // df.show(false) def costingWithin(originalValue: Double, years: Int): Double = (originalValue*0.95)/(years*365) sqlContext.udf.register("costingWithin", costingWithin _) def costingBeyond(originalValue: Double): Double = originalValue*0.05/365 sqlContext.udf.register("costingBeyond", costingBeyond _) def expire(acceptanceDate: String, years: Int): Boolean = timeUtil.dateStrAddYears2TimeStamp(acceptanceDate, timeUtil.SECOND_TIME_FORMAT, years) > System.currentTimeMillis() sqlContext.udf.register("expire", expire _) val costDay = sqlContext .sql( "select " + "ID" + ",USE_DEPARTMENT_ID as FIRST_DEPARTMENT_ID" + ",case when expire(ACCEPTANCE_DATE, DEPRECIABLE_LIVES_NAME) then costingWithin(ORIGINAL_VALUE, DEPRECIABLE_LIVES_NAME) else costingBeyond(ORIGINAL_VALUE) end as ACTUAL_COST" + ",ORIGINAL_VALUE" + ",current_timestamp() as GENERATION_TIME" + " from asset_normal" ) costDay.printSchema() println(costDay.count()) costDay.col("ORIGINAL_VALUE") costDay.describe("ORIGINAL_VALUE").show() costDay.show(false) // costDay.write // .format("org.apache.phoenix.spark") // .mode("overwrite") // .option("table", "ASSET_FINANCIAL_DETAIL_DAY") // .option("zkUrl", "node3,node4,node5:2181") // .save() } }
执行结果参考《Spark教程——（10）Spark SQL读取Phoenix数据本地执行计算》，如下：
19/09/24 00:52:14 INFO spark.SparkContext: Running Spark version 1.6.0 19/09/24 00:52:16 INFO spark.SecurityManager: Changing view acls to: cf_pc 19/09/24 00:52:16 INFO spark.SecurityManager: Changing modify acls to: cf_pc 19/09/24 00:52:16 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(cf_pc); users with modify permissions: Set(cf_pc) 19/09/24 00:52:16 INFO util.Utils: Successfully started service ‘sparkDriver‘ on port 54691. 19/09/24 00:52:17 INFO slf4j.Slf4jLogger: Slf4jLogger started 19/09/24 00:52:17 INFO Remoting: Starting remoting 19/09/24 00:52:17 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@10.200.74.155:54708] 19/09/24 00:52:17 INFO Remoting: Remoting now listens on addresses: [akka.tcp://sparkDriverActorSystem@10.200.74.155:54708] 19/09/24 00:52:17 INFO util.Utils: Successfully started service ‘sparkDriverActorSystem‘ on port 54708. 19/09/24 00:52:17 INFO spark.SparkEnv: Registering MapOutputTracker 19/09/24 00:52:17 INFO spark.SparkEnv: Registering BlockManagerMaster 19/09/24 00:52:17 INFO storage.DiskBlockManager: Created local directory at C:Userscf_pcAppDataLocalTemplockmgr-9012d6b9-eab0-41f0-8e9e-b63c3fe7fb09 19/09/24 00:52:17 INFO storage.MemoryStore: MemoryStore started with capacity 478.2 MB 19/09/24 00:52:17 INFO spark.SparkEnv: Registering OutputCommitCoordinator 19/09/24 00:52:18 INFO server.Server: jetty-8.y.z-SNAPSHOT 19/09/24 00:52:18 INFO server.AbstractConnector: Started SelectChannelConnector@0.0.0.0:4040 19/09/24 00:52:18 INFO util.Utils: Successfully started service ‘SparkUI‘ on port 4040. 19/09/24 00:52:18 INFO ui.SparkUI: Started SparkUI at http://10.200.74.155:4040 19/09/24 00:52:18 INFO executor.Executor: Starting executor ID driver on host localhost 19/09/24 00:52:18 INFO util.Utils: Successfully started service ‘org.apache.spark.network.netty.NettyBlockTransferService‘ on port 54720. 19/09/24 00:52:18 INFO netty.NettyBlockTransferService: Server created on 54720 19/09/24 00:52:18 INFO storage.BlockManagerMaster: Trying to register BlockManager 19/09/24 00:52:18 INFO storage.BlockManagerMasterEndpoint: Registering block manager localhost:54720 with 478.2 MB RAM, BlockManagerId(driver, localhost, 54720) 19/09/24 00:52:18 INFO storage.BlockManagerMaster: Registered BlockManager 19/09/24 00:52:21 INFO storage.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 247.5 KB, free 477.9 MB) 19/09/24 00:52:22 INFO storage.MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 15.4 KB, free 477.9 MB) 19/09/24 00:52:22 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:54720 (size: 15.4 KB, free: 478.2 MB) 19/09/24 00:52:22 INFO spark.SparkContext: Created broadcast 0 from newAPIHadoopRDD at PhoenixRDD.scala:49 19/09/24 00:52:22 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:22 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:23 INFO log.QueryLoggerDisruptor: Starting QueryLoggerDisruptor for with ringbufferSize=8192, waitStrategy=BlockingWaitStrategy, exceptionHandler=org.apache.phoenix.log.QueryLoggerDefaultExceptionHandler@4f169009... 19/09/24 00:52:23 INFO query.ConnectionQueryServicesImpl: An instance of ConnectionQueryServices was created. 19/09/24 00:52:23 INFO zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x4bc59b27 connecting to ZooKeeper ensemble=node3:2181 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:host.name=DESKTOP-0CDQ4PM 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.version=1.8.0_212 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.vendor=Oracle Corporation 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.home=C:3rdJavajdk1.8.0_212jre 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.class.path=C:3rdJavajdk1.8.0_212jrelibcharsets.jar;C:3rdJavajdk1.8.0_212jrelibdeploy.jar;C:3rdJavajdk1.8.0_212jrelibextaccess-bridge-64.jar;C:3rdJavajdk1.8.0_212jrelibextcldrdata.jar;C:3rdJavajdk1.8.0_212jrelibextdnsns.jar;C:3rdJavajdk1.8.0_212jrelibextjaccess.jar;C:3rdJavajdk1.8.0_212jrelibextjfxrt.jar;C:3rdJavajdk1.8.0_212jrelibextlocaledata.jar;C:3rdJavajdk1.8.0_212jrelibext ashorn.jar;C:3rdJavajdk1.8.0_212jrelibextsunec.jar;C:3rdJavajdk1.8.0_212jrelibextsunjce_provider.jar;C:3rdJavajdk1.8.0_212jrelibextsunmscapi.jar;C:3rdJavajdk1.8.0_212jrelibextsunpkcs11.jar;C:3rdJavajdk1.8.0_212jrelibextzipfs.jar;C:3rdJavajdk1.8.0_212jrelibjavaws.jar;C:3rdJavajdk1.8.0_212jrelibjce.jar;C:3rdJavajdk1.8.0_212jrelibjfr.jar;C:3rdJavajdk1.8.0_212jrelibjfxswt.jar;C:3rdJavajdk1.8.0_212jrelibjsse.jar;C:3rdJavajdk1.8.0_212jrelibmanagement-agent.jar;C:3rdJavajdk1.8.0_212jrelibplugin.jar;C:3rdJavajdk1.8.0_212jrelib esources.jar;C:3rdJavajdk1.8.0_212jrelib t.jar;C:developmentcfscalapp argetclasses;C:3rdscalascala-2.10.5libscala-actors-migration.jar;C:3rdscalascala-2.10.5libscala-actors.jar;C:3rdscalascala-2.10.5libscala-library.jar;C:3rdscalascala-2.10.5libscala-reflect.jar;C:3rdscalascala-2.10.5libscala-swing.jar;C:3rdscalascala-2.10.5srcscala-actors-src.jar;C:3rdscalascala-2.10.5srcscala-library-src.jar;C:3rdscalascala-2.10.5srcscala-reflect-src.jar;C:3rdscalascala-2.10.5srcscala-swing-src.jar;C:developmentMavenRepositoryorgapachesparkspark-core_2.101.6.0-cdh5.14.2spark-core_2.10-1.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapacheavroavro-mapred1.7.6-cdh5.14.2avro-mapred-1.7.6-cdh5.14.2-hadoop2.jar;C:developmentMavenRepositoryorgapacheavroavro-ipc1.7.6-cdh5.14.2avro-ipc-1.7.6-cdh5.14.2.jar;C:developmentMavenRepositoryorgapacheavroavro-ipc1.7.6-cdh5.14.2avro-ipc-1.7.6-cdh5.14.2-tests.jar;C:developmentMavenRepositorycom witterchill_2.100.5.0chill_2.10-0.5.0.jar;C:developmentMavenRepositorycomesotericsoftwarekryokryo2.21kryo-2.21.jar;C:developmentMavenRepositorycomesotericsoftware eflectasm eflectasm1.07 eflectasm-1.07-shaded.jar;C:developmentMavenRepositorycomesotericsoftwareminlogminlog1.2minlog-1.2.jar;C:developmentMavenRepositoryorgobjenesisobjenesis1.2objenesis-1.2.jar;C:developmentMavenRepositorycom witterchill-java0.5.0chill-java-0.5.0.jar;C:developmentMavenRepositoryorgapachexbeanxbean-asm5-shaded4.4xbean-asm5-shaded-4.4.jar;C:developmentMavenRepositoryorgapachehadoophadoop-client2.6.0-cdh5.14.2hadoop-client-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-hdfs2.6.0-cdh5.14.2hadoop-hdfs-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryxercesxercesImpl2.9.1xercesImpl-2.9.1.jar;C:developmentMavenRepositoryxml-apisxml-apis1.3.04xml-apis-1.3.04.jar;C:developmentMavenRepositoryorgapachehadoophadoop-mapreduce-client-app2.6.0-cdh5.14.2hadoop-mapreduce-client-app-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-mapreduce-client-common2.6.0-cdh5.14.2hadoop-mapreduce-client-common-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-yarn-client2.6.0-cdh5.14.2hadoop-yarn-client-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-yarn-server-common2.6.0-cdh5.14.2hadoop-yarn-server-common-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-mapreduce-client-shuffle2.6.0-cdh5.14.2hadoop-mapreduce-client-shuffle-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-yarn-api2.6.0-cdh5.14.2hadoop-yarn-api-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-mapreduce-client-jobclient2.6.0-cdh5.14.2hadoop-mapreduce-client-jobclient-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-aws2.6.0-cdh5.14.2hadoop-aws-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositorycomamazonawsaws-java-sdk-bundle1.11.134aws-java-sdk-bundle-1.11.134.jar;C:developmentMavenRepositoryorgapachesparkspark-launcher_2.101.6.0-cdh5.14.2spark-launcher_2.10-1.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachesparkspark-network-common_2.101.6.0-cdh5.14.2spark-network-common_2.10-1.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgfusesourceleveldbjnileveldbjni-all1.8leveldbjni-all-1.8.jar;C:developmentMavenRepositorycomfasterxmljacksoncorejackson-annotations2.2.3jackson-annotations-2.2.3.jar;C:developmentMavenRepositoryorgapachesparkspark-network-shuffle_2.101.6.0-cdh5.14.2spark-network-shuffle_2.10-1.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachesparkspark-unsafe_2.101.6.0-cdh5.14.2spark-unsafe_2.10-1.6.0-cdh5.14.2.jar;C:developmentMavenRepository etjavadevjets3tjets3t0.9.0jets3t-0.9.0.jar;C:developmentMavenRepositoryorgapachehttpcomponentshttpcore4.1.2httpcore-4.1.2.jar;C:developmentMavenRepositorycomjamesmurtyutilsjava-xmlbuilder0.4java-xmlbuilder-0.4.jar;C:developmentMavenRepositoryorgapachecuratorcurator-recipes2.7.1curator-recipes-2.7.1.jar;C:developmentMavenRepositoryorgapachecuratorcurator-framework2.7.1curator-framework-2.7.1.jar;C:developmentMavenRepositoryorgapachezookeeperzookeeper3.4.6zookeeper-3.4.6.jar;C:developmentMavenRepositoryorgeclipsejettyorbitjavax.servlet3.0.0.v201112011016javax.servlet-3.0.0.v201112011016.jar;C:developmentMavenRepositoryorgapachecommonscommons-lang33.3.2commons-lang3-3.3.2.jar;C:developmentMavenRepositoryorgapachecommonscommons-math33.4.1commons-math3-3.4.1.jar;C:developmentMavenRepositorycomgooglecodefindbugsjsr3051.3.9jsr305-1.3.9.jar;C:developmentMavenRepositoryorgslf4jslf4j-api1.7.5slf4j-api-1.7.5.jar;C:developmentMavenRepositoryorgslf4jjul-to-slf4j1.7.5jul-to-slf4j-1.7.5.jar;C:developmentMavenRepositoryorgslf4jjcl-over-slf4j1.7.5jcl-over-slf4j-1.7.5.jar;C:developmentMavenRepositorylog4jlog4j1.2.17log4j-1.2.17.jar;C:developmentMavenRepositoryorgslf4jslf4j-log4j121.7.5slf4j-log4j12-1.7.5.jar;C:developmentMavenRepositorycom ingcompress-lzf1.0.3compress-lzf-1.0.3.jar;C:developmentMavenRepositoryorgxerialsnappysnappy-java1.0.4.1snappy-java-1.0.4.1.jar;C:developmentMavenRepository etjpountzlz4lz41.3.0lz4-1.3.0.jar;C:developmentMavenRepositoryorg oaringbitmapRoaringBitmap0.5.11RoaringBitmap-0.5.11.jar;C:developmentMavenRepositorycommons-netcommons-net2.2commons-net-2.2.jar;C:developmentMavenRepositoryorgspark-projectakkaakka-remote_2.102.2.3-shaded-protobufakka-remote_2.10-2.2.3-shaded-protobuf.jar;C:developmentMavenRepositoryorgspark-projectakkaakka-actor_2.102.2.3-shaded-protobufakka-actor_2.10-2.2.3-shaded-protobuf.jar;C:developmentMavenRepositorycom ypesafeconfig1.0.2config-1.0.2.jar;C:developmentMavenRepositoryorgspark-projectprotobufprotobuf-java2.4.1-shadedprotobuf-java-2.4.1-shaded.jar;C:developmentMavenRepositoryorguncommonsmathsuncommons-maths1.2.2auncommons-maths-1.2.2a.jar;C:developmentMavenRepositoryorgspark-projectakkaakka-slf4j_2.102.2.3-shaded-protobufakka-slf4j_2.10-2.2.3-shaded-protobuf.jar;C:developmentMavenRepositoryorgscala-langscala-library2.10.5scala-library-2.10.5.jar;C:developmentMavenRepositoryorgjson4sjson4s-jackson_2.103.2.10json4s-jackson_2.10-3.2.10.jar;C:developmentMavenRepositoryorgjson4sjson4s-core_2.103.2.10json4s-core_2.10-3.2.10.jar;C:developmentMavenRepositoryorgjson4sjson4s-ast_2.103.2.10json4s-ast_2.10-3.2.10.jar;C:developmentMavenRepositoryorgscala-langscalap2.10.0scalap-2.10.0.jar;C:developmentMavenRepositoryorgscala-langscala-compiler2.10.0scala-compiler-2.10.0.jar;C:developmentMavenRepositorycomsunjerseyjersey-server1.9jersey-server-1.9.jar;C:developmentMavenRepositoryasmasm3.1asm-3.1.jar;C:developmentMavenRepositorycomsunjerseyjersey-core1.9jersey-core-1.9.jar;C:developmentMavenRepositoryorgapachemesosmesos0.21.1mesos-0.21.1-shaded-protobuf.jar;C:developmentMavenRepositoryio etty etty-all4.0.29.Final etty-all-4.0.29.Final.jar;C:developmentMavenRepositorycomclearspringanalyticsstream2.7.0stream-2.7.0.jar;C:developmentMavenRepositoryiodropwizardmetricsmetrics-core3.1.2metrics-core-3.1.2.jar;C:developmentMavenRepositoryiodropwizardmetricsmetrics-jvm3.1.2metrics-jvm-3.1.2.jar;C:developmentMavenRepositoryiodropwizardmetricsmetrics-json3.1.2metrics-json-3.1.2.jar;C:developmentMavenRepositoryiodropwizardmetricsmetrics-graphite3.1.2metrics-graphite-3.1.2.jar;C:developmentMavenRepositorycomfasterxmljacksoncorejackson-databind2.2.3jackson-databind-2.2.3.jar;C:developmentMavenRepositorycomfasterxmljacksoncorejackson-core2.2.3jackson-core-2.2.3.jar;C:developmentMavenRepositorycomfasterxmljacksonmodulejackson-module-scala_2.102.2.3jackson-module-scala_2.10-2.2.3.jar;C:developmentMavenRepositorycom houghtworksparanamerparanamer2.3paranamer-2.3.jar;C:developmentMavenRepositoryorgapacheivyivy2.4.0ivy-2.4.0.jar;C:developmentMavenRepositoryorooro2.0.8oro-2.0.8.jar;C:developmentMavenRepositoryorg achyonproject achyon-client0.8.2 achyon-client-0.8.2.jar;C:developmentMavenRepositorycommons-langcommons-lang2.4commons-lang-2.4.jar;C:developmentMavenRepositorycommons-iocommons-io2.4commons-io-2.4.jar;C:developmentMavenRepositoryorg achyonproject achyon-underfs-hdfs0.8.2 achyon-underfs-hdfs-0.8.2.jar;C:developmentMavenRepositoryorg achyonproject achyon-underfs-s30.8.2 achyon-underfs-s3-0.8.2.jar;C:developmentMavenRepositoryorg achyonproject achyon-underfs-local0.8.2 achyon-underfs-local-0.8.2.jar;C:developmentMavenRepository et azorvinepyrolite4.9pyrolite-4.9.jar;C:developmentMavenRepository etsfpy4jpy4j0.9py4j-0.9.jar;C:developmentMavenRepositorycomintelchimerachimera0.9.2chimera-0.9.2.jar;C:developmentMavenRepositoryorgspark-projectsparkunused1.0.0unused-1.0.0.jar;C:developmentMavenRepositoryorgapachesparkspark-sql_2.101.6.0-cdh5.14.2spark-sql_2.10-1.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachesparkspark-catalyst_2.101.6.0-cdh5.14.2spark-catalyst_2.10-1.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgscala-langscala-reflect2.10.5scala-reflect-2.10.5.jar;C:developmentMavenRepositoryorgcodehausjaninojanino2.7.8janino-2.7.8.jar;C:developmentMavenRepositoryorgcodehausjaninocommons-compiler2.7.8commons-compiler-2.7.8.jar;C:developmentMavenRepositorycom witterparquet-column1.5.0-cdh5.14.2parquet-column-1.5.0-cdh5.14.2.jar;C:developmentMavenRepositorycom witterparquet-common1.5.0-cdh5.14.2parquet-common-1.5.0-cdh5.14.2.jar;C:developmentMavenRepositorycom witterparquet-encoding1.5.0-cdh5.14.2parquet-encoding-1.5.0-cdh5.14.2.jar;C:developmentMavenRepositorycom witterparquet-hadoop1.5.0-cdh5.14.2parquet-hadoop-1.5.0-cdh5.14.2.jar;C:developmentMavenRepositorycom witterparquet-format2.1.0-cdh5.14.2parquet-format-2.1.0-cdh5.14.2.jar;C:developmentMavenRepositorycom witterparquet-jackson1.5.0-cdh5.14.2parquet-jackson-1.5.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehbasehbase-spark1.2.0-cdh5.14.2hbase-spark-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-common2.6.0-cdh5.14.2hadoop-common-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryxmlencxmlenc0.52xmlenc-0.52.jar;C:developmentMavenRepositorycommons-httpclientcommons-httpclient3.1commons-httpclient-3.1.jar;C:developmentMavenRepositoryorgmortbayjettyjetty6.1.26.cloudera.4jetty-6.1.26.cloudera.4.jar;C:developmentMavenRepositoryorgmortbayjettyjetty-util6.1.26.cloudera.4jetty-util-6.1.26.cloudera.4.jar;C:developmentMavenRepositorycomsunjerseyjersey-json1.9jersey-json-1.9.jar;C:developmentMavenRepositoryorgcodehausjettisonjettison1.1jettison-1.1.jar;C:developmentMavenRepositorycomsunxmlindjaxb-impl2.2.3-1jaxb-impl-2.2.3-1.jar;C:developmentMavenRepository omcatjasper-compiler5.5.23jasper-compiler-5.5.23.jar;C:developmentMavenRepository omcatjasper-runtime5.5.23jasper-runtime-5.5.23.jar;C:developmentMavenRepositorycommons-elcommons-el1.0commons-el-1.0.jar;C:developmentMavenRepositorycommons-configurationcommons-configuration1.6commons-configuration-1.6.jar;C:developmentMavenRepositorycommons-digestercommons-digester1.8commons-digester-1.8.jar;C:developmentMavenRepositorycommons-beanutilscommons-beanutils1.7.0commons-beanutils-1.7.0.jar;C:developmentMavenRepositorycommons-beanutilscommons-beanutils-core1.8.0commons-beanutils-core-1.8.0.jar;C:developmentMavenRepositorycomgooglecodegsongson2.2.4gson-2.2.4.jar;C:developmentMavenRepositoryorgapachehadoophadoop-auth2.6.0-cdh5.14.2hadoop-auth-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachedirectoryserverapacheds-kerberos-codec2.0.0-M15apacheds-kerberos-codec-2.0.0-M15.jar;C:developmentMavenRepositoryorgapachedirectoryserverapacheds-i18n2.0.0-M15apacheds-i18n-2.0.0-M15.jar;C:developmentMavenRepositoryorgapachedirectoryapiapi-asn1-api1.0.0-M20api-asn1-api-1.0.0-M20.jar;C:developmentMavenRepositoryorgapachedirectoryapiapi-util1.0.0-M20api-util-1.0.0-M20.jar;C:developmentMavenRepositorycomjcraftjsch0.1.42jsch-0.1.42.jar;C:developmentMavenRepositoryorgapachecuratorcurator-client2.7.1curator-client-2.7.1.jar;C:developmentMavenRepositoryorgapachehtracehtrace-core44.0.1-incubatinghtrace-core4-4.0.1-incubating.jar;C:developmentMavenRepositoryorgapachecommonscommons-compress1.4.1commons-compress-1.4.1.jar;C:developmentMavenRepositoryorg ukaanixz1.0xz-1.0.jar;C:developmentMavenRepositoryorgapachehbasehbase-client1.2.0-cdh5.14.2hbase-client-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgjrubyjcodingsjcodings1.0.8jcodings-1.0.8.jar;C:developmentMavenRepositorycomyammermetricsmetrics-core2.2.0metrics-core-2.2.0.jar;C:developmentMavenRepositoryorgapachehbasehbase-protocol1.2.0-cdh5.14.2hbase-protocol-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehbasehbase-server1.2.0-cdh5.14.2hbase-server-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehbasehbase-procedure1.2.0-cdh5.14.2hbase-procedure-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehbasehbase-common1.2.0-cdh5.14.2hbase-common-1.2.0-cdh5.14.2-tests.jar;C:developmentMavenRepositoryorgapachehbasehbase-prefix-tree1.2.0-cdh5.14.2hbase-prefix-tree-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositorycomgithubstephenchigh-scale-libhigh-scale-lib1.1.1high-scale-lib-1.1.1.jar;C:developmentMavenRepositoryorgapachecommonscommons-math2.1commons-math-2.1.jar;C:developmentMavenRepositoryorgmortbayjettyjetty-sslengine6.1.26.cloudera.4jetty-sslengine-6.1.26.cloudera.4.jar;C:developmentMavenRepositoryorgmortbayjettyjsp-2.16.1.14jsp-2.1-6.1.14.jar;C:developmentMavenRepositoryorgmortbayjettyjsp-api-2.16.1.14jsp-api-2.1-6.1.14.jar;C:developmentMavenRepositoryorgmortbayjettyservlet-api-2.56.1.14servlet-api-2.5-6.1.14.jar;C:developmentMavenRepositoryorgcodehausjacksonjackson-jaxrs1.8.8jackson-jaxrs-1.8.8.jar;C:developmentMavenRepositoryorgjamonjamon-runtime2.4.1jamon-runtime-2.4.1.jar;C:developmentMavenRepositoryorghamcresthamcrest-core1.3hamcrest-core-1.3.jar;C:developmentMavenRepositoryorgapachehadoophadoop-core2.6.0-mr1-cdh5.14.2hadoop-core-2.6.0-mr1-cdh5.14.2.jar;C:developmentMavenRepositoryjavaxservletjspjsp-api2.1jsp-api-2.1.jar;C:developmentMavenRepositoryhsqldbhsqldb1.8.0.10hsqldb-1.8.0.10.jar;C:developmentMavenRepositoryorgeclipsejdtcore3.1.1core-3.1.1.jar;C:developmentMavenRepositoryorgapachehadoophadoop-hdfs2.6.0-cdh5.14.2hadoop-hdfs-2.6.0-cdh5.14.2-tests.jar;C:developmentMavenRepositorycommons-daemoncommons-daemon1.0.13commons-daemon-1.0.13.jar;C:developmentMavenRepositorycomgoogleprotobufprotobuf-java2.5.0protobuf-java-2.5.0.jar;C:developmentMavenRepositorycommons-loggingcommons-logging1.2commons-logging-1.2.jar;C:developmentMavenRepositorycomgithubstephencfindbugsfindbugs-annotations1.3.9-1findbugs-annotations-1.3.9-1.jar;C:developmentMavenRepositoryorgapachephoenixphoenix-spark4.14.0-cdh5.14.2phoenix-spark-4.14.0-cdh5.14.2.jar;C:developmentMavenRepositorycomlmaxdisruptor3.3.8disruptor-3.3.8.jar;C:developmentMavenRepositoryorgapachephoenixphoenix-core4.14.0-cdh5.14.2phoenix-core-4.14.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapache ephra ephra-api0.14.0-incubating ephra-api-0.14.0-incubating.jar;C:developmentMavenRepositoryorgapache ephra ephra-core0.14.0-incubating ephra-core-0.14.0-incubating.jar;C:developmentMavenRepositorycomgoogleinjectguice3.0guice-3.0.jar;C:developmentMavenRepositoryjavaxinjectjavax.inject1javax.inject-1.jar;C:developmentMavenRepositoryaopallianceaopalliance1.0aopalliance-1.0.jar;C:developmentMavenRepositorycomgoogleinjectextensionsguice-assistedinject3.0guice-assistedinject-3.0.jar;C:developmentMavenRepositoryorgapache hriftlibthrift0.9.0libthrift-0.9.0.jar;C:developmentMavenRepositoryitunimidsifastutil6.5.6fastutil-6.5.6.jar;C:developmentMavenRepositoryorgapache will will-common0.8.0 will-common-0.8.0.jar;C:developmentMavenRepositoryorgapache will will-core0.8.0 will-core-0.8.0.jar;C:developmentMavenRepositoryorgapache will will-api0.8.0 will-api-0.8.0.jar;C:developmentMavenRepositoryorgow2asmasm-all5.0.2asm-all-5.0.2.jar;C:developmentMavenRepositoryorgapache will will-discovery-api0.8.0 will-discovery-api-0.8.0.jar;C:developmentMavenRepositoryorgapache will will-discovery-core0.8.0 will-discovery-core-0.8.0.jar;C:developmentMavenRepositoryorgapache will will-zookeeper0.8.0 will-zookeeper-0.8.0.jar;C:developmentMavenRepositoryorgapache ephra ephra-hbase-compat-1.2-cdh0.14.0-incubating ephra-hbase-compat-1.2-cdh-0.14.0-incubating.jar;C:developmentMavenRepositoryorgantlrantlr-runtime3.5.2antlr-runtime-3.5.2.jar;C:developmentMavenRepositoryjlinejline2.11jline-2.11.jar;C:developmentMavenRepositorysqllinesqlline1.2.0sqlline-1.2.0.jar;C:developmentMavenRepositorycomgoogleguavaguava13.0.1guava-13.0.1.jar;C:developmentMavenRepositoryjoda-timejoda-time1.6joda-time-1.6.jar;C:developmentMavenRepositorycomgithubstephencjcipjcip-annotations1.0-1jcip-annotations-1.0-1.jar;C:developmentMavenRepositoryorgcodehausjacksonjackson-core-asl1.8.8jackson-core-asl-1.8.8.jar;C:developmentMavenRepositoryorgcodehausjacksonjackson-mapper-asl1.8.8jackson-mapper-asl-1.8.8.jar;C:developmentMavenRepositoryjunitjunit4.12junit-4.12.jar;C:developmentMavenRepositoryorgapachehttpcomponentshttpclient4.0.1httpclient-4.0.1.jar;C:developmentMavenRepositoryorgiq80snappysnappy0.3snappy-0.3.jar;C:developmentMavenRepositoryorgapachehtracehtrace-core3.2.0-incubatinghtrace-core-3.2.0-incubating.jar;C:developmentMavenRepositorycommons-clicommons-cli1.2commons-cli-1.2.jar;C:developmentMavenRepositorycommons-codeccommons-codec1.7commons-codec-1.7.jar;C:developmentMavenRepositorycommons-collectionscommons-collections3.2.2commons-collections-3.2.2.jar;C:developmentMavenRepositoryorgapachecommonscommons-csv1.0commons-csv-1.0.jar;C:developmentMavenRepositoryorgapachehbasehbase-annotations1.2.0-cdh5.14.2hbase-annotations-1.2.0-cdh5.14.2.jar;C:3rdJavajdk1.8.0_212lib ools.jar;C:developmentMavenRepositoryorgapachehbasehbase-common1.2.0-cdh5.14.2hbase-common-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehbasehbase-hadoop-compat1.2.0-cdh5.14.2hbase-hadoop-compat-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehbasehbase-hadoop2-compat1.2.0-cdh5.14.2hbase-hadoop2-compat-1.2.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-annotations2.6.0-cdh5.14.2hadoop-annotations-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgjrubyjonijoni2.1.2joni-2.1.2.jar;C:developmentMavenRepositorycomsalesforcei18ni18n-util1.0.4i18n-util-1.0.4.jar;C:developmentMavenRepositorycomibmicuicu4j60.2icu4j-60.2.jar;C:developmentMavenRepositorycomibmicuicu4j-localespi60.2icu4j-localespi-60.2.jar;C:developmentMavenRepositorycomibmicuicu4j-charset60.2icu4j-charset-60.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-mapreduce-client-core2.6.0-cdh5.14.2hadoop-mapreduce-client-core-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryorgapachehadoophadoop-yarn-common2.6.0-cdh5.14.2hadoop-yarn-common-2.6.0-cdh5.14.2.jar;C:developmentMavenRepositoryjavaxxmlindjaxb-api2.2.2jaxb-api-2.2.2.jar;C:developmentMavenRepositoryjavaxxmlstreamstax-api1.0-2stax-api-1.0-2.jar;C:developmentMavenRepositoryjavaxactivationactivation1.1activation-1.1.jar;C:developmentMavenRepositoryjavaxservletservlet-api2.5servlet-api-2.5.jar;C:developmentMavenRepositorycomsunjerseyjersey-client1.9jersey-client-1.9.jar;C:developmentMavenRepositoryorgcodehausjacksonjackson-xc1.8.8jackson-xc-1.8.8.jar;C:developmentMavenRepositorycomsunjerseycontribsjersey-guice1.9jersey-guice-1.9.jar;C:developmentMavenRepositoryorgapacheavroavro1.7.6-cdh5.14.2avro-1.7.6-cdh5.14.2.jar;C:developmentMavenRepositorycomgoogleinjectextensionsguice-servlet3.0guice-servlet-3.0.jar;C:developmentMavenRepositoryio etty etty3.10.5.Final etty-3.10.5.Final.jar;C:developmentMavenRepositorymysqlmysql-connector-java5.1.43mysql-connector-java-5.1.43.jar;C:3rdJetBrainsIntelliJ IDEA 2019.1.3libidea_rt.jar 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.library.path=C:3rdJavajdk1.8.0_212in;C:WINDOWSSunJavain;C:WINDOWSsystem32;C:WINDOWS;C:3rdAnaconda2;C:3rdAnaconda2Librarymingw-w64in;C:3rdAnaconda2Libraryusrin;C:3rdAnaconda2Libraryin;C:3rdAnaconda2Scripts;C:Program Files (x86)IntelIntel(R) Management Engine ComponentsiCLS;C:Program FilesIntelIntel(R) Management Engine ComponentsiCLS;C:WINDOWSsystem32;C:WINDOWS;C:WINDOWSSystem32Wbem;C:WINDOWSSystem32WindowsPowerShellv1.0;C:Program Files (x86)IntelIntel(R) Management Engine ComponentsDAL;C:Program FilesIntelIntel(R) Management Engine ComponentsDAL;C:Program Files (x86)IntelIntel(R) Management Engine ComponentsIPT;C:Program FilesIntelIntel(R) Management Engine ComponentsIPT;C:3rdMATLABR2016a untimewin64;C:3rdMATLABR2016ain;C:3rdMATLABR2016apolyspacein;C:WINDOWSSystem32OpenSSH;C:Program FilesTortoiseSVNin;C:Userscf_pcDocumentsCaffeRelease;C:3rdscalascala-2.10.5in;C:3rdJavajdk1.8.0_212in;C:developmentapache-maven-3.6.1in;C:3rdmysql-8.0.16-winx64in;C:Program FilesIntelWiFiin;C:Program FilesCommon FilesIntelWirelessCommon;C:3rdhadoop-common-2.6.0in;C:Userscf_pcAppDataLocalMicrosoftWindowsApps;;C:Program FilesMicrosoft VS Codein;. 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.io.tmpdir=C:Userscf_pcAppDataLocalTemp19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.compiler=<NA> 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.name=Windows 10 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.arch=amd64 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.version=10.0 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.name=cf_pc 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.home=C:Userscf_pc 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.dir=C:developmentcfscalapp 19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=node3:2181 sessionTimeout=90000 watcher=hconnection-0x4bc59b270x0, quorum=node3:2181, baseZNode=/hbase 19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Opening socket connection to server node3/10.200.101.133:2181. Will not attempt to authenticate using SASL (unknown error) 19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Socket connection established to node3/10.200.101.133:2181, initiating session 19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Session establishment complete on server node3/10.200.101.133:2181, sessionid = 0x36ca2ccfed69558, negotiated timeout = 60000 19/09/24 00:52:24 INFO query.ConnectionQueryServicesImpl: HConnection established. Stacktrace for informational purposes: hconnection-0x4bc59b27 java.lang.Thread.getStackTrace(Thread.java:1559) org.apache.phoenix.util.LogUtil.getCallerStackTrace(LogUtil.java:55) org.apache.phoenix.query.ConnectionQueryServicesImpl.openConnection(ConnectionQueryServicesImpl.java:427) org.apache.phoenix.query.ConnectionQueryServicesImpl.access$400(ConnectionQueryServicesImpl.java:267) org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2515) org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2491) org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:76) org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2491) org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:255) org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:150) org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:221) java.sql.DriverManager.getConnection(DriverManager.java:664) java.sql.DriverManager.getConnection(DriverManager.java:208) org.apache.phoenix.mapreduce.util.ConnectionUtil.getConnection(ConnectionUtil.java:113) org.apache.phoenix.mapreduce.util.ConnectionUtil.getInputConnection(ConnectionUtil.java:58) org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getSelectColumnMetadataList(PhoenixConfigurationUtil.java:354) org.apache.phoenix.spark.PhoenixRDD.toDataFrame(PhoenixRDD.scala:118) org.apache.phoenix.spark.SparkSqlContextFunctions.phoenixTableAsDataFrame(SparkSqlContextFunctions.scala:39) com.fc.phoenixConnectMode$.getMode1(phoenixConnectMode.scala:16) com.fc.costDay$.main(costDay.scala:113) com.fc.costDay.main(costDay.scala) 19/09/24 00:52:25 INFO Configuration.deprecation: hadoop.native.lib is deprecated. Instead, use io.native.lib.available 19/09/24 00:52:27 INFO mapreduce.PhoenixInputFormat: UseSelectColumns=true, selectColumnList.size()=86, selectColumnList=ID,ASSET_ID,ASSET_NAME,ASSET_FIRST_DEGREE_ID,ASSET_FIRST_DEGREE_NAME,ASSET_SECOND_DEGREE_ID,ASSET_SECOND_DEGREE_NAME,GB_DEGREE_ID,GB_DEGREE_NAME,ASSET_USE_FIRST_DEGREE_ID,ASSET_USE_FIRST_DEGREE_NAME,ASSET_USE_SECOND_DEGREE_ID,ASSET_USE_SECOND_DEGREE_NAME,MANAGEMENT_TYPE_ID,MANAGEMENT_TYPE_NAME,ASSET_MODEL,FACTORY_NUMBER,ASSET_COUNTRY_ID,ASSET_COUNTRY_NAME,MANUFACTURER,SUPPLIER,SUPPLIER_TEL,ORIGINAL_VALUE,USE_DEPARTMENT_ID,USE_DEPARTMENT_NAME,USER_ID,USER_NAME,ASSET_LOCATION_OF_PARK_ID,ASSET_LOCATION_OF_PARK_NAME,ASSET_LOCATION_OF_BUILDING_ID,ASSET_LOCATION_OF_BUILDING_NAME,ASSET_LOCATION_OF_ROOM_ID,ASSET_LOCATION_OF_ROOM_NUMBER,PRODUCTION_DATE,ACCEPTANCE_DATE,REQUISITION_DATE,PERFORMANCE_INDEX,ASSET_STATE_ID,ASSET_STATE_NAME,INSPECTION_TYPE_ID,INSPECTION_TYPE_NAME,SEAL_DATE,SEAL_CAUSE,COST_ITEM_ID,COST_ITEM_NAME,ITEM_COMMENTS,UNSEAL_DATE,SCRAP_DATE,PURCHASE_NUMBER,WARRANTY_PERIOD,DEPRECIABLE_LIVES_ID,DEPRECIABLE_LIVES_NAME,MEASUREMENT_UNITS_ID,MEASUREMENT_UNITS_NAME,ANNEX,REMARK,ACCOUNTING_TYPE_ID,ACCOUNTING_TYPE_NAME,SYSTEM_TYPE_ID,SYSTEM_TYPE_NAME,ASSET_ID_PARENT,CLASSIFIED_LEVEL_ID,CLASSIFIED_LEVEL_NAME,ASSET_PICTURE,MILITARY_SPECIAL_CODE,CHECK_CYCLE_ID,CHECK_CYCLE_NAME,CHECK_DATE,CHECK_EFFECTIVE_DATE,CHECK_MODE_ID,CHECK_MODE_NAME,CHECK_DEPARTMENT_ID,CHECK_DEPARTMENT_NAME,RENT_STATUS_ID,RENT_STATUS_NAME,STORAGE_TIME,UPDATE_USER,UPDATE_TIME,IS_ON_PROCESS,IS_DELETED,FIRST_DEPARTMENT_ID,FIRST_DEPARTMENT_NAME,SECOND_DEPARTMENT_ID,SECOND_DEPARTMENT_NAME,CREATE_USER,CREATE_TIME root |-- ID: string (nullable = true) |-- FIRST_DEPARTMENT_ID: string (nullable = true) |-- ACTUAL_COST: double (nullable = true) |-- ORIGINAL_VALUE: double (nullable = true) |-- GENERATION_TIME: timestamp (nullable = false) 19/09/24 00:52:32 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:32 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:32 INFO mapreduce.PhoenixInputFormat: UseSelectColumns=true, selectColumnList.size()=86, selectColumnList=ID,ASSET_ID,ASSET_NAME,ASSET_FIRST_DEGREE_ID,ASSET_FIRST_DEGREE_NAME,ASSET_SECOND_DEGREE_ID,ASSET_SECOND_DEGREE_NAME,GB_DEGREE_ID,GB_DEGREE_NAME,ASSET_USE_FIRST_DEGREE_ID,ASSET_USE_FIRST_DEGREE_NAME,ASSET_USE_SECOND_DEGREE_ID,ASSET_USE_SECOND_DEGREE_NAME,MANAGEMENT_TYPE_ID,MANAGEMENT_TYPE_NAME,ASSET_MODEL,FACTORY_NUMBER,ASSET_COUNTRY_ID,ASSET_COUNTRY_NAME,MANUFACTURER,SUPPLIER,SUPPLIER_TEL,ORIGINAL_VALUE,USE_DEPARTMENT_ID,USE_DEPARTMENT_NAME,USER_ID,USER_NAME,ASSET_LOCATION_OF_PARK_ID,ASSET_LOCATION_OF_PARK_NAME,ASSET_LOCATION_OF_BUILDING_ID,ASSET_LOCATION_OF_BUILDING_NAME,ASSET_LOCATION_OF_ROOM_ID,ASSET_LOCATION_OF_ROOM_NUMBER,PRODUCTION_DATE,ACCEPTANCE_DATE,REQUISITION_DATE,PERFORMANCE_INDEX,ASSET_STATE_ID,ASSET_STATE_NAME,INSPECTION_TYPE_ID,INSPECTION_TYPE_NAME,SEAL_DATE,SEAL_CAUSE,COST_ITEM_ID,COST_ITEM_NAME,ITEM_COMMENTS,UNSEAL_DATE,SCRAP_DATE,PURCHASE_NUMBER,WARRANTY_PERIOD,DEPRECIABLE_LIVES_ID,DEPRECIABLE_LIVES_NAME,MEASUREMENT_UNITS_ID,MEASUREMENT_UNITS_NAME,ANNEX,REMARK,ACCOUNTING_TYPE_ID,ACCOUNTING_TYPE_NAME,SYSTEM_TYPE_ID,SYSTEM_TYPE_NAME,ASSET_ID_PARENT,CLASSIFIED_LEVEL_ID,CLASSIFIED_LEVEL_NAME,ASSET_PICTURE,MILITARY_SPECIAL_CODE,CHECK_CYCLE_ID,CHECK_CYCLE_NAME,CHECK_DATE,CHECK_EFFECTIVE_DATE,CHECK_MODE_ID,CHECK_MODE_NAME,CHECK_DEPARTMENT_ID,CHECK_DEPARTMENT_NAME,RENT_STATUS_ID,RENT_STATUS_NAME,STORAGE_TIME,UPDATE_USER,UPDATE_TIME,IS_ON_PROCESS,IS_DELETED,FIRST_DEPARTMENT_ID,FIRST_DEPARTMENT_NAME,SECOND_DEPARTMENT_ID,SECOND_DEPARTMENT_NAME,CREATE_USER,CREATE_TIME 19/09/24 00:52:32 INFO mapreduce.PhoenixInputFormat: Select Statement: SELECT "ID","0"."ASSET_ID","0"."ASSET_NAME","0"."ASSET_FIRST_DEGREE_ID","0"."ASSET_FIRST_DEGREE_NAME","0"."ASSET_SECOND_DEGREE_ID","0"."ASSET_SECOND_DEGREE_NAME","0"."GB_DEGREE_ID","0"."GB_DEGREE_NAME","0"."ASSET_USE_FIRST_DEGREE_ID","0"."ASSET_USE_FIRST_DEGREE_NAME","0"."ASSET_USE_SECOND_DEGREE_ID","0"."ASSET_USE_SECOND_DEGREE_NAME","0"."MANAGEMENT_TYPE_ID","0"."MANAGEMENT_TYPE_NAME","0"."ASSET_MODEL","0"."FACTORY_NUMBER","0"."ASSET_COUNTRY_ID","0"."ASSET_COUNTRY_NAME","0"."MANUFACTURER","0"."SUPPLIER","0"."SUPPLIER_TEL","0"."ORIGINAL_VALUE","0"."USE_DEPARTMENT_ID","0"."USE_DEPARTMENT_NAME","0"."USER_ID","0"."USER_NAME","0"."ASSET_LOCATION_OF_PARK_ID","0"."ASSET_LOCATION_OF_PARK_NAME","0"."ASSET_LOCATION_OF_BUILDING_ID","0"."ASSET_LOCATION_OF_BUILDING_NAME","0"."ASSET_LOCATION_OF_ROOM_ID","0"."ASSET_LOCATION_OF_ROOM_NUMBER","0"."PRODUCTION_DATE","0"."ACCEPTANCE_DATE","0"."REQUISITION_DATE","0"."PERFORMANCE_INDEX","0"."ASSET_STATE_ID","0"."ASSET_STATE_NAME","0"."INSPECTION_TYPE_ID","0"."INSPECTION_TYPE_NAME","0"."SEAL_DATE","0"."SEAL_CAUSE","0"."COST_ITEM_ID","0"."COST_ITEM_NAME","0"."ITEM_COMMENTS","0"."UNSEAL_DATE","0"."SCRAP_DATE","0"."PURCHASE_NUMBER","0"."WARRANTY_PERIOD","0"."DEPRECIABLE_LIVES_ID","0"."DEPRECIABLE_LIVES_NAME","0"."MEASUREMENT_UNITS_ID","0"."MEASUREMENT_UNITS_NAME","0"."ANNEX","0"."REMARK","0"."ACCOUNTING_TYPE_ID","0"."ACCOUNTING_TYPE_NAME","0"."SYSTEM_TYPE_ID","0"."SYSTEM_TYPE_NAME","0"."ASSET_ID_PARENT","0"."CLASSIFIED_LEVEL_ID","0"."CLASSIFIED_LEVEL_NAME","0"."ASSET_PICTURE","0"."MILITARY_SPECIAL_CODE","0"."CHECK_CYCLE_ID","0"."CHECK_CYCLE_NAME","0"."CHECK_DATE","0"."CHECK_EFFECTIVE_DATE","0"."CHECK_MODE_ID","0"."CHECK_MODE_NAME","0"."CHECK_DEPARTMENT_ID","0"."CHECK_DEPARTMENT_NAME","0"."RENT_STATUS_ID","0"."RENT_STATUS_NAME","0"."STORAGE_TIME","0"."UPDATE_USER","0"."UPDATE_TIME","0"."IS_ON_PROCESS","0"."IS_DELETED","0"."FIRST_DEPARTMENT_ID","0"."FIRST_DEPARTMENT_NAME","0"."SECOND_DEPARTMENT_ID","0"."SECOND_DEPARTMENT_NAME","0"."CREATE_USER","0"."CREATE_TIME" FROM ASSET_NORMAL 19/09/24 00:52:32 INFO zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x22617270 connecting to ZooKeeper ensemble=node3:2181 19/09/24 00:52:32 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=node3:2181 sessionTimeout=90000 watcher=hconnection-0x226172700x0, quorum=node3:2181, baseZNode=/hbase 19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Opening socket connection to server node3/10.200.101.133:2181. Will not attempt to authenticate using SASL (unknown error) 19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Socket connection established to node3/10.200.101.133:2181, initiating session 19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Session establishment complete on server node3/10.200.101.133:2181, sessionid = 0x36ca2ccfed69559, negotiated timeout = 60000 19/09/24 00:52:32 INFO util.RegionSizeCalculator: Calculating region sizes for table "IDX_ASSET_NORMAL". 19/09/24 00:52:32 INFO client.ConnectionManager$HConnectionImplementation: Closing master protocol: MasterService 19/09/24 00:52:32 INFO client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x36ca2ccfed69559 19/09/24 00:52:32 INFO zookeeper.ZooKeeper: Session: 0x36ca2ccfed69559 closed 19/09/24 00:52:32 INFO zookeeper.ClientCnxn: EventThread shut down 19/09/24 00:52:32 INFO spark.SparkContext: Starting job: count at costDay.scala:139 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Registering RDD 7 (count at costDay.scala:139) 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Got job 0 (count at costDay.scala:139) with 1 output partitions 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Final stage: ResultStage 1 (count at costDay.scala:139) 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 0) 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 0) 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at count at costDay.scala:139), which has no missing parents 19/09/24 00:52:33 INFO storage.MemoryStore: Block broadcast_1 stored as values in memory (estimated size 24.8 KB, free 477.9 MB) 19/09/24 00:52:33 INFO storage.MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 9.5 KB, free 477.9 MB) 19/09/24 00:52:33 INFO storage.BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:54720 (size: 9.5 KB, free: 478.1 MB) 19/09/24 00:52:33 INFO spark.SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:33 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at count at costDay.scala:139) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:33 INFO scheduler.TaskSchedulerImpl: Adding task set 0.0 with 1 tasks 19/09/24 00:52:33 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, executor driver, partition 0, ANY, 2538 bytes) 19/09/24 00:52:33 INFO executor.Executor: Running task 0.0 in stage 0.0 (TID 0) 19/09/24 00:52:33 INFO rdd.NewHadoopRDD: Input split: org.apache.phoenix.mapreduce.PhoenixInputSplit@20b488 19/09/24 00:52:33 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:33 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:33 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:33 INFO codegen.GeneratePredicate: Code generated in 309.4486 ms 19/09/24 00:52:33 INFO codegen.GenerateUnsafeProjection: Code generated in 20.3535 ms 19/09/24 00:52:34 INFO codegen.GenerateMutableProjection: Code generated in 10.5156 ms 19/09/24 00:52:34 INFO codegen.GenerateMutableProjection: Code generated in 10.4614 ms 19/09/24 00:52:34 INFO codegen.GenerateUnsafeRowJoiner: Code generated in 6.8774 ms 19/09/24 00:52:34 INFO codegen.GenerateUnsafeProjection: Code generated in 6.7907 ms 19/09/24 00:52:43 INFO executor.Executor: Finished task 0.0 in stage 0.0 (TID 0). 2629 bytes result sent to driver 19/09/24 00:52:43 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 10699 ms on localhost (executor driver) (1/1) 19/09/24 00:52:43 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 19/09/24 00:52:43 INFO scheduler.DAGScheduler: ShuffleMapStage 0 (count at costDay.scala:139) finished in 10.745 s 19/09/24 00:52:43 INFO scheduler.DAGScheduler: looking for newly runnable stages 19/09/24 00:52:43 INFO scheduler.DAGScheduler: running: Set() 19/09/24 00:52:43 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 1) 19/09/24 00:52:43 INFO scheduler.DAGScheduler: failed: Set() 19/09/24 00:52:43 INFO scheduler.DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[10] at count at costDay.scala:139), which has no missing parents 19/09/24 00:52:43 INFO storage.MemoryStore: Block broadcast_2 stored as values in memory (estimated size 25.3 KB, free 477.9 MB) 19/09/24 00:52:43 INFO storage.MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 9.8 KB, free 477.8 MB) 19/09/24 00:52:43 INFO storage.BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:54720 (size: 9.8 KB, free: 478.1 MB) 19/09/24 00:52:43 INFO spark.SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:43 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at count at costDay.scala:139) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:43 INFO scheduler.TaskSchedulerImpl: Adding task set 1.0 with 1 tasks 19/09/24 00:52:43 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, executor driver, partition 0, NODE_LOCAL, 1999 bytes) 19/09/24 00:52:43 INFO executor.Executor: Running task 0.0 in stage 1.0 (TID 1) 19/09/24 00:52:43 INFO storage.ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks 19/09/24 00:52:44 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 20 ms 19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 22.44 ms 19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 13.992 ms 19/09/24 00:52:44 INFO executor.Executor: Finished task 0.0 in stage 1.0 (TID 1). 1959 bytes result sent to driver 19/09/24 00:52:44 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 199 ms on localhost (executor driver) (1/1) 19/09/24 00:52:44 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool 19/09/24 00:52:44 INFO scheduler.DAGScheduler: ResultStage 1 (count at costDay.scala:139) finished in 0.205 s 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Job 0 finished: count at costDay.scala:139, took 11.157661 s 50858 19/09/24 00:52:44 INFO spark.SparkContext: Starting job: describe at costDay.scala:141 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Registering RDD 14 (describe at costDay.scala:141) 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Got job 1 (describe at costDay.scala:141) with 1 output partitions 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Final stage: ResultStage 3 (describe at costDay.scala:141) 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 2) 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 2) 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[14] at describe at costDay.scala:141), which has no missing parents 19/09/24 00:52:44 INFO storage.MemoryStore: Block broadcast_3 stored as values in memory (estimated size 27.2 KB, free 477.8 MB) 19/09/24 00:52:44 INFO storage.MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 10.4 KB, free 477.8 MB) 19/09/24 00:52:44 INFO storage.BlockManagerInfo: Added broadcast_3_piece0 in memory on localhost:54720 (size: 10.4 KB, free: 478.1 MB) 19/09/24 00:52:44 INFO spark.SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:44 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[14] at describe at costDay.scala:141) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:44 INFO scheduler.TaskSchedulerImpl: Adding task set 2.0 with 1 tasks 19/09/24 00:52:44 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 2.0 (TID 2, localhost, executor driver, partition 0, ANY, 2538 bytes) 19/09/24 00:52:44 INFO executor.Executor: Running task 0.0 in stage 2.0 (TID 2) 19/09/24 00:52:44 INFO rdd.NewHadoopRDD: Input split: org.apache.phoenix.mapreduce.PhoenixInputSplit@20b488 19/09/24 00:52:44 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:44 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:44 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:44 INFO codegen.GenerateUnsafeProjection: Code generated in 11.405 ms 19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 10.5886 ms 19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 39.2201 ms 19/09/24 00:52:44 INFO codegen.GenerateUnsafeRowJoiner: Code generated in 8.3737 ms 19/09/24 00:52:44 INFO codegen.GenerateUnsafeProjection: Code generated in 22.542 ms 19/09/24 00:52:45 INFO storage.BlockManagerInfo: Removed broadcast_2_piece0 on localhost:54720 in memory (size: 9.8 KB, free: 478.1 MB) 19/09/24 00:52:53 INFO executor.Executor: Finished task 0.0 in stage 2.0 (TID 2). 2629 bytes result sent to driver 19/09/24 00:52:53 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 2.0 (TID 2) in 8824 ms on localhost (executor driver) (1/1) 19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool 19/09/24 00:52:53 INFO scheduler.DAGScheduler: ShuffleMapStage 2 (describe at costDay.scala:141) finished in 8.825 s 19/09/24 00:52:53 INFO scheduler.DAGScheduler: looking for newly runnable stages 19/09/24 00:52:53 INFO scheduler.DAGScheduler: running: Set() 19/09/24 00:52:53 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 3) 19/09/24 00:52:53 INFO scheduler.DAGScheduler: failed: Set() 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting ResultStage 3 (MapPartitionsRDD[18] at describe at costDay.scala:141), which has no missing parents 19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_4 stored as values in memory (estimated size 28.7 KB, free 477.8 MB) 19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 11.0 KB, free 477.8 MB) 19/09/24 00:52:53 INFO storage.BlockManagerInfo: Added broadcast_4_piece0 in memory on localhost:54720 (size: 11.0 KB, free: 478.1 MB) 19/09/24 00:52:53 INFO spark.SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[18] at describe at costDay.scala:141) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Adding task set 3.0 with 1 tasks 19/09/24 00:52:53 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 3.0 (TID 3, localhost, executor driver, partition 0, NODE_LOCAL, 1999 bytes) 19/09/24 00:52:53 INFO executor.Executor: Running task 0.0 in stage 3.0 (TID 3) 19/09/24 00:52:53 INFO storage.ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks 19/09/24 00:52:53 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms 19/09/24 00:52:53 INFO codegen.GenerateMutableProjection: Code generated in 21.5454 ms 19/09/24 00:52:53 INFO codegen.GenerateMutableProjection: Code generated in 10.7314 ms 19/09/24 00:52:53 INFO codegen.GenerateUnsafeProjection: Code generated in 13.0969 ms 19/09/24 00:52:53 INFO codegen.GenerateSafeProjection: Code generated in 8.2397 ms 19/09/24 00:52:53 INFO executor.Executor: Finished task 0.0 in stage 3.0 (TID 3). 2138 bytes result sent to driver 19/09/24 00:52:53 INFO scheduler.DAGScheduler: ResultStage 3 (describe at costDay.scala:141) finished in 0.097 s 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Job 1 finished: describe at costDay.scala:141, took 8.948356 s 19/09/24 00:52:53 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 3.0 (TID 3) in 97 ms on localhost (executor driver) (1/1) 19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool +-------+--------------------+ |summary| ORIGINAL_VALUE| +-------+--------------------+ | count| 50858| | mean|2.427485546653306E12| | stddev|5.474385018305400...| | min| -7970934.0| | max|1.234567890123456...| +-------+--------------------+ 19/09/24 00:52:53 INFO spark.SparkContext: Starting job: show at costDay.scala:142 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Got job 2 (show at costDay.scala:142) with 1 output partitions 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Final stage: ResultStage 4 (show at costDay.scala:142) 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Parents of final stage: List() 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Missing parents: List() 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting ResultStage 4 (MapPartitionsRDD[22] at show at costDay.scala:142), which has no missing parents 19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_5 stored as values in memory (estimated size 23.6 KB, free 477.8 MB) 19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 9.0 KB, free 477.8 MB) 19/09/24 00:52:53 INFO storage.BlockManagerInfo: Added broadcast_5_piece0 in memory on localhost:54720 (size: 9.0 KB, free: 478.1 MB) 19/09/24 00:52:53 INFO spark.SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1004 19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[22] at show at costDay.scala:142) (first 15 tasks are for partitions Vector(0)) 19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Adding task set 4.0 with 1 tasks 19/09/24 00:52:53 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 4.0 (TID 4, localhost, executor driver, partition 0, ANY, 2549 bytes) 19/09/24 00:52:53 INFO executor.Executor: Running task 0.0 in stage 4.0 (TID 4) 19/09/24 00:52:53 INFO rdd.NewHadoopRDD: Input split: org.apache.phoenix.mapreduce.PhoenixInputSplit@20b488 19/09/24 00:52:53 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum 19/09/24 00:52:53 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase 19/09/24 00:52:53 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster 19/09/24 00:52:53 INFO codegen.GenerateUnsafeProjection: Code generated in 33.6563 ms 19/09/24 00:52:53 INFO codegen.GenerateSafeProjection: Code generated in 7.0589 ms 19/09/24 00:52:54 INFO executor.Executor: Finished task 0.0 in stage 4.0 (TID 4). 5544 bytes result sent to driver 19/09/24 00:52:54 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 4.0 (TID 4) in 476 ms on localhost (executor driver) (1/1) 19/09/24 00:52:54 INFO scheduler.DAGScheduler: ResultStage 4 (show at costDay.scala:142) finished in 0.466 s 19/09/24 00:52:54 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 4.0, whose tasks have all completed, from pool 19/09/24 00:52:54 INFO scheduler.DAGScheduler: Job 2 finished: show at costDay.scala:142, took 0.489113 s +--------------------------------+-------------------+---------------------+--------------+-----------------------+ |ID |FIRST_DEPARTMENT_ID|ACTUAL_COST |ORIGINAL_VALUE|GENERATION_TIME | +--------------------------------+-------------------+---------------------+--------------+-----------------------+ |d25bb550a290457382c175b0e57c0982|1149564326588321792|0.0010410958904109589|2.0 |2019-09-24 00:52:31.864| |492016e2f7ec4cd18615c164c92c6c6d|1149564326584127489|5.205479452054794E-4 |1.0 |2019-09-24 00:52:31.864| |1d138a7401bd493da12f8f8323e9dee0|1149564326588321710|0.00353972602739726 |34.0 |2019-09-24 00:52:31.864| |09718925d34b4a099e09a30a0621ded8|1149564326588321710|0.002891643835616438 |11.11 |2019-09-24 00:52:31.864| |d5cfd5e898464130b71530d74b43e9d1|1149564326584127489|3135.8751712328767 |9638690.0 |2019-09-24 00:52:31.864| |6b39ac96b8734103b2413520d3195ee6|1149564326584127489|1744.2413835616437 |6701559.0 |2019-09-24 00:52:31.864| |8d20d0abd04d49cea3e52d9ca67e39da|1149569393202696195|0.22175342465753425 |852.0 |2019-09-24 00:52:31.864| |66ae7e7c7a104cea99615358e12c03b0|1149569393202696195|1.0410958904109588 |2000.0 |2019-09-24 00:52:31.864| |d49b0324bbf14b70adefe8b1d9163db2|1149569393202696195|1.0410958904109588 |2000.0 |2019-09-24 00:52:31.864| |d4d701514a2a425e8192acf47bb57f9b|1149569393202696195|0.032534246575342464 |100.0 |2019-09-24 00:52:31.864| |d6a016c618c1455ca0e2c7d73ba947ac|1149569393202696195|0.6506849315068494 |2000.0 |2019-09-24 00:52:31.864| |5dfa3be825464ddd98764b2790720fae|1149569393202696195|147.91532534246576 |454645.0 |2019-09-24 00:52:31.864| |6e5653ef4aaa4c03bcd00fbeb1e6811d|1149569393202696195|0.1952054794520548 |600.0 |2019-09-24 00:52:31.864| |32bd2654082645cba35527d50e0d52f9|1149569393202696195|0.6506849315068494 |2000.0 |2019-09-24 00:52:31.864| |8ed4424408bc458dbe200acffe5733bf|1149564326584127488|5.205479452054794E-4 |1.0 |2019-09-24 00:52:31.864| |1b2faa31f139461488847e77eacd794a|1149564326584127488|33499.042109589034 |6.4353423E7 |2019-09-24 00:52:31.864| |f398245c9ccc4760a5eb3251db3680bf|1149564326584127488|33499.042109589034 |6.4353423E7 |2019-09-24 00:52:31.864| |2696de9733d247e5bf88573244f36ba2|1149564326584127488|0.011452054794520548 |22.0 |2019-09-24 00:52:31.864| |9c8cfad3d4334b37a7b9beb56b528c22|1149569976173203457|0.06506849315068493 |200.0 |2019-09-24 00:52:31.864| |3e2721b79e754a798d0be940ae011d72|1149569976173203457|0.004001712328767123 |12.3 |2019-09-24 00:52:31.864| +--------------------------------+-------------------+---------------------+--------------+-----------------------+ only showing top 20 rows 19/09/24 00:52:54 INFO spark.SparkContext: Invoking stop() from shutdown hook 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/static/sql,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/execution,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/metrics/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/api,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/static,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/json,null} 19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs,null} 19/09/24 00:52:54 INFO ui.SparkUI: Stopped Spark web UI at http://10.200.74.155:4040 19/09/24 00:52:54 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 19/09/24 00:52:54 INFO storage.MemoryStore: MemoryStore cleared 19/09/24 00:52:54 INFO storage.BlockManager: BlockManager stopped 19/09/24 00:52:54 INFO storage.BlockManagerMaster: BlockManagerMaster stopped 19/09/24 00:52:54 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 19/09/24 00:52:54 INFO spark.SparkContext: Successfully stopped SparkContext 19/09/24 00:52:54 INFO util.ShutdownHookManager: Shutdown hook called 19/09/24 00:52:54 INFO util.ShutdownHookManager: Deleting directory C:Userscf_pcAppDataLocalTempspark-4fcfa10d-c258-46b7-b4f5-ee977276fa00 Process finished with exit code 0
执行打包操作，返回信息：
C:3rdJavajdk1.8.0_212injava.exe -Dmaven.multiModuleProjectDirectory=C:developmentcfscalapp -Dmaven.home=C:developmentapache-maven-3.6.1 -Dclassworlds.conf=C:developmentapache-maven-3.6.1inm2.conf "-javaagent:C:3rdJetBrainsIntelliJ IDEA 2019.1.3libidea_rt.jar=49792:C:3rdJetBrainsIntelliJ IDEA 2019.1.3in" -Dfile.encoding=UTF-8 -classpath C:developmentapache-maven-3.6.1ootplexus-classworlds-2.6.0.jar org.codehaus.classworlds.Launcher -Didea.version2019.1.3 -s C:developmentapache-maven-3.6.1confsettings.xml -Dmaven.repo.local=C:developmentMavenRepository package [INFO] Scanning for projects... [INFO] [INFO] ---------------------------< com.fc:scalapp >--------------------------- [INFO] Building scalapp 1.0-SNAPSHOT [INFO] --------------------------------[ jar ]--------------------------------- [INFO] [INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ scalapp --- [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] Copying 0 resource [INFO] [INFO] --- maven-compiler-plugin:3.1:compile (default-compile) @ scalapp --- [INFO] Nothing to compile - all classes are up to date [INFO] [INFO] --- maven-scala-plugin:2.15.2:compile (default) @ scalapp --- [INFO] Checking for multiple versions of scala [WARNING] Expected all dependencies to require Scala version: 2.10.5 [WARNING] com.twitter:chill_2.10:0.5.0 requires scala version: 2.10.4 [WARNING] Multiple versions of scala libraries detected! [INFO] includes = [**/*.java,**/*.scala,] [INFO] excludes = [] [INFO] C:developmentcfscalappsrcmainscala:-1: info: compiling [INFO] Compiling 4 source files to C:developmentcfscalapp argetclasses at 1569255082006 [WARNING] warning: there were 1 feature warning(s); re-run with -feature for details [WARNING] one warning found [INFO] prepare-compile in 0 s [INFO] compile in 7 s [INFO] [INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ scalapp --- [WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent! [INFO] skip non existing resourceDirectory C:developmentcfscalappsrc est esources [INFO] [INFO] --- maven-compiler-plugin:3.1:testCompile (default-testCompile) @ scalapp --- [INFO] Nothing to compile - all classes are up to date [INFO] [INFO] --- maven-scala-plugin:2.15.2:testCompile (default) @ scalapp --- [INFO] Checking for multiple versions of scala [WARNING] Expected all dependencies to require Scala version: 2.10.5 [WARNING] com.twitter:chill_2.10:0.5.0 requires scala version: 2.10.4 [WARNING] Multiple versions of scala libraries detected! [INFO] includes = [**/*.java,**/*.scala,] [INFO] excludes = [] [WARNING] No source files found. [INFO] [INFO] --- maven-surefire-plugin:2.12.4:test (default-test) @ scalapp --- [INFO] [INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ scalapp --- [INFO] Building jar: C:developmentcfscalapp argetscalapp-1.0-SNAPSHOT.jar [INFO] [INFO] --- maven-shade-plugin:3.1.0:shade (default) @ scalapp --- [INFO] Including org.apache.spark:spark-core_2.10:jar:1.6.0-cdh5.14.2 in the shaded jar. [INFO] Including org.apache.avro:avro-mapred:jar:hadoop2:1.7.6-cdh5.14.2 in the shaded jar. …… [INFO] Including mysql:mysql-connector-java:jar:5.1.43 in the shaded jar. [WARNING] commons-collections-3.2.2.jar, commons-beanutils-1.7.0.jar, commons-beanutils-core-1.8.0.jar define 10 overlapping classes: [WARNING] - org.apache.commons.collections.FastHashMap$EntrySet [WARNING] - org.apache.commons.collections.FastHashMap$KeySet [WARNING] - org.apache.commons.collections.FastHashMap$CollectionView$CollectionViewIterator [WARNING] - org.apache.commons.collections.ArrayStack [WARNING] - org.apache.commons.collections.FastHashMap$Values [WARNING] - org.apache.commons.collections.FastHashMap$CollectionView [WARNING] - org.apache.commons.collections.FastHashMap$1 [WARNING] - org.apache.commons.collections.Buffer [WARNING] - org.apache.commons.collections.FastHashMap [WARNING] - org.apache.commons.collections.BufferUnderflowException …… [WARNING] mvn dependency:tree -Ddetail=true and the above output. [WARNING] See http://maven.apache.org/plugins/maven-shade-plugin/ [INFO] Replacing original artifact with shaded artifact. [INFO] Replacing C:developmentcfscalapp argetscalapp-1.0-SNAPSHOT.jar with C:developmentcfscalapp argetscalapp-1.0-SNAPSHOT-shaded.jar [INFO] ------------------------------------------------------------------------ [INFO] BUILD SUCCESS [INFO] ------------------------------------------------------------------------ [INFO] Total time: 01:00 min [INFO] Finished at: 2019-09-24T00:12:08+08:00 [INFO] ------------------------------------------------------------------------ Process finished with exit code 0
上传到服务器上，执行命令：
spark-submit --class com.fc.costDay --executor-memory 500m --total-executor-cores 1 /home/cf/scalapp-1.0-SNAPSHOT.jar
报错：
java.lang.ClassNotFoundException: Class org.apache.phoenix.spark.PhoenixRecordWritable not found
参考：https://www.jianshu.com/p/f336f7e5f31b，更改执行命令为：
spark-submit --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2 --class com.fc.costDay --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar
仍然报错：
File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/……
完整错误信息如下：
[root@node1 ~]# spark-submit --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2 --class com.fc.costDay --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar 19/09/24 00:01:36 INFO client.RMProxy: Connecting to ResourceManager at node1/10.200.101.131:8032 19/09/24 00:01:36 INFO yarn.Client: Requesting a new application from cluster with 3 NodeManagers 19/09/24 00:01:36 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (40874 MB per container) 19/09/24 00:01:36 INFO yarn.Client: Will allocate AM container, with 4505 MB memory including 409 MB overhead 19/09/24 00:01:36 INFO yarn.Client: Setting up container launch context for our AM 19/09/24 00:01:36 INFO yarn.Client: Setting up the launch environment for our AM container 19/09/24 00:01:37 INFO yarn.Client: Preparing resources for our AM container 19/09/24 00:01:37 INFO yarn.Client: Uploading resource file:/home/cf/scalapp-1.0-SNAPSHOT.jar -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/scalapp-1.0-SNAPSHOT.jar 19/09/24 00:01:38 INFO yarn.Client: Uploading resource file:/tmp/spark-2c548285-8012-414b-ab4e-797a164e38bc/__spark_conf__8997139291240118668.zip -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip 19/09/24 00:01:38 INFO spark.SecurityManager: Changing view acls to: root 19/09/24 00:01:38 INFO spark.SecurityManager: Changing modify acls to: root 19/09/24 00:01:38 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root) 19/09/24 00:01:38 INFO yarn.Client: Submitting application 105 to ResourceManager 19/09/24 00:01:38 INFO impl.YarnClientImpl: Submitted application application_1566100765602_0105 19/09/24 00:01:39 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:39 INFO yarn.Client: client token: N/A diagnostics: N/A ApplicationMaster host: N/A ApplicationMaster RPC port: -1 queue: root.users.root start time: 1569254498669 final status: UNDEFINED tracking URL: http://node1:8088/proxy/application_1566100765602_0105/ user: root 19/09/24 00:01:40 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:41 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:42 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:43 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:44 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:45 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:46 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:47 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:48 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:49 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:50 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:51 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:52 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:53 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:54 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:55 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:56 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:57 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:58 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:01:59 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:02:00 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED) 19/09/24 00:02:01 INFO yarn.Client: Application report for application_1566100765602_0105 (state: FAILED) 19/09/24 00:02:01 INFO yarn.Client: client token: N/A diagnostics: Application application_1566100765602_0105 failed 2 times due to AM Container for appattempt_1566100765602_0105_000002 exited with exitCode: -1000 For more detailed output, check application tracking page:http://node1:8088/proxy/application_1566100765602_0105/Then, click on links to logs of each attempt. Diagnostics: File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip java.io.FileNotFoundException: File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1269) at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1261) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1261) at org.apache.hadoop.yarn.util.FSDownload.copy(FSDownload.java:251) at org.apache.hadoop.yarn.util.FSDownload.access$000(FSDownload.java:61) at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:364) at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:362) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1920) at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:361) at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:60) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Failing this attempt. Failing the application. ApplicationMaster host: N/A ApplicationMaster RPC port: -1 queue: root.users.root start time: 1569254498669 final status: FAILED tracking URL: http://node1:8088/cluster/app/application_1566100765602_0105 user: root Exception in thread "main" org.apache.spark.SparkException: Application application_1566100765602_0105 finished with failed status at org.apache.spark.deploy.yarn.Client.run(Client.scala:1025) at org.apache.spark.deploy.yarn.Client$.main(Client.scala:1072) at org.apache.spark.deploy.yarn.Client.main(Client.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:730) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 19/09/24 00:02:01 INFO util.ShutdownHookManager: Shutdown hook called 19/09/24 00:02:01 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-2c548285-8012-414b-ab4e-797a164e38bc
参考：https://blog.csdn.net/adorechen/article/details/78746363，将源代码中设定本地执行的语句注释掉：
val conf = new SparkConf() .setAppName("fdsf") // .setMaster("local") //本地执行
重新打包，执行以上语句，成功！返回如下信息：
[root@node1 ~]# spark-submit --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2 --class com.fc.costDay --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar 19/09/24 00:13:58 INFO client.RMProxy: Connecting to ResourceManager at node1/10.200.101.131:8032 19/09/24 00:13:58 INFO yarn.Client: Requesting a new application from cluster with 3 NodeManagers 19/09/24 00:13:58 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (40874 MB per container) 19/09/24 00:13:58 INFO yarn.Client: Will allocate AM container, with 4505 MB memory including 409 MB overhead 19/09/24 00:13:58 INFO yarn.Client: Setting up container launch context for our AM 19/09/24 00:13:58 INFO yarn.Client: Setting up the launch environment for our AM container 19/09/24 00:13:58 INFO yarn.Client: Preparing resources for our AM container 19/09/24 00:13:59 INFO yarn.Client: Uploading resource file:/home/cf/scalapp-1.0-SNAPSHOT.jar -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0106/scalapp-1.0-SNAPSHOT.jar 19/09/24 00:14:00 INFO yarn.Client: Uploading resource file:/tmp/spark-875aaa2a-1c5d-4c6a-95a6-12f276a80054/__spark_conf__4530217919668141816.zip -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0106/__spark_conf__4530217919668141816.zip 19/09/24 00:14:00 INFO spark.SecurityManager: Changing view acls to: root 19/09/24 00:14:00 INFO spark.SecurityManager: Changing modify acls to: root 19/09/24 00:14:00 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root) 19/09/24 00:14:00 INFO yarn.Client: Submitting application 106 to ResourceManager 19/09/24 00:14:00 INFO impl.YarnClientImpl: Submitted application application_1566100765602_0106 19/09/24 00:14:01 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:01 INFO yarn.Client: client token: N/A diagnostics: N/A ApplicationMaster host: N/A ApplicationMaster RPC port: -1 queue: root.users.root start time: 1569255240698 final status: UNDEFINED tracking URL: http://node1:8088/proxy/application_1566100765602_0106/ user: root 19/09/24 00:14:02 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:03 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:04 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:05 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED) 19/09/24 00:14:06 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:06 INFO yarn.Client: client token: N/A diagnostics: N/A ApplicationMaster host: 10.200.101.135 ApplicationMaster RPC port: 0 queue: root.users.root start time: 1569255240698 final status: UNDEFINED tracking URL: http://node1:8088/proxy/application_1566100765602_0106/ user: root 19/09/24 00:14:07 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:08 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:09 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:10 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:11 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:12 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:13 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:14 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:15 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:16 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:17 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:18 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:19 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:20 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:21 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:22 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:23 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:24 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:25 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:26 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:27 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:28 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:29 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:30 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:31 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:32 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:33 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:34 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING) 19/09/24 00:14:35 INFO yarn.Client: Application report for application_1566100765602_0106 (state: FINISHED) 19/09/24 00:14:35 INFO yarn.Client: client token: N/A diagnostics: N/A ApplicationMaster host: 10.200.101.135 ApplicationMaster RPC port: 0 queue: root.users.root start time: 1569255240698 final status: SUCCEEDED tracking URL: http://node1:8088/proxy/application_1566100765602_0106/ user: root 19/09/24 00:14:35 INFO util.ShutdownHookManager: Shutdown hook called 19/09/24 00:14:35 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-875aaa2a-1c5d-4c6a-95a6-12f276a80054
编排Spark任务：
选择jar包：
添加jar包：
填写jar包和主类等信息：
填写执行模式信息：
保存任务：
执行，返回执行信息： 
错误详细信息为：
2019-09-24 01:30:02,556 ERROR org.apache.oozie.command.wf.SignalXCommand: SERVER[node1] USER[admin] GROUP[-] TOKEN[] APP[Cf0924] JOB[0000000-190922201507304-oozie-oozi-W] ACTION[0000000-190922201507304-oozie-oozi-W@:start:] Workflow action failed : E0700: XML error, For input string: "" org.apache.oozie.workflow.WorkflowException: E0700: XML error, For input string: "" at org.apache.oozie.service.LiteWorkflowStoreService.getUserRetryMax(LiteWorkflowStoreService.java:171) at org.apache.oozie.service.LiteWorkflowStoreService.liteExecute(LiteWorkflowStoreService.java:122) at org.apache.oozie.service.LiteWorkflowStoreService$LiteActionHandler.start(LiteWorkflowStoreService.java:252) at org.apache.oozie.workflow.lite.ActionNodeHandler.enter(ActionNodeHandler.java:33) at org.apache.oozie.workflow.lite.LiteWorkflowInstance.signal(LiteWorkflowInstance.java:214) at org.apache.oozie.workflow.lite.LiteWorkflowInstance.signal(LiteWorkflowInstance.java:290) at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:212) at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:82) at org.apache.oozie.command.XCommand.call(XCommand.java:286) at org.apache.oozie.command.XCommand.call(XCommand.java:356) at org.apache.oozie.command.wf.ActionEndXCommand.execute(ActionEndXCommand.java:280) at org.apache.oozie.command.wf.ActionEndXCommand.execute(ActionEndXCommand.java:61) at org.apache.oozie.command.XCommand.call(XCommand.java:286) at org.apache.oozie.command.XCommand.call(XCommand.java:356) at org.apache.oozie.command.wf.ActionStartXCommand.callActionEnd(ActionStartXCommand.java:340) at org.apache.oozie.command.wf.ActionStartXCommand.execute(ActionStartXCommand.java:326) at org.apache.oozie.command.wf.ActionStartXCommand.execute(ActionStartXCommand.java:63) at org.apache.oozie.command.XCommand.call(XCommand.java:286) at org.apache.oozie.command.XCommand.call(XCommand.java:356) at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:459) at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:82) at org.apache.oozie.command.XCommand.call(XCommand.java:286) at org.apache.oozie.DagEngine.start(DagEngine.java:202) at org.apache.oozie.servlet.V1JobServlet.startWorkflowJob(V1JobServlet.java:332) at org.apache.oozie.servlet.V1JobServlet.startJob(V1JobServlet.java:76) at org.apache.oozie.servlet.BaseJobServlet.doPut(BaseJobServlet.java:80) at javax.servlet.http.HttpServlet.service(HttpServlet.java:730) at org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:289) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.oozie.servlet.AuthFilter$2.doFilter(AuthFilter.java:171) at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:631) at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:579) at org.apache.oozie.servlet.AuthFilter.doFilter(AuthFilter.java:176) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.oozie.servlet.HostnameFilter.doFilter(HostnameFilter.java:86) at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235) at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206) at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233) at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191) at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127) at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103) at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109) at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293) at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859) at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:610) at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:503) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.NumberFormatException: For input string: "" at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65) at java.lang.Integer.parseInt(Integer.java:592) at java.lang.Integer.parseInt(Integer.java:615) at org.apache.oozie.service.LiteWorkflowStoreService.getUserRetryMax(LiteWorkflowStoreService.java:164) ... 49 more
关联到oozie任务信息：
点开链接，查看任务详情：
附依赖文件：
phoenixConnectMode.scala ：
package com.fc import org.apache.hadoop.conf.Configuration import org.apache.spark.sql.{DataFrame, SQLContext} import org.apache.phoenix.spark._ object phoenixConnectMode { private val zookeeper = "node3:2181" def getMode1(sqlContext: SQLContext, tableName: String, columns: Array[String]): DataFrame = { val configuration = new Configuration() configuration.set("phoenix.schema.isNamespaceMappingEnabled", "true") configuration.set("phoenix.schema.mapSystemTablesToNamespace", "true") configuration.set("hbase.zookeeper.quorum", zookeeper) val df = sqlContext.phoenixTableAsDataFrame(tableName, columns, conf = configuration) df } }
其中 configuration.set("phoenix.schema.isNamespaceMappingEnabled", "true") 需要根据具体环境修改为 true 或者 false
timeUtil.scala ：
package com.fc import java.text.SimpleDateFormat import java.util.Calendar import org.joda.time.DateTime object timeUtil { final val ONE_DAY_Millis = 3600*24*1000 final val SECOND_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss" final val MILLISECOND_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS" final val DAY_DATE_FORMAT_ONE = "yyyy-MM-dd" final val DAY_DATE_FORMAT_TWO = "yyyyMMdd" final val MONTH_DATE_FORMAT = "yyyy-MM" /** * 时间字符串to时间戳 * @param dateStr 时间字符串 * @param pattern 时间格式 * @return */ def convertDateStr2TimeStamp(dateStr: String, pattern: String): Long = { new SimpleDateFormat(pattern).parse(dateStr).getTime } /** *时间戳to字符串 * @param timestamp 时间戳 * @param pattern 字符串 * @return */ def convertTimeStamp2DateStr(timestamp: Long, pattern: String): String = { new DateTime(timestamp).toString(pattern) } /** * 时间字符串+天数to时间戳 * @param dateStr 时间字符串 * @param pattern 时间格式 * @param days 增加天数 * @return */ def dateStrAddDays2TimeStamp(dateStr: String, pattern: String, days: Int): Long = { convertDateStr2Date(dateStr, pattern).plusDays(days).toDate.getTime } /** * 时间字符串+年数to时间戳 * @param dateStr 时间字符串 * @param pattern 时间格式 * @param years 增加年数 * @return */ def dateStrAddYears2TimeStamp(dateStr: String, pattern: String, years: Int): Long = { convertDateStr2Date(dateStr, pattern).plusYears(years).toDate.getTime } def dateStrAddYears2Str(dateStr: String, pattern: String, years: Int): String = { val t = convertDateStr2Date(dateStr, pattern).plusYears(years).toDate.getTime convertTimeStamp2DateStr(t, pattern) } /** * 时间字符串to日期 * @param dateStr 时间字符串 * @param pattern 时间格式 * @return */ def convertDateStr2Date(dateStr: String, pattern: String): DateTime = { new DateTime(new SimpleDateFormat(pattern).parse(dateStr)) } /** * 月差 * @param stDate 开始时间 * @param endDate 结束时间 * @return */ def getMonthSpace(stDate: String, endDate: String): Int = { val c1 = Calendar.getInstance() val c2 = Calendar.getInstance() c1.setTime(new SimpleDateFormat(MONTH_DATE_FORMAT).parse(stDate)) c2.setTime(new SimpleDateFormat(MONTH_DATE_FORMAT).parse(endDate)) val month1 = c2.get(Calendar.MONTH) - c1.get(Calendar.MONTH) val month2 = (c2.get(Calendar.YEAR) - c1.get(Calendar.YEAR)) * 12 Math.abs(month1 + month2) } /** * 日差 * @param stDate 开始时间 * @param endDate 结束时间 * @return */ def getDaySpace(stDate: String, endDate: String): Long = { val c1 = Calendar.getInstance() val c2 = Calendar.getInstance() c1.setTime(new SimpleDateFormat(SECOND_TIME_FORMAT).parse(stDate)) c2.setTime(new SimpleDateFormat(SECOND_TIME_FORMAT).parse(endDate)) val difference=c2.getTimeInMillis -c1.getTimeInMillis val days = difference/ONE_DAY_Millis Math.abs(days) } def getLastYearDateStr(dateStr: String): String = { val e = timeUtil.dateStrAddYears2TimeStamp(dateStr, "yyyy", -1) val f = timeUtil.convertTimeStamp2DateStr(e, "yyyy") f+"-12-31 23:59:59.999" } def getCurDateStr: String = { convertTimeStamp2DateStr(System.currentTimeMillis(), timeUtil.SECOND_TIME_FORMAT) } def lastYearEnd: String = { timeUtil.getLastYearDateStr(getCurDateStr) } def getCurMonthDays: Int = { val a = Calendar.getInstance() a.set(Calendar.DATE, 1) a.roll(Calendar.DATE, -1) val days = a.get(Calendar.DATE) days } }
pom.xml ：
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <groupId>com.fc</groupId> <artifactId>scalapp</artifactId> <version>1.0-SNAPSHOT</version> <dependencies> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-core_2.10</artifactId> <version>1.6.0-cdh5.14.2</version> </dependency> <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_2.10</artifactId> <version>1.6.0-cdh5.14.2</version> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-spark</artifactId> <version>1.2.0-cdh5.14.2</version> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-spark</artifactId> <version>4.14.0-cdh5.14.2</version> </dependency> <dependency> <groupId>com.lmax</groupId> <artifactId>disruptor</artifactId> <version>3.3.8</version> </dependency> <dependency> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-core</artifactId> <version>4.14.0-cdh5.14.2</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> <version>2.6.0-cdh5.14.2</version> </dependency> <!--MySQL驱动--> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.43</version> </dependency> </dependencies> <repositories> <repository> <id>mvnrepository</id> <name>mvnrepository</name> <url>http://mvnrepository.com/</url> <layout>default</layout> <releases> <enabled>true</enabled> </releases> <snapshots> <enabled>false</enabled> </snapshots> </repository> <repository> <id>cloudera</id> <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url> </repository> <repository> <id>hortonworks</id> <name>hortonworks</name> <url>http://repo.hortonworks.com/content/repositories/releases/</url> </repository> </repositories> <pluginRepositories> <pluginRepository> <id>hortonworks</id> <name>hortonworks</name> <url>http://repo.hortonworks.com/content/repositories/releases/</url> </pluginRepository> </pluginRepositories> <build> <sourceDirectory>src/main/scala</sourceDirectory> <plugins> <plugin> <groupId>org.scala-tools</groupId> <artifactId>maven-scala-plugin</artifactId> <version>2.15.2</version> <executions> <execution> <goals> <goal>compile</goal> <goal>testCompile</goal> </goals> </execution> </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> <version>3.1.0</version> <executions> <execution> <phase>package</phase> <goals> <goal>shade</goal> </goals> <configuration> <createDependencyReducedPom>false</createDependencyReducedPom> <filters> <filter> <artifact>*:*</artifact> <excludes> <exclude>META-INF/*.SF</exclude> <exclude>META-INF/*.DSA</exclude> <exclude>META-INF/*.RSA</exclude> </excludes> </filter> </filters> </configuration> </execution> </executions> </plugin> </plugins> </build> </project>
参考：
https://www.jianshu.com/p/f336f7e5f31b
https://blog.csdn.net/adorechen/article/details/78746363
以上是关于Spark教程——（11）Spark程序本地执行和集群执行的差异的主要内容，如果未能解决你的问题，请参考以下文章