spark简单文件配置

Posted 2020-11-02 努力中国

cd /usr/local/spark/spark-2.2.1-bin-hadoop2.7/conf
cp slaves.template slaves
cp spark-env.sh.template spark-env.sh  

修改slaves和spark-env.sh文件
slaves

hadoop-3
hadoop-4
hadoop-5

spark-env.sh

export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
export SCALA_HOME=/usr/share/scala  
export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.3
export SPARK_MASTER_IP=192.168.1.116
export SPARK_MASTER_PORT=7077  
export SPARK_MASTER_WEBUI_PORT=7070  
export SPARK_WORKER_CORES=2  
export SPARK_WORKER_MEMORY=1024m  
export SPARK_WORKER_INSTANCES=2  
export SPARK_CLASSPATH=$HBASE_HOME/lib/hbase-protocol-1.2.4.jar:$HBASE_HOME/lib/hbase-common-1.2.4.jar:$HBASE_HOME/lib/htrace-core-3.1.0-incubating.jar:$HBAS
E_HOME/lib/hbase-server-1.2.4.jar:$HBASE_HOME/lib/hbase-client-1.2.4.jar:$HBASE_HOME/lib/metrics-core-2.2.0.jar:$SPARK_CLASSPATH  
export SPARK_LOCAL_DIR="/mnt/spark/tmp"  
export SPARK_JAVA_OPTS="-Dspark.storage.blockManagerHeartBeatMs=60000-Dspark.local.dir=$SPARK_LOCAL_DIR -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:$S
PARK_HOME/logs/gc.log -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=60"