Spark安装

Posted 2020-12-02 gispathfinder

SPARK_MASTER_HOST=sparkmaster.smartmap.com
SPARK_MASTER_PORT=7077
SPARK_WORKER_CORES=1
SPARK_WORKER_MEMORY=1024M
SPARK_MASTER_WEBUI_PORT=8080
SPARK_CONF_DIR=/app/spark/spark-2.3.0-bin-hadoop2.7/conf
 
JAVA_HOME=/app/soft/jdk1.8.0_171
JAVA_HOME
 
 
 
vi /etc/network/interfaces
 
 
iface ens33 inet static
address 192.168.1.42
netmask 255.255.255.0
gateway 192.168.1.1

sudo /etc/init.d/networking restart

vi /etc/resov.conf
sudo vi /etc/resolvconf/resolv.conf.d/base

sudo ufw disable

sudo service apparmor stop
 
 
192.168.1.30 master
192.168.1.31 slave1
192.168.1.32 slave2
192.168.1.33 slave3

192.168.9.127 head0.parrot
192.168.9.128 data0.parrot

scp [email protected]:/root/.ssh/id_rsa.pub authorized_keys_slave1
scp [email protected]:/root/.ssh/id_rsa.pub authorized_keys_slave2
scp [email protected]:/root/.ssh/id_rsa.pub authorized_keys_slave3

scp [email protected]:/root/.ssh/authorized_keys /root/.ssh/authorized_keys

scp -R [email protected]:/usr/lib/jvm/jdk1.8.0_171 /usr/lib/jvm/

export JAVA_HOME=/usr/lib/jvm/jdk1.8.0
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib
export PATH=${JAVA_HOMR}/bin:$PATH

ntpdate -u 2.asia.pool.ntp.org