cdh部署及接入kdc认证

Posted 2022-01-21 sellsa

部署环境

主机名	IP	配置	操作系统	角色	CDH版本
c1.heboan.com	10.1.1.120	2核/4G	CentOS Linux release 7.2.1511	cm-server、cm-agent、mariadb	CDH-5.15.0
c2.heboan.com	10.1.1.121	2核/6G	CentOS Linux release 7.2.1511	cm-agent
c3.heboan.com	10.1.1.122	2核/6G	CentOS Linux release 7.2.1511	cm-agent
c4.heboan.com	10.1.1.123	2核/6G	CentOS Linux release 7.2.1511	cm-agent

准备的软件包，cloudera官网可以直接下载

• https://archive.cloudera.com/cdh5
• https://archive.cloudera.com/cm5

10.1.1.120   c1.heboan.com
10.1.1.121   c2.heboan.com
10.1.1.122   c3.heboan.com
10.1.1.123   c4.heboan.com

所有节点配置hosts解析

ssh-keygen -t rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub c1.heboan.com
ssh-copy-id -i ~/.ssh/id_rsa.pub c2.heboan.com
ssh-copy-id -i ~/.ssh/id_rsa.pub c3.heboan.com
ssh-copy-id -i ~/.ssh/id_rsa.pub c4.heboan.com

所有节点配置ssh证书互信

yum install -y epel-release
yum install -y chrony

c1
# grep -Ev "#|^$" /etc/chrony.conf 
server cn.pool.ntp.org iburst
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
allow 10.1.1.0/24
logdir /var/log/chrony

c2-c4
# grep -Ev "#|^$" /etc/chrony.conf 
server 10.1.1.120 iburst
driftfile /var/lib/chrony/drift
makestep 1.0 3
rtcsync
logdir /var/log/chrony

systemctl start chronyd
systemctl enable chronyd

NTP时钟同步

#vim /etc/selinux/config
SELINUX=disabled

所有节点禁用SELINUX

echo vm.swappiness = 10 >> /etc/sysctl.conf
sysctl -p

所有节点设置swap

vi /etc/security/limits.conf
* soft nofile 65535
* hard nofile 65535
* soft nproc 32000
* hard nproc 32000

所有节点设置文件打开最大数

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

所有节点关闭透明大页面压缩

yum install oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm

vim /etc/profile.d/java.sh
    export JAVA_HOME=/usr/java/jdk1.7.0_67-cloudera
    export JAVA_BIN=/usr/java/jdk1.7.0_67-cloudera/bin
    export PATH=$PATH:$JAVA_HOME/bin
    export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
    export JAVA_HOME JAVA_BIN PATH CLASSPATH
source /etc/profile.d/java.sh

mkdir -p /usr/share/java
mv mysql-connector-java-5.1.22.jar /usr/share/java/mysql-connector-java.jar 
chmod +x /usr/share/java/mysql-connector-java.jar

所有节点配置jdk和mysql驱动

 

c1节点配置Mariadb

yum install -y mariadb mariadb-server mariadb-devel

vim /etc/my.cnf.d/server.cnf
    ...
    [mysqld]
    lower_case_table_names=1
    character-set-server = utf8
    collation-server = utf8_general_ci

systemctl start mariadb
systemctl enable mariadb

mysql_secure_installation 

create database cm;
create database am;
create database hive;
create database hue;
create database oozie;

grant all privileges on *.* to ‘cdh‘@‘%‘ identified by ‘123456‘;

配置数据库

 

配置yum仓库源

yum install -y httpd createrepo
systemctl start httpd
mkdir /var/www/html/cm
cp cloudera-manager-* /var/www/html/cm/
cd /var/www/html/cm/
createrepo .

c1节点配置yum仓库

vim /etc/yum.repos.d/cm.repo
    [cmrepo]
    name = cm_repo
    baseurl = http://c1.heboan.com/cm
    enable = true
    gpgcheck = false

yum repolist

所有节点加入仓库

 

c1节点安装Cloudera-manager-server

通过yum安装

yum -y install cloudera-manager-server

初始化cm数据库

/usr/share/cmf/schema/scm_prepare_database.sh mysql cm cdh ‘123456‘

# 数据库类型   数据库名  账号  密码

拷贝cdh包到/opt/cloudera/parcel-repo/

cp CDH-5.15.0-1.cdh5.15.0.p0.21-el7.parcel /opt/cloudera/parcel-repo/

计算cdh包的sha1sum值并写入文件CDH-5.15.0-1.cdh5.15.0.p0.21-el7.parcel.sha

设置属主

chown cloudera-scm.cloudera-scm /opt/cloudera/parcel-repo/*

重启Cloudera Manager Server

systemctl start cloudera-scm-server

备注：

• 机器配置低的情况下，该服务启动时间较长
• 启动过程可查看日志：/var/log/cloudera-scm-server/cloudera-scm-server.log
• 该服务会启动两个端口: 7180（提供web界面给我们操作，7182用于和agent通信）

 

所有节点安装Cloudera-manager-agent

通过yum安装

yum -y install cloudera-manager-agent

修改agent配置文件

# vim /etc/cloudera-scm-agent/config.ini
...
server_host=c1.heboan.com   #指向c1.heboan.com

启动

systemctl start cloudera-scm-agent

 

浏览器访问c1节点的7180端口进行web界面操作

账号密码： admin/admin

接受条款协议

选择免费版

选择集群主机

打开更多选项

去掉远程存储库地址

选择CDH版本继续