结合ZooKeeper配置Hadoop HA

Posted 2020-10-02 代号菜鸟

参考官方文档 http://hadoop.apache.org/docs/r2.5.2/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html

架构图

 

 

---

hdfs-site.xml配置

<configuration>
        <property>
        <name>dfs.nameservices</name>
        <value>ns1</value>
    </property>
    <property>
    <name>dfs.ha.namenodes.ns1</name>
        <value>nn1,nn2</value>
    </property>
    <!-- NameNode RPC ADDRESS -->
    <property>
        <name>dfs.namenode.rpc-address.ns1.nn1</name>
        <value>hadoop-senior1.jason.com:8020</value>
    </property>
    <property>
        <name>dfs.namenode.rpc-address.ns1.nn2</name>
        <value>hadoop-senior2.jason.com:8020</value>
    </property>
    <!-- NameNode HTTP WEB ADDRESS -->
    <property>
        <name>dfs.namenode.http-address.ns1.nn1</name>
        <value>hadoop-senior1.jason.com:50070</value>
    </property>
    <property>
        <name>dfs.namenode.http-address.ns1.nn2</name>
        <value>hadoop-senior2.jason.com:50070</value>
    </property>
    <!-- NameNode Shared Edits Address -->
    <property>
        <name>dfs.namenode.shared.edits.dir</name>
        <value>qjournal://hadoop-senior1.jason.com:8485;hadoop-senior2.jason.com:8485;hadoop-senior3.jason.com:8485/ns1</value>
    </property>
    <!-- HDFS PROXY CLIENT -->
    <property>
        <name>dfs.client.failover.proxy.provider.ns1</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    </property>
    <!-- Fencing Method -->
    <property>
        <name>dfs.ha.fencing.methods</name>
        <value>sshfence</value>
    </property>
    <property>
        <name>dfs.ha.fencing.ssh.private-key-files</name>
        <value>/home/jason/.ssh/id_rsa</value>
    </property>
    
    <!-- DIR Location -->
    <property>
        <name>dfs.journalnode.edits.dir</name>
        <value>/opt/app/hadoop-2.5.0/data/dfs/jn</value>
    </property>
    
     <property>
       <name>dfs.ha.automatic-failover.enabled</name>
       <value>true</value>
     </property>
    
</configuration>

core-site.xml配置

<configuration>    
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://ns1</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/app/hadoop-2.5.0/data/tmp</value>
    </property>
    <property>
        <name>fs.trash.interval</name>
        <value>420</value>
    </property>
    <property>
       <name>ha.zookeeper.quorum</name>
       <value>hadoop-senior1.jason.com:2181,hadoop-senior2.jason.com:2181,hadoop-senior3.jason.com:2181</value>
    </property>
</configuration>