yarn-site.xml的配置内容

Posted 2020-10-19 vv.past

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->

<property>
  <name>yarn.resourcemanager.ha.enabled</name>
  <value>true</value>
</property>

<property>
  <name>yarn.resourcemanager.cluster-id</name>
  <value>zz</value>
</property>

<property>
  <name>yarn.resourcemanager.ha.rm-ids</name>
  <value>rm1,rm2</value>
</property>

<property>
  <name>yarn.resourcemanager.hostname.rm1</name>
  <value>host1</value>
</property>

<property>
  <name>yarn.resourcemanager.hostname.rm2</name>
  <value>host2</value>
</property>

<property>
  <name>yarn.resourcemanager.webapp.address.rm1</name>
  <value>host1:8088</value>
</property>

<property>
  <name>yarn.resourcemanager.webapp.address.rm2</name>
  <value>host2:8088</value>
</property>

<!--
   <property>
                <name>yarn.resourcemanager.address.rm1</name>
                <value>host1:8132</value>
        </property>
        <property>
                <name>yarn.resourcemanager.address.rm2</name>
                <value>host2:8132</value>
        </property>
        <property>
                <name>yarn.resourcemanager.scheduler.address.rm1</name>
                <value>host1:8130</value>
        </property>
        <property>
                <name>yarn.resourcemanager.scheduler.address.rm2</name>
                <value>host2:8130</value>
        </property>
        <property>
                <name>yarn.resourcemanager.resource-tracker.address.rm1</name>
                <value>host1:8131</value>
        </property>
        <property>
               <name>yarn.resourcemanager.resource-tracker.address.rm2</name>
                <value>host2:8131</value>
        </property>
        <property>
                <name>yarn.resourcemanager.webapp.address.rm1</name>
                <value>host1:8088</value>
        </property>
        <property>
                <name>yarn.resourcemanager.webapp.address.rm2</name>
                <value>host2:8088</value>
        </property>
-->

<property>
  <name>ha.zookeeper.quorum</name>
  <value>host1:2181,host2:2181,host3:2181</value> 
</property>

<property> 
  <name>yarn.resourcemanager.zk-state-store.address</name> 
  <value>host1:2181,host2:2181,host3:2181</value>
</property> 

<property> 
  <name>yarn.resourcemanager.store.class</name> 
  <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> 
</property> 


<property>
  <name>yarn.resourcemanager.zk-address</name>
  <value>host1:2181,host2:2181,host3:2181</value>
</property>

<property>
  <name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
  <value>true</value>
</property>

<property> 
   <name>yarn.resourcemanager.connect.retry-interval.ms</name> 
   <value>2000</value>
</property>

<property>
  <name>yarn.resourcemanager.recovery.enabled</name> 
  <value>true</value> 
</property>

<property> 
  <name>yarn.app.mapreduce.am.scheduler.connection.wait.interval-ms</name> 
  <value>5000</value> 
</property> 

<property> 
  <name>yarn.client.failover-proxy-provider</name> 
  <value>org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider</value> 
</property> 

<property>
    <name>yarn.resourcemanager.ha.automatic-failover.zk-base-path</name>
    <value>/yarn-leader-election</value>
</property>


 <!-- 资源调度模型  -->
<property> 
  <name>yarn.nodemanager.aux-services</name> 
  <value>mapreduce_shuffle,spark_shuffle</value> 
</property> 

<property> 
  <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name> 
  <value>org.apache.hadoop.mapred.ShuffleHandler</value> 
</property> 

 <!-- spark on yarn 动态资源分配调度模型类  -->
<property> 
  <name>yarn.nodemanager.aux-services.spark_shuffle.class</name> 
  <value>org.apache.spark.network.yarn.YarnShuffleService</value> 
</property> 

 <!-- spark on yarn 动态资源分配服务接口  -->
<property> 
  <name>spark.shuffle.service.port</name> 
  <value>7337</value> 
</property> 

<property> 
  <name>yarn.nodemanager.local-dirs</name> 
  <value>/home/cloudz/install/hadoop-2.9.0/tmp</value> 
</property> 

<property> 
  <name>yarn.nodemanager.log-dirs</name> 
  <value>/home/cloudz/install/hadoop-2.9.0/logs</value> 
</property>  


 <!-- 一个container可申请的最小内存  -->
<property>
    <name>yarn.scheduler.minimum-allocation-mb</name>
    <value>512</value>
</property>

 <!-- 一个container可以申请的最大内存  -->
<property>
    <name>yarn.scheduler.maximum-allocation-mb</name>
    <value>4096</value>
</property>

 <!-- nodemanager可使用内存资源大小  -->
<property>
    <name>yarn.nodemanager.resource.memory-mb</name>
    <value>16384</value>
</property>

 <!--  虚拟内存倍数  -->
<property>
    <name>yarn.nodemanager.vmem-pmem-ratio</name>
    <value>4.2</value>
    <description>允许的虚拟内存倍数</description>
</property>

 <!-- nodemanager可使用的最大cpu个数
      是个虚数并不一定需要跟物理cpu个数匹配可以更多  -->
<property>
    <name>yarn.nodemanager.resource.cpu-vcores</name>
    <value>8</value>
</property>

 <!-- 开启mapreduce中间过程压缩  -->
<property>
  <name>mapreduce.map.output.compress</name>  
  <value>true</value>
</property>

<property>
  <name>mapreduce.map.output.compress.codec</name>  
  <value>org.apache.hadoop.io.compress.SnappyCodec</value>
</property>

<property>
    <name>mapreduce.output.fileoutputformat.compress</name>
    <value>true</value>
 </property>

  <property>
    <name>mapreduce.output.fileoutputformat.compress.codec</name>
    <value>org.apache.hadoop.io.compress.SnappyCodec</value>
    <description>数据的压缩类型,这里使用Snappy压缩</description>
  </property>

  <property>
    <name>mapreduce.output.fileoutputformat.compress.type</name>
    <value>BLOCK</value>
    <description>数据的压缩级别,这里设置按数据块压缩</description>
  </property>

  <property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
    <description>开启application 日志聚合功能</description>
  </property>

  <property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>259200</value>
    <description>设置聚合日志保存时间3天</description>
  </property>

  <property>
    <name>yarn.log-aggregation.retain-check-interval-seconds</name>
    <value>86400</value>
    <description>清理过期聚合日志程序的执行间隔时间</description>
  </property>

  <property>
    <name>yarn.nodemanager.remote-app-log-dir</name>
    <value>/tmp/logs</value>
    <description>聚合日志在hdfs上的目录</description>
  </property>

  <!--
  <property>
    <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
    <value></value>
    <description>聚合日志在hdfs上的目录分层方式</description>
  </property>
  -->

  <property>
    <name>yarn.log.server.url</name>
    <value>http://host1.bjcq.zybang.com:19888/jobhistory/logs</value>
    <description>历史日志对应路径</description>
  </property>

  <!-- 开启容量调度模式 -->
  <property>
    <name>yarn.resourcemanager.scheduler.class</name>
    <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
    <description>
    配置yarn启用容量调度模式(默认即是容量调度),配置文件为capacity-scheduler.xml
    </description>
  </property>



</configuration>