EMR的fair-scheduler.xml

Posted 2021-03-22 hongfeng2019

cat /etc/ecm/hadoop-conf/fair-scheduler.xml

 

<?xml version="1.0"?>

<allocations>

<aclSubmitApps>*</aclSubmitApps>

<weight>2</weight>

<minResources>10000 mb, 10vcores</minResources>

<maxChildResources>34000 mb,24 vcores</maxChildResources>

<maxRunningApps>50</maxRunningApps>

<maxAMShare>1</maxAMShare>

<maxResources>400000 mb, 200vcores</maxResources> #限制队列最大使用资源

<aclAdministerApps>*</aclAdministerApps>

<schedulingPolicy>fair</schedulingPolicy>

<queue name="default">

<aclSubmitApps>*</aclSubmitApps>

<minResources>10000 mb, 10vcores</minResources>

<aclAdministerApps>*</aclAdministerApps>

<weight>1</weight>

<maxRunningApps>10</maxRunningApps>

<maxAMShare>0.5</maxAMShare>

<maxResources>200000 mb, 100vcores</maxResources>

</queue>

<queue name="collects">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>8</weight>

<maxAMShare>0.8</maxAMShare>

<minResources>50 mb, 2vcores</minResources>

<maxResources>400000 mb, 200vcores</maxResources>

<maxRunningApps>50</maxRunningApps>

</queue>

<queue name="data_bi">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>4</weight>

<minResources>100 mb, 1vcores</minResources>

<maxResources>30000 mb, 50vcores</maxResources>

<maxRunningApps>5</maxRunningApps>

</queue>

<queue name="opay_collects">

<weight>20</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>400000 mb, 200vcores</maxResources>

<maxRunningApps>20</maxRunningApps>

<maxAMShare>0.5</maxAMShare>

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

</queue>

<queue name="opos_collects">

<weight>5</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>80000 mb, 50vcores</maxResources>

<maxRunningApps>10</maxRunningApps>

</queue>

<queue name="users" type="parent">

<weight>5</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>10000 mb, 150vcores</maxResources>

<maxRunningApps>30</maxRunningApps>

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

</queue>

<queue name="airflow">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>8</weight>

<minResources>10 mb, 2vcores</minResources>

<maxResources>200000 mb, 150vcores</maxResources>

<maxRunningApps>30</maxRunningApps>

</queue>

<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>

<userMaxAppsDefault>50</userMaxAppsDefault>

<queueMaxAppsDefault>50</queueMaxAppsDefault>

<queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>

<defaultFairSharePreemptionThreshold>0.5</defaultFairSharePreemptionThreshold>

<queueMaxResourcesDefault>34000 mb,24vcores</queueMaxResourcesDefault>

<defaultFairSharePreemptionTimeout>9223372036854775807</defaultFairSharePreemptionTimeout>

<defaultMinSharePreemptionTimeout>9223372036854775807</defaultMinSharePreemptionTimeout>

</allocations>