EMR的fair-scheduler.xml

注意点进emr的yarn里时,配置要选fair-scheduler.修改完点保存,再刷新.

cat /etc/ecm/hadoop-conf/fair-scheduler.xml

<?xml version="1.0"?>

<allocations>

<aclSubmitApps>*</aclSubmitApps>

<weight>2</weight>

<minResources>10000 mb, 10vcores</minResources>

<maxChildResources>34000 mb,24 vcores</maxChildResources>

<maxRunningApps>50</maxRunningApps>

<maxAMShare>1</maxAMShare>

<maxResources>400000 mb, 200vcores</maxResources> #限制队列最大使用资源

<aclAdministerApps>*</aclAdministerApps>

<schedulingPolicy>fair</schedulingPolicy>

<queue name="default">

<aclSubmitApps>*</aclSubmitApps>

<minResources>10000 mb, 10vcores</minResources>

<aclAdministerApps>*</aclAdministerApps>

<weight>1</weight>

<maxRunningApps>10</maxRunningApps>

<maxAMShare>0.5</maxAMShare>

<maxResources>200000 mb, 100vcores</maxResources>

</queue>

<queue name="collects">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>8</weight>

<maxAMShare>0.8</maxAMShare>

<minResources>50 mb, 2vcores</minResources>

<maxResources>400000 mb, 200vcores</maxResources>

<maxRunningApps>50</maxRunningApps>

</queue>

<queue name="data_bi">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>4</weight>

<minResources>100 mb, 1vcores</minResources>

<maxResources>30000 mb, 50vcores</maxResources>

<maxRunningApps>5</maxRunningApps>

</queue>

<queue name="opay_collects">

<weight>20</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>400000 mb, 200vcores</maxResources>

<maxRunningApps>20</maxRunningApps>

<maxAMShare>0.5</maxAMShare>

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

</queue>

<queue name="opos_collects">

<weight>5</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>80000 mb, 50vcores</maxResources>

<maxRunningApps>10</maxRunningApps>

</queue>

<queue name="users" type="parent">

<weight>5</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>10000 mb, 150vcores</maxResources>

<maxRunningApps>30</maxRunningApps>

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

</queue>

<queue name="airflow">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>8</weight>

<minResources>10 mb, 2vcores</minResources>

<maxResources>200000 mb, 150vcores</maxResources>

<maxRunningApps>30</maxRunningApps>

</queue>

<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>

<userMaxAppsDefault>50</userMaxAppsDefault>

<queueMaxAppsDefault>50</queueMaxAppsDefault>

<queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>

<defaultFairSharePreemptionThreshold>0.5</defaultFairSharePreemptionThreshold>

<queueMaxResourcesDefault>34000 mb,24vcores</queueMaxResourcesDefault>

<defaultFairSharePreemptionTimeout>9223372036854775807</defaultFairSharePreemptionTimeout>

<defaultMinSharePreemptionTimeout>9223372036854775807</defaultMinSharePreemptionTimeout>

</allocations>

#新的xml, 不带root限制: 放emr的yarn-配置-fair-scheduler

<?xml version="1.0" encoding="utf-8"?>

<allocations>
<queue name="root">
<queue name="default">
<aclSubmitApps>*</aclSubmitApps>
<minResources>10000 mb, 10vcores</minResources>
<aclAdministerApps>*</aclAdministerApps>
<weight>1</weight>
<maxRunningApps>10</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<maxResources>200000 mb, 80vcores</maxResources>
</queue>
<queue name="collects">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>8</weight>
<maxAMShare>0.5</maxAMShare>
<minResources>10000 mb, 10vcores</minResources>
<maxResources>190000 mb, 60vcores</maxResources>
<maxRunningApps>50</maxRunningApps>
</queue>
<queue name="algo_spark">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>6</weight>
<maxAMShare>0.8</maxAMShare>
<minResources>10000 mb, 10vcores</minResources>
<maxResources>200000 mb, 50vcores</maxResources>
<maxRunningApps>30</maxRunningApps>
</queue>
<queue name="data_bi">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>6</weight>
<minResources>10000 mb, 10vcores</minResources>
<maxResources>200000 mb, 80vcores</maxResources>
<maxRunningApps>20</maxRunningApps>
</queue>
<queue name="opay_collects">
<weight>8</weight>
<minResources>10000 mb, 10vcores</minResources>
<maxResources>130000 mb, 45vcores</maxResources>
<maxRunningApps>20</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
</queue>
<queue name="opos_collects">
<weight>8</weight>
<minResources>10000 mb, 10vcores</minResources>
<maxResources>130000 mb, 45vcores</maxResources>
<maxRunningApps>20</maxRunningApps>
</queue>
<queue name="airflow">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>8</weight>
<minResources>100000 mb, 50vcores</minResources>
<maxResources>500000 mb, 150vcores</maxResources>
<maxRunningApps>50</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<schedulingPolicy>fair</schedulingPolicy>
</queue>
</queue>
<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>
<userMaxAppsDefault>50</userMaxAppsDefault>
<queueMaxAppsDefault>50</queueMaxAppsDefault>
<queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>
<defaultFairSharePreemptionThreshold>0.5</defaultFairSharePreemptionThreshold>
<defaultFairSharePreemptionTimeout>9223372036854775807</defaultFairSharePreemptionTimeout>
<defaultMinSharePreemptionTimeout>9223372036854775807</defaultMinSharePreemptionTimeout>

</allocations>