totalCounters = currentJobData.getAllCountersInGroup(COUNTER_GROUP_NAME);
+
+ long dataLocalMaps = totalCounters.containsKey(DATA_LOCAL_MAPS) ? totalCounters.get(DATA_LOCAL_MAPS) : 0;
+ long rackLocalMaps = totalCounters.containsKey(RACK_LOCAL_MAPS) ? totalCounters.get(RACK_LOCAL_MAPS) : 0;
+ long otherLocalMaps = totalCounters.containsKey(OTHER_LOCAL_MAPS) ? totalCounters.get(OTHER_LOCAL_MAPS) : 0;
+ long totalLaunchedMaps = totalCounters.get(TOTAL_LAUNCHED_MAPS);
+
+ long nodeLocalityRatio = (dataLocalMaps + otherLocalMaps) / totalLaunchedMaps;
+ long rackLocalityRatio = rackLocalMaps/totalLaunchedMaps;
+
+
+ Severity severity = getTaskLocalitySeverity(nodeLocalityRatio);
+
+ severity = Severity.min(severity, getTaskLocalitySeverity(rackLocalityRatio));
+
+ HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(),
+ _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, data.getMapperData().length));
+
+ result.addResultDetail("Data Local Map Tasks", Long.toString(dataLocalMaps));
+ result.addResultDetail("Other Local Map Tasks", Long.toString(otherLocalMaps));
+ result.addResultDetail("Rack Local Map Tasks", Long.toString(rackLocalMaps));
+ result.addResultDetail("Total Launched Map Tasks", Long.toString(totalLaunchedMaps));
+
+ return result;
+ }
+
+ private Severity getTaskLocalitySeverity(double ratio) {
+ return Severity.getSeverityDescending(
+ ratio, nodeLocalityRatioLimits[0], nodeLocalityRatioLimits[1], nodeLocalityRatioLimits[2], nodeLocalityRatioLimits[3]);
+ }
+
+
+
+
+}
diff --git a/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java b/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java
index c04edd7fd..deec8b33a 100644
--- a/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java
+++ b/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java
@@ -15,6 +15,8 @@
*/
package com.linkedin.drelephant.tez;
+
+import com.google.common.base.Strings;
import com.linkedin.drelephant.analysis.*;
import com.linkedin.drelephant.configurations.aggregator.AggregatorConfigurationData;
import com.linkedin.drelephant.tez.data.TezApplicationData;
@@ -29,7 +31,9 @@ public class TezMetricsAggregator implements HadoopMetricsAggregator {
private static final Logger logger = Logger.getLogger(TezMetricsAggregator.class);
- private static final String TEZ_CONTAINER_CONFIG = "hive.tez.container.size";
+
+ private static final String TEZ_CONTAINER_CONFIG = "tez.task.resource.memory.mb";
+ private static final String HIVE_TEZ_CONTAINER_CONFIG = "hive.tez.container.size";
private static final String MAP_CONTAINER_CONFIG = "mapreduce.map.memory.mb";
private static final String REDUCER_CONTAINER_CONFIG = "mapreduce.reduce.memory.mb";
private static final String REDUCER_SLOW_START_CONFIG = "mapreduce.job.reduce.slowstart.completedmaps";
@@ -84,26 +88,37 @@ public HadoopAggregatedData getResult() {
}
private long getMapContainerSize(HadoopApplicationData data) {
- try {
- long mapContainerSize = Long.parseLong(data.getConf().getProperty(TEZ_CONTAINER_CONFIG));
- if (mapContainerSize > 0)
- return mapContainerSize;
- else
- return Long.parseLong(data.getConf().getProperty(MAP_CONTAINER_CONFIG));
- } catch ( NumberFormatException ex) {
- return CONTAINER_MEMORY_DEFAULT_BYTES;
+ long mapperContainerSize;
+ if(!Strings.isNullOrEmpty(data.getConf().getProperty(TEZ_CONTAINER_CONFIG)) && Long.valueOf(data.getConf().getProperty(TEZ_CONTAINER_CONFIG)) > 0){
+ mapperContainerSize = Long.valueOf(data.getConf().getProperty(TEZ_CONTAINER_CONFIG));
+ }
+ else if(!Strings.isNullOrEmpty(data.getConf().getProperty(HIVE_TEZ_CONTAINER_CONFIG)) && Long.valueOf(data.getConf().getProperty(HIVE_TEZ_CONTAINER_CONFIG)) > 0){
+ mapperContainerSize = Long.valueOf(data.getConf().getProperty(HIVE_TEZ_CONTAINER_CONFIG));
+ }
+ else if(!Strings.isNullOrEmpty(data.getConf().getProperty(MAP_CONTAINER_CONFIG)) && Long.valueOf(data.getConf().getProperty(MAP_CONTAINER_CONFIG)) > 0){
+ mapperContainerSize = Long.valueOf(data.getConf().getProperty(MAP_CONTAINER_CONFIG));
+ }
+ else {
+ mapperContainerSize = CONTAINER_MEMORY_DEFAULT_BYTES;
}
+ return mapperContainerSize;
}
private long getReducerContainerSize(HadoopApplicationData data) {
- try {
- long reducerContainerSize = Long.parseLong(data.getConf().getProperty(TEZ_CONTAINER_CONFIG));
- if (reducerContainerSize > 0)
- return reducerContainerSize;
- else
- return Long.parseLong(data.getConf().getProperty(REDUCER_CONTAINER_CONFIG));
- } catch ( NumberFormatException ex) {
- return CONTAINER_MEMORY_DEFAULT_BYTES;
+ long reducerContainerSize;
+ if(!Strings.isNullOrEmpty(data.getConf().getProperty(TEZ_CONTAINER_CONFIG)) && Long.valueOf(data.getConf().getProperty(TEZ_CONTAINER_CONFIG)) > 0){
+ reducerContainerSize = Long.valueOf(data.getConf().getProperty(TEZ_CONTAINER_CONFIG));
}
+ else if(!Strings.isNullOrEmpty(data.getConf().getProperty(HIVE_TEZ_CONTAINER_CONFIG)) && Long.valueOf(data.getConf().getProperty(HIVE_TEZ_CONTAINER_CONFIG)) > 0){
+ reducerContainerSize = Long.valueOf(data.getConf().getProperty(HIVE_TEZ_CONTAINER_CONFIG));
+ }
+ else if(!Strings.isNullOrEmpty(data.getConf().getProperty(REDUCER_CONTAINER_CONFIG)) && Long.valueOf(data.getConf().getProperty(REDUCER_CONTAINER_CONFIG)) > 0){
+ reducerContainerSize = Long.valueOf(data.getConf().getProperty(REDUCER_CONTAINER_CONFIG));
+ }
+ else {
+ reducerContainerSize = CONTAINER_MEMORY_DEFAULT_BYTES;
+ }
+ return reducerContainerSize;
+
}
}
diff --git a/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java
index 7447d62c5..f4c3658fc 100644
--- a/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java
+++ b/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java
@@ -82,6 +82,7 @@ private void loadParameters() {
}
+
public GenericMemoryHeuristic(String mapredContainerMemConf, HeuristicConfigurationData heuristicConfData) {
this._mapredContainerMemConf = mapredContainerMemConf;
this._heuristicConfData = heuristicConfData;
@@ -135,7 +136,6 @@ public HeuristicResult apply(TezApplicationData data) {
String containerSizeStr;
-
if(!Strings.isNullOrEmpty(data.getConf().getProperty(TEZ_MAPPER_MEMORY_CONF)) && Long.valueOf(data.getConf().getProperty(TEZ_MAPPER_MEMORY_CONF)) > 0){
containerSizeStr = data.getConf().getProperty(TEZ_MAPPER_MEMORY_CONF);
}
@@ -144,6 +144,7 @@ else if(!Strings.isNullOrEmpty(data.getConf().getProperty(HIVE_MAPPER_MEMORY_CON
}
else if(!Strings.isNullOrEmpty(data.getConf().getProperty(_mapredContainerMemConf)) && Long.valueOf(data.getConf().getProperty(_mapredContainerMemConf)) > 0) {
containerSizeStr = data.getConf().getProperty(_mapredContainerMemConf);
+
}
else {
containerSizeStr = getContainerMemDefaultMBytes();
diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java
index 3a52fa92d..f487b8234 100644
--- a/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java
+++ b/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java
@@ -33,8 +33,10 @@ public class MapperMemoryHeuristic extends GenericMemoryHeuristic {
private static final Logger logger = Logger.getLogger(MapperMemoryHeuristic.class);
public static final String MAPRED_MAPPER_MEMORY_CONF = "mapreduce.map.memory.mb";
+
public MapperMemoryHeuristic(HeuristicConfigurationData __heuristicConfData) {
super(MAPRED_MAPPER_MEMORY_CONF, __heuristicConfData);
+
}
@Override
diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java
index be8820e86..27a03abba 100644
--- a/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java
+++ b/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java
@@ -34,6 +34,7 @@ public class ReducerMemoryHeuristic extends GenericMemoryHeuristic {
public ReducerMemoryHeuristic(HeuristicConfigurationData __heuristicConfData) {
super(MAPRED_REDUCER_MEMORY_CONF, __heuristicConfData);
+
}
@Override
diff --git a/app/com/linkedin/drelephant/util/InfoExtractor.java b/app/com/linkedin/drelephant/util/InfoExtractor.java
index 6ff0f4d1b..045350cd9 100644
--- a/app/com/linkedin/drelephant/util/InfoExtractor.java
+++ b/app/com/linkedin/drelephant/util/InfoExtractor.java
@@ -22,7 +22,6 @@
import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData;
import com.linkedin.drelephant.tez.data.TezApplicationData;
-import com.linkedin.drelephant.clients.WorkflowClient;
import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData;
import com.linkedin.drelephant.schedulers.Scheduler;
diff --git a/app/views/help/mapreduce/helpMapperTaskLocality.scala.html b/app/views/help/mapreduce/helpMapperTaskLocality.scala.html
new file mode 100644
index 000000000..d5c5e7e5a
--- /dev/null
+++ b/app/views/help/mapreduce/helpMapperTaskLocality.scala.html
@@ -0,0 +1,71 @@
+@*
+* Copyright 2016 LinkedIn Corp.
+*
+* Licensed under the Apache License, Version 2.0 (the "License"); you may not
+* use this file except in compliance with the License. You may obtain a copy of
+* the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+* License for the specific language governing permissions and limitations under
+* the License.
+*@
+
+
+ This analysis shows the data locality for your apps.
+ We check the ratio between your job's node local map tasks AND the total number of map tasks.
+ Also check the ratio between your job's rack local map tasks AND the total number of tasks
+ The setting in question here is container memory is "yarn.scheduler.capacity.node-locality-delay". This is the Number of missed scheduling opportunities after which the capacity-scheduler attempts to schedule
+ rack local containers.
+ For Cloud based storage systems: This number should be low (0 or -1) as there is no notion of data locality and there is no point even checking for it
+ For on-prem storage/hdfs with cloud or on-prem compute, this number should be equal to the number of nodes in the cluster to guarantee a scheduling attempt to each data local node.
+ If this heuristic is above MODERATE, it means you are making good use of hadoop's most important feature of data locality.
+
+
+Example
+
+
+
+ Mapper Task Locality
+
+ | Severity: Critical |
|---|
+
+
+ | Data Local Map Tasks |
+ 1 |
+
+
+ | Other Local Map Tasks |
+ 1 |
+
+
+ | Rack local Map Tasks |
+ 56 |
+
+
+ | Total Launched Map Tasks |
+ 127 |
+
+
+
+
+
Suggestions
+
+
+
+ The problem indicates your tasks are not running local to the data. This will cause large network data flow and could lead to bottlenecks for very large jobs.
+
+ You should try to increase yarn.scheduler.capacity.node-locality-delay to atleast the number of nodes on the cluster
+
+ See yarn.scheduler.fair.locality.threshold.node and yarn.scheduler.fair.locality.threshold.rack for fair scheduler
+ This will make sure that there is at least one scheduling attempt to each data local node
+
+
+ See Capacity Scheduler Node Locality Settings for further information.
+
+ See Fair Scheduler Node Locality Settings for further information.
\ No newline at end of file