From 55a3f2b5725727027a1978b07f605594a3ee23be Mon Sep 17 00:00:00 2001 From: skakker Date: Wed, 10 Jan 2018 11:31:11 +0530 Subject: [PATCH 01/15] Spark Executor Spill Heuristic - (Depends on Custom SHS - Requires totalMemoryBytesSpilled metric) (#310) --- app-conf/HeuristicConf.xml | 8 +- .../fetchers/statusapiv1/statusapiv1.scala | 6 +- .../ExecutorStorageSpillHeuristic.scala | 133 +++++++++++++++++ .../legacydata/LegacyDataConverters.scala | 1 + .../spark/legacydata/SparkExecutorData.java | 3 +- .../drelephant/util/MemoryFormatUtils.java | 3 + ...lpExecutorStorageSpillHeuristic.scala.html | 23 +++ .../spark/SparkMetricsAggregatorTest.scala | 1 + .../heuristics/ExecutorGcHeuristicTest.scala | 3 +- .../ExecutorStorageSpillHeuristicTest.scala | 136 ++++++++++++++++++ .../heuristics/ExecutorsHeuristicTest.scala | 1 + 11 files changed, 313 insertions(+), 5 deletions(-) create mode 100644 app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala create mode 100644 app/views/help/spark/helpExecutorStorageSpillHeuristic.scala.html create mode 100644 test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala diff --git a/app-conf/HeuristicConf.xml b/app-conf/HeuristicConf.xml index d4fc7b9c1..3a8d833d4 100644 --- a/app-conf/HeuristicConf.xml +++ b/app-conf/HeuristicConf.xml @@ -316,5 +316,11 @@ com.linkedin.drelephant.spark.heuristics.ExecutorGcHeuristic views.html.help.spark.helpExecutorGcHeuristic - + + spark + Executor spill + com.linkedin.drelephant.spark.heuristics.ExecutorStorageSpillHeuristic + views.html.help.spark.helpExecutorStorageSpillHeuristic + + diff --git a/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala b/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala index 41d88e1b8..ea3d29dbc 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala @@ -87,6 +87,7 @@ trait ExecutorSummary{ def totalShuffleWrite: Long def maxMemory: Long def totalGCTime: Long + def totalMemoryBytesSpilled: Long def executorLogs: Map[String, String]} trait JobData{ @@ -160,7 +161,7 @@ trait StageData{ def schedulingPool: String def accumulatorUpdates: Seq[AccumulableInfo] - def tasks: Option[Map[Long, TaskData]] + def tasks: Option[Map[Long, TaskDataImpl]] def executorSummary: Option[Map[String, ExecutorStageSummary]]} trait TaskData{ @@ -293,6 +294,7 @@ class ExecutorSummaryImpl( var totalShuffleWrite: Long, var maxMemory: Long, var totalGCTime: Long, + var totalMemoryBytesSpilled: Long, var executorLogs: Map[String, String]) extends ExecutorSummary class JobDataImpl( @@ -366,7 +368,7 @@ class StageDataImpl( var schedulingPool: String, var accumulatorUpdates: Seq[AccumulableInfoImpl], - var tasks: Option[Map[Long, TaskData]], + var tasks: Option[Map[Long, TaskDataImpl]], var executorSummary: Option[Map[String, ExecutorStageSummaryImpl]]) extends StageData class TaskDataImpl( diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala new file mode 100644 index 000000000..a625b441e --- /dev/null +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala @@ -0,0 +1,133 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.linkedin.drelephant.spark.heuristics + +import com.linkedin.drelephant.analysis.Severity +import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ExecutorStageSummary, ExecutorSummary, StageData} +import com.linkedin.drelephant.analysis._ +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData +import com.linkedin.drelephant.spark.data.SparkApplicationData +import com.linkedin.drelephant.util.MemoryFormatUtils + +import scala.collection.JavaConverters + + +/** + * A heuristic based on memory spilled. + * + */ +class ExecutorStorageSpillHeuristic(private val heuristicConfigurationData: HeuristicConfigurationData) + extends Heuristic[SparkApplicationData] { + + import ExecutorStorageSpillHeuristic._ + import JavaConverters._ + + val spillFractionOfExecutorsThreshold: Double = + if(heuristicConfigurationData.getParamMap.get(SPILL_FRACTION_OF_EXECUTORS_THRESHOLD_KEY) == null) DEFAULT_SPILL_FRACTION_OF_EXECUTORS_THRESHOLD + else heuristicConfigurationData.getParamMap.get(SPILL_FRACTION_OF_EXECUTORS_THRESHOLD_KEY).toDouble + + val spillMaxMemoryThreshold: Double = + if(heuristicConfigurationData.getParamMap.get(SPILL_MAX_MEMORY_THRESHOLD_KEY) == null) DEFAULT_SPILL_MAX_MEMORY_THRESHOLD + else heuristicConfigurationData.getParamMap.get(SPILL_MAX_MEMORY_THRESHOLD_KEY).toDouble + + val sparkExecutorCoresThreshold : Int = + if(heuristicConfigurationData.getParamMap.get(SPARK_EXECUTOR_CORES_THRESHOLD_KEY) == null) DEFAULT_SPARK_EXECUTOR_CORES_THRESHOLD + else heuristicConfigurationData.getParamMap.get(SPARK_EXECUTOR_CORES_THRESHOLD_KEY).toInt + + val sparkExecutorMemoryThreshold : String = + if(heuristicConfigurationData.getParamMap.get(SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY) == null) DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD + else heuristicConfigurationData.getParamMap.get(SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY) + + override def getHeuristicConfData(): HeuristicConfigurationData = heuristicConfigurationData + + override def apply(data: SparkApplicationData): HeuristicResult = { + val evaluator = new Evaluator(this, data) + var resultDetails = Seq( + new HeuristicResultDetails("Total memory spilled", MemoryFormatUtils.bytesToString(evaluator.totalMemorySpilled)), + new HeuristicResultDetails("Max memory spilled", MemoryFormatUtils.bytesToString(evaluator.maxMemorySpilled)), + new HeuristicResultDetails("Mean memory spilled", MemoryFormatUtils.bytesToString(evaluator.meanMemorySpilled)), + new HeuristicResultDetails("Fraction of executors having non zero bytes spilled", evaluator.fractionOfExecutorsHavingBytesSpilled.toString) + ) + + if(evaluator.severity != Severity.NONE){ + resultDetails :+ new HeuristicResultDetails("Note", "Your execution memory is being spilled. Kindly look into it.") + if(evaluator.sparkExecutorCores >= sparkExecutorCoresThreshold && evaluator.sparkExecutorMemory >= MemoryFormatUtils.stringToBytes(sparkExecutorMemoryThreshold)) { + resultDetails :+ new HeuristicResultDetails("Recommendation", "You can try decreasing the number of cores to reduce the number of concurrently running tasks.") + } else if (evaluator.sparkExecutorMemory <= MemoryFormatUtils.stringToBytes(sparkExecutorMemoryThreshold)) { + resultDetails :+ new HeuristicResultDetails("Recommendation", "You can try increasing the executor memory to reduce spill.") + } + } + + val result = new HeuristicResult( + heuristicConfigurationData.getClassName, + heuristicConfigurationData.getHeuristicName, + evaluator.severity, + 0, + resultDetails.asJava + ) + result + } +} + +object ExecutorStorageSpillHeuristic { + val SPARK_EXECUTOR_MEMORY = "spark.executor.memory" + val SPARK_EXECUTOR_CORES = "spark.executor.cores" + val SPILL_FRACTION_OF_EXECUTORS_THRESHOLD_KEY = "spill_fraction_of_executors_threshold" + val SPILL_MAX_MEMORY_THRESHOLD_KEY = "spill_max_memory_threshold" + val SPARK_EXECUTOR_CORES_THRESHOLD_KEY = "spark_executor_cores_threshold" + val SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY = "spark_executor_memory_threshold" + val DEFAULT_SPILL_FRACTION_OF_EXECUTORS_THRESHOLD : Double = 0.2 + val DEFAULT_SPILL_MAX_MEMORY_THRESHOLD : Double = 0.05 + val DEFAULT_SPARK_EXECUTOR_CORES_THRESHOLD : Int = 4 + val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD : String ="10GB" + + class Evaluator(executorStorageSpillHeuristic: ExecutorStorageSpillHeuristic, data: SparkApplicationData) { + lazy val executorSummaries: Seq[ExecutorSummary] = data.executorSummaries + lazy val appConfigurationProperties: Map[String, String] = + data.appConfigurationProperties + val maxMemorySpilled: Long = executorSummaries.map(_.totalMemoryBytesSpilled).max + val meanMemorySpilled = executorSummaries.map(_.totalMemoryBytesSpilled).sum / executorSummaries.size + val totalMemorySpilled = executorSummaries.map(_.totalMemoryBytesSpilled).sum + val fractionOfExecutorsHavingBytesSpilled: Double = executorSummaries.count(_.totalMemoryBytesSpilled > 0).toDouble / executorSummaries.size.toDouble + val severity: Severity = { + if (fractionOfExecutorsHavingBytesSpilled != 0) { + if (fractionOfExecutorsHavingBytesSpilled < executorStorageSpillHeuristic.spillFractionOfExecutorsThreshold + && maxMemorySpilled < executorStorageSpillHeuristic.spillMaxMemoryThreshold * sparkExecutorMemory) { + Severity.LOW + } + else if (fractionOfExecutorsHavingBytesSpilled < executorStorageSpillHeuristic.spillFractionOfExecutorsThreshold + && meanMemorySpilled < executorStorageSpillHeuristic.spillMaxMemoryThreshold * sparkExecutorMemory) { + Severity.MODERATE + } + + else if (fractionOfExecutorsHavingBytesSpilled >= executorStorageSpillHeuristic.spillFractionOfExecutorsThreshold + && meanMemorySpilled < executorStorageSpillHeuristic.spillMaxMemoryThreshold * sparkExecutorMemory) { + Severity.SEVERE + } + else if (fractionOfExecutorsHavingBytesSpilled >= executorStorageSpillHeuristic.spillFractionOfExecutorsThreshold + && meanMemorySpilled >= executorStorageSpillHeuristic.spillMaxMemoryThreshold * sparkExecutorMemory) { + Severity.CRITICAL + } else Severity.NONE + } + else Severity.NONE + } + + lazy val sparkExecutorMemory: Long = (appConfigurationProperties.get(SPARK_EXECUTOR_MEMORY).map(MemoryFormatUtils.stringToBytes)).getOrElse(0) + lazy val sparkExecutorCores: Int = (appConfigurationProperties.get(SPARK_EXECUTOR_CORES).map(_.toInt)).getOrElse(0) + } +} + diff --git a/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala b/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala index 62a58695b..f9cc59394 100644 --- a/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala +++ b/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala @@ -174,6 +174,7 @@ object LegacyDataConverters { executorInfo.shuffleWrite, executorInfo.maxMem, executorInfo.totalGCTime, + executorInfo.totalMemoryBytesSpilled, executorLogs = Map.empty ) } diff --git a/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java b/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java index 4e2ad4de3..e107a5c4d 100644 --- a/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java +++ b/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java @@ -44,6 +44,7 @@ public static class ExecutorInfo { public long outputBytes = 0L; public long shuffleRead = 0L; public long totalGCTime = 0L; + public long totalMemoryBytesSpilled = 0L; public long shuffleWrite = 0L; public String toString() { @@ -51,7 +52,7 @@ public String toString() { + ", maxMem: " + maxMem + ", diskUsed: " + diskUsed + ", totalTasks" + totalTasks + ", tasksActive: " + activeTasks + ", tasksComplete: " + completedTasks + ", tasksFailed: " + failedTasks + ", duration: " + duration + ", inputBytes: " + inputBytes + ", outputBytes:" + outputBytes + ", shuffleRead: " + shuffleRead - + ", shuffleWrite: " + shuffleWrite + ", totalGCTime: " + totalGCTime + "}"; + + ", shuffleWrite: " + shuffleWrite + ", totalGCTime: " + totalGCTime + ", totalMemoryBytesSpilled: " + totalMemoryBytesSpilled + "}"; } } diff --git a/app/com/linkedin/drelephant/util/MemoryFormatUtils.java b/app/com/linkedin/drelephant/util/MemoryFormatUtils.java index b32f61fb9..8ed49fbc0 100644 --- a/app/com/linkedin/drelephant/util/MemoryFormatUtils.java +++ b/app/com/linkedin/drelephant/util/MemoryFormatUtils.java @@ -94,6 +94,9 @@ public static long stringToBytes(String formattedString) { return 0L; } + //handling if the string has , for eg. 1,000MB + formattedString = formattedString.replace(",", ""); + Matcher matcher = REGEX_MATCHER.matcher(formattedString); if (!matcher.matches()) { throw new IllegalArgumentException( diff --git a/app/views/help/spark/helpExecutorStorageSpillHeuristic.scala.html b/app/views/help/spark/helpExecutorStorageSpillHeuristic.scala.html new file mode 100644 index 000000000..a23efb735 --- /dev/null +++ b/app/views/help/spark/helpExecutorStorageSpillHeuristic.scala.html @@ -0,0 +1,23 @@ +@* +* Copyright 2016 LinkedIn Corp. +* +* Licensed under the Apache License, Version 2.0 (the "License"); you may not +* use this file except in compliance with the License. You may obtain a copy of +* the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*@ +

Spark performs best when data is kept in memory. Spilled execution memory is tracked by memoryBytesSpilled, which is available executor level. If execution memory is being spilled, then the warnings are as follows:

+

Low: memoryBytesSpilled is non-zero for 1 or more executors, greater than zero for < 20% of executors, and max size is < .05 * spark.executor.memory.

+

Moderate: memoryBytesSpilled is non-zero for 1 or more executors, greater than zero for < 20% of executors, and avg size is < .05 * spark.executor.memory.

+

Severe: memoryBytes Spilled is greater than zero for > 20% of executors and avg size is < .05 * spark.executor.memory.

+

Critical: memoryBytes Spilled is greater than zero for > 20% of executors and/or avg size is >= .05 * spark.executor.memory.

+

Suggestions

+

If number of cores (spark.executor.cores) is more than 4 and executor memory is > 10GB : Try decreasing the number of cores which would decrese the number of tasks running in parallel, hence decreasing the number of bytes spilled.

+

You can also try increasing the spark.executor.memory which will reduce memory spilled.

diff --git a/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala b/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala index 77e3e1d29..66605dd3e 100644 --- a/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala +++ b/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala @@ -195,6 +195,7 @@ object SparkMetricsAggregatorTest { totalShuffleWrite = 0, maxMemory = 0, totalGCTime = 0, + totalMemoryBytesSpilled = 0, executorLogs = Map.empty ) } diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala index 869b9cb67..d9a8fc69f 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala @@ -117,8 +117,9 @@ object ExecutorGcHeuristicTest { totalInputBytes=0, totalShuffleRead=0, totalShuffleWrite= 0, - maxMemory= 0, + maxMemory = 0, totalGCTime, + totalMemoryBytesSpilled = 0, executorLogs = Map.empty ) diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala new file mode 100644 index 000000000..2a061e8d3 --- /dev/null +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala @@ -0,0 +1,136 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.linkedin.drelephant.spark.heuristics + +import scala.collection.JavaConverters +import com.linkedin.drelephant.analysis.{ApplicationType, Severity, SeverityThresholds} +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData +import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} +import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl, StageDataImpl} +import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate +import org.scalatest.{FunSpec, Matchers} + + +class ExecutorStorageSpillHeuristicTest extends FunSpec with Matchers { + import ExecutorStorageSpillHeuristicTest._ + + describe("ExecutorStorageSpillHeuristic") { + val heuristicConfigurationData = newFakeHeuristicConfigurationData( + Map.empty + ) + val executorStorageSpillHeuristic = new ExecutorStorageSpillHeuristic(heuristicConfigurationData) + + val appConfigurationProperties = Map("spark.executor.memory" -> "4g", "spark.executor.cores"->"4", "spark.executor.instances"->"4") + + val executorSummaries = Seq( + newFakeExecutorSummary( + id = "1", + totalMemoryBytesSpilled = 200000L + ), + newFakeExecutorSummary( + id = "2", + totalMemoryBytesSpilled = 100000L + ), + newFakeExecutorSummary( + id = "3", + totalMemoryBytesSpilled = 300000L + ), + newFakeExecutorSummary( + id = "4", + totalMemoryBytesSpilled = 200000L + ) + ) + + describe(".apply") { + val data1 = newFakeSparkApplicationData(executorSummaries, appConfigurationProperties) + val heuristicResult = executorStorageSpillHeuristic.apply(data1) + val heuristicResultDetails = heuristicResult.getHeuristicResultDetails + + it("returns the severity") { + heuristicResult.getSeverity should be(Severity.SEVERE) + } + + it("returns the total memory spilled") { + val details = heuristicResultDetails.get(0) + details.getName should include("Total memory spilled") + details.getValue should be("781.25 KB") + } + + it("returns the max memory spilled") { + val details = heuristicResultDetails.get(1) + details.getName should include("Max memory spilled") + details.getValue should be("292.97 KB") + } + + it("returns the mean memory spilled") { + val details = heuristicResultDetails.get(2) + details.getName should include("Mean memory spilled") + details.getValue should be("195.31 KB") + } + } + } +} + +object ExecutorStorageSpillHeuristicTest { + import JavaConverters._ + + def newFakeHeuristicConfigurationData(params: Map[String, String] = Map.empty): HeuristicConfigurationData = + new HeuristicConfigurationData("heuristic", "class", "view", new ApplicationType("type"), params.asJava) + + def newFakeExecutorSummary( + id: String, + totalMemoryBytesSpilled: Long + ): ExecutorSummaryImpl = new ExecutorSummaryImpl( + id, + hostPort = "", + rddBlocks = 0, + memoryUsed=0, + diskUsed = 0, + activeTasks = 0, + failedTasks = 0, + completedTasks = 0, + totalTasks = 0, + totalDuration=0, + totalInputBytes=0, + totalShuffleRead=0, + totalShuffleWrite= 0, + maxMemory= 0, + totalGCTime = 0, + totalMemoryBytesSpilled, + executorLogs = Map.empty + ) + + def newFakeSparkApplicationData( + executorSummaries: Seq[ExecutorSummaryImpl], + appConfigurationProperties: Map[String, String] + ): SparkApplicationData = { + val appId = "application_1" + + val restDerivedData = SparkRestDerivedData( + new ApplicationInfoImpl(appId, name = "app", Seq.empty), + jobDatas = Seq.empty, + stageDatas = Seq.empty, + executorSummaries = executorSummaries + ) + + val logDerivedData = SparkLogDerivedData( + SparkListenerEnvironmentUpdate(Map("Spark Properties" -> appConfigurationProperties.toSeq)) + ) + + SparkApplicationData(appId, restDerivedData, Some(logDerivedData)) + } +} diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala index 7dbeea921..b4095ccf9 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala @@ -250,6 +250,7 @@ object ExecutorsHeuristicTest { totalShuffleWrite, maxMemory, totalGCTime = 0, + totalMemoryBytesSpilled = 0, executorLogs = Map.empty ) From eec5ff7636ea3daa287732ece4c7c4f887afb745 Mon Sep 17 00:00:00 2001 From: skakker Date: Wed, 10 Jan 2018 11:37:14 +0530 Subject: [PATCH 02/15] Spark Stages with Failed tasks Heuristic - (Depends on Custom SHS - Requires stages/failedTasks Rest API) (#288) --- app-conf/FetcherConf.xml | 5 +- app-conf/HeuristicConf.xml | 6 + .../spark/data/SparkApplicationData.scala | 6 +- .../spark/data/SparkRestDerivedData.scala | 1 + .../spark/fetchers/SparkFetcher.scala | 1 + .../spark/fetchers/SparkRestClient.scala | 14 +- .../StagesWithFailedTasksHeuristic.scala | 146 +++++++++++++++ .../legacydata/LegacyDataConverters.scala | 32 +++- .../helpStagesWithFailedTasks.scala.html | 22 +++ .../spark/SparkMetricsAggregatorTest.scala | 6 +- .../spark/data/SparkApplicationDataTest.scala | 3 +- .../spark/fetchers/SparkFetcherTest.scala | 3 +- .../spark/fetchers/SparkRestClientTest.scala | 23 ++- .../ConfigurationHeuristicTest.scala | 3 +- .../heuristics/ExecutorGcHeuristicTest.scala | 3 +- .../heuristics/ExecutorsHeuristicTest.scala | 3 +- .../spark/heuristics/JobsHeuristicTest.scala | 3 +- .../heuristics/StagesHeuristicTest.scala | 3 +- .../StagesWithFailedTasksHeuristicTest.scala | 166 ++++++++++++++++++ .../drelephant/util/InfoExtractorTest.java | 6 +- 20 files changed, 437 insertions(+), 18 deletions(-) create mode 100644 app/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristic.scala create mode 100644 app/views/help/spark/helpStagesWithFailedTasks.scala.html create mode 100644 test/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristicTest.scala diff --git a/app-conf/FetcherConf.xml b/app-conf/FetcherConf.xml index 5448c9f40..dc78a64eb 100644 --- a/app-conf/FetcherConf.xml +++ b/app-conf/FetcherConf.xml @@ -88,7 +88,10 @@ --> spark - com.linkedin.drelephant.spark.fetchers.FSFetcher + com.linkedin.drelephant.spark.fetchers.SparkFetcher + + true + - + mapreduce com.linkedin.drelephant.mapreduce.fetchers.MapReduceFSFetcherHadoop2 @@ -68,7 +68,7 @@ PST - + + spark Spark Job Metrics com.linkedin.drelephant.spark.heuristics.JobsHeuristic views.html.help.spark.helpJobsHeuristic + spark Spark Stage Metrics com.linkedin.drelephant.spark.heuristics.StagesHeuristic views.html.help.spark.helpStagesHeuristic + spark - Peak Unified Memory + Executor Peak Unified Memory com.linkedin.drelephant.spark.heuristics.UnifiedMemoryHeuristic views.html.help.spark.helpUnifiedMemoryHeuristic + + spark - JVM Used Memory + Executor JVM Used Memory com.linkedin.drelephant.spark.heuristics.JvmUsedMemoryHeuristic views.html.help.spark.helpJvmUsedMemoryHeuristic + + spark Stages with failed tasks com.linkedin.drelephant.spark.heuristics.StagesWithFailedTasksHeuristic views.html.help.spark.helpStagesWithFailedTasks + spark Executor GC com.linkedin.drelephant.spark.heuristics.ExecutorGcHeuristic views.html.help.spark.helpExecutorGcHeuristic + + spark Executor spill com.linkedin.drelephant.spark.heuristics.ExecutorStorageSpillHeuristic views.html.help.spark.helpExecutorStorageSpillHeuristic + + + + + spark + Driver Metrics + com.linkedin.drelephant.spark.heuristics.DriverHeuristic + views.html.help.spark.helpDriverHeuristic + diff --git a/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala b/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala index 42c5ae668..21c4fb1a3 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala @@ -223,7 +223,7 @@ class SparkRestClient(sparkConf: SparkConf) { } private def getExecutorSummaries(attemptTarget: WebTarget): Seq[ExecutorSummaryImpl] = { - val target = attemptTarget.path("executors") + val target = attemptTarget.path("allexecutors") try { get(target, SparkRestObjectMapper.readValue[Seq[ExecutorSummaryImpl]]) } catch { diff --git a/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala index 0c0193ef5..5af1ed94f 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala @@ -29,7 +29,7 @@ import com.linkedin.drelephant.math.Statistics * A heuristic based on an app's known configuration. * * The results from this heuristic primarily inform users about key app configuration settings, including - * driver memory, driver cores, executor cores, executor instances, executor memory, and the serializer. + * executor cores, executor instances, executor memory, and the serializer. * * It also checks whether the values specified are within threshold. */ @@ -55,10 +55,6 @@ class ConfigurationHeuristic(private val heuristicConfigurationData: HeuristicCo property.getOrElse("Not presented. Using default.") val resultDetails = Seq( - new HeuristicResultDetails( - SPARK_DRIVER_MEMORY_KEY, - formatProperty(evaluator.driverMemoryBytes.map(MemoryFormatUtils.bytesToString)) - ), new HeuristicResultDetails( SPARK_EXECUTOR_MEMORY_KEY, formatProperty(evaluator.executorMemoryBytes.map(MemoryFormatUtils.bytesToString)) @@ -80,16 +76,16 @@ class ConfigurationHeuristic(private val heuristicConfigurationData: HeuristicCo formatProperty(evaluator.isDynamicAllocationEnabled.map(_.toString)) ), new HeuristicResultDetails( - SPARK_DRIVER_CORES_KEY, - formatProperty(evaluator.driverCores.map(_.toString)) + SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD, + evaluator.sparkYarnExecutorMemoryOverhead ), new HeuristicResultDetails( - SPARK_YARN_DRIVER_MEMORY_OVERHEAD, - evaluator.sparkYarnDriverMemoryOverhead + SPARK_DYNAMIC_ALLOCATION_MIN_EXECUTORS, + evaluator.dynamicMinExecutors.getOrElse(0).toString ), new HeuristicResultDetails( - SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD, - evaluator.sparkYarnExecutorMemoryOverhead + SPARK_DYNAMIC_ALLOCATION_MAX_EXECUTORS, + evaluator.dynamicMaxExecutors.getOrElse(0).toString ) ) // Constructing a mutable ArrayList for resultDetails, otherwise addResultDetail method HeuristicResult cannot be used. @@ -109,21 +105,24 @@ class ConfigurationHeuristic(private val heuristicConfigurationData: HeuristicCo result.addResultDetail(SPARK_SHUFFLE_SERVICE_ENABLED, formatProperty(evaluator.isShuffleServiceEnabled.map(_.toString)), "Spark shuffle service is not enabled.") } - if (evaluator.severityMinExecutors == Severity.CRITICAL) { - result.addResultDetail("Minimum Executors", "The minimum executors for Dynamic Allocation should be <=1. Please change it in the " + SPARK_DYNAMIC_ALLOCATION_MIN_EXECUTORS + " field.") + if (evaluator.severityMinExecutors != Severity.NONE) { + result.addResultDetail("Minimum Executors", "The minimum executors for Dynamic Allocation should be "+ THRESHOLD_MIN_EXECUTORS + ". Please change it in the " + SPARK_DYNAMIC_ALLOCATION_MIN_EXECUTORS + " field.") } - if (evaluator.severityMaxExecutors == Severity.CRITICAL) { - result.addResultDetail("Maximum Executors", "The maximum executors for Dynamic Allocation should be <=900. Please change it in the " + SPARK_DYNAMIC_ALLOCATION_MAX_EXECUTORS + " field.") + if (evaluator.severityMaxExecutors != Severity.NONE) { + result.addResultDetail("Maximum Executors", "The maximum executors for Dynamic Allocation should be <=" + THRESHOLD_MAX_EXECUTORS + ". Please change it in the " + SPARK_DYNAMIC_ALLOCATION_MAX_EXECUTORS + " field.") } - if (evaluator.jarsSeverity == Severity.CRITICAL) { + if (evaluator.jarsSeverity != Severity.NONE) { result.addResultDetail("Jars notation", "It is recommended to not use * notation while specifying jars in the field " + SPARK_YARN_JARS) } - if(evaluator.severityDriverMemoryOverhead.getValue >= Severity.SEVERE.getValue) { - result.addResultDetail("Driver Overhead Memory", "Please do not specify excessive amount of overhead memory for Driver. Change it in the field " + SPARK_YARN_DRIVER_MEMORY_OVERHEAD) - } - if(evaluator.severityExecutorMemoryOverhead.getValue >= Severity.SEVERE.getValue) { + if(evaluator.severityExecutorMemoryOverhead != Severity.NONE) { result.addResultDetail("Executor Overhead Memory", "Please do not specify excessive amount of overhead memory for Executors. Change it in the field " + SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD) } + if(evaluator.severityExecutorCores != Severity.NONE) { + result.addResultDetail("Executor cores", "The number of executor cores should be <=" + evaluator.DEFAULT_SPARK_CORES_THRESHOLDS.low + ". Please change it in the field " + SPARK_EXECUTOR_CORES_KEY) + } + if(evaluator.severityExecutorMemory != Severity.NONE) { + result.addResultDetail("Executor memory", "Please do not specify excessive amount of executor memory. Change it in the field " + SPARK_EXECUTOR_MEMORY_KEY) + } result } } @@ -134,7 +133,6 @@ object ConfigurationHeuristic { val SERIALIZER_IF_NON_NULL_RECOMMENDATION_KEY = "serializer_if_non_null_recommendation" - val SPARK_DRIVER_MEMORY_KEY = "spark.driver.memory" val SPARK_EXECUTOR_MEMORY_KEY = "spark.executor.memory" val SPARK_EXECUTOR_INSTANCES_KEY = "spark.executor.instances" val SPARK_EXECUTOR_CORES_KEY = "spark.executor.cores" @@ -142,12 +140,10 @@ object ConfigurationHeuristic { val SPARK_APPLICATION_DURATION = "spark.application.duration" val SPARK_SHUFFLE_SERVICE_ENABLED = "spark.shuffle.service.enabled" val SPARK_DYNAMIC_ALLOCATION_ENABLED = "spark.dynamicAllocation.enabled" - val SPARK_DRIVER_CORES_KEY = "spark.driver.cores" val SPARK_DYNAMIC_ALLOCATION_MIN_EXECUTORS = "spark.dynamicAllocation.minExecutors" val SPARK_DYNAMIC_ALLOCATION_MAX_EXECUTORS = "spark.dynamicAllocation.maxExecutors" val SPARK_YARN_JARS = "spark.yarn.secondary.jars" val SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD = "spark.yarn.executor.memoryOverhead" - val SPARK_YARN_DRIVER_MEMORY_OVERHEAD = "spark.yarn.driver.memoryOverhead" val THRESHOLD_MIN_EXECUTORS: Int = 1 val THRESHOLD_MAX_EXECUTORS: Int = 900 val SPARK_OVERHEAD_MEMORY_THRESHOLD_KEY = "spark.overheadMemory.thresholds.key" @@ -159,9 +155,6 @@ object ConfigurationHeuristic { lazy val appConfigurationProperties: Map[String, String] = data.appConfigurationProperties - lazy val driverMemoryBytes: Option[Long] = - Try(getProperty(SPARK_DRIVER_MEMORY_KEY).map(MemoryFormatUtils.stringToBytes)).getOrElse(None) - lazy val executorMemoryBytes: Option[Long] = Try(getProperty(SPARK_EXECUTOR_MEMORY_KEY).map(MemoryFormatUtils.stringToBytes)).getOrElse(None) @@ -171,9 +164,6 @@ object ConfigurationHeuristic { lazy val executorCores: Option[Int] = Try(getProperty(SPARK_EXECUTOR_CORES_KEY).map(_.toInt)).getOrElse(None) - lazy val driverCores: Option[Int] = - Try(getProperty(SPARK_DRIVER_CORES_KEY).map(_.toInt)).getOrElse(None) - lazy val dynamicMinExecutors: Option[Int] = Try(getProperty(SPARK_DYNAMIC_ALLOCATION_MIN_EXECUTORS).map(_.toInt)).getOrElse(None) @@ -196,8 +186,6 @@ object ConfigurationHeuristic { lazy val sparkYarnExecutorMemoryOverhead: String = if (getProperty(SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD).getOrElse("0").matches("(.*)[0-9]")) MemoryFormatUtils.bytesToString(MemoryFormatUtils.stringToBytes(getProperty(SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD).getOrElse("0") + "MB")) else (getProperty(SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD).getOrElse("0")) - lazy val sparkYarnDriverMemoryOverhead: String = if (getProperty(SPARK_YARN_DRIVER_MEMORY_OVERHEAD).getOrElse("0").matches("(.*)[0-9]")) - MemoryFormatUtils.bytesToString(MemoryFormatUtils.stringToBytes(getProperty(SPARK_YARN_DRIVER_MEMORY_OVERHEAD).getOrElse("0") + "MB")) else getProperty(SPARK_YARN_DRIVER_MEMORY_OVERHEAD).getOrElse("0") lazy val serializer: Option[String] = getProperty(SPARK_SERIALIZER_KEY) @@ -211,7 +199,7 @@ object ConfigurationHeuristic { case Some(_) => DEFAULT_SERIALIZER_IF_NON_NULL_SEVERITY_IF_RECOMMENDATION_UNMET } - //The following thresholds are for checking if the memory and cores values (executor and driver) are above normal. These thresholds are experimental, and may change in the future. + //The following thresholds are for checking if the memory and cores values of executors are above normal. These thresholds are experimental, and may change in the future. val DEFAULT_SPARK_MEMORY_THRESHOLDS = SeverityThresholds(low = MemoryFormatUtils.stringToBytes("10G"), MemoryFormatUtils.stringToBytes("15G"), severe = MemoryFormatUtils.stringToBytes("20G"), critical = MemoryFormatUtils.stringToBytes("25G"), ascending = true) @@ -219,8 +207,6 @@ object ConfigurationHeuristic { SeverityThresholds(low = 4, moderate = 6, severe = 8, critical = 10, ascending = true) val severityExecutorMemory = DEFAULT_SPARK_MEMORY_THRESHOLDS.severityOf(executorMemoryBytes.getOrElse(0).asInstanceOf[Number].longValue) - val severityDriverMemory = DEFAULT_SPARK_MEMORY_THRESHOLDS.severityOf(driverMemoryBytes.getOrElse(0).asInstanceOf[Number].longValue) - val severityDriverCores = DEFAULT_SPARK_CORES_THRESHOLDS.severityOf(driverCores.getOrElse(0).asInstanceOf[Number].intValue) val severityExecutorCores = DEFAULT_SPARK_CORES_THRESHOLDS.severityOf(executorCores.getOrElse(0).asInstanceOf[Number].intValue) val severityMinExecutors = if (dynamicMinExecutors.getOrElse(0).asInstanceOf[Number].intValue > THRESHOLD_MIN_EXECUTORS) { Severity.CRITICAL @@ -233,12 +219,10 @@ object ConfigurationHeuristic { Severity.NONE } val severityExecutorMemoryOverhead = configurationHeuristic.sparkOverheadMemoryThreshold.severityOf(MemoryFormatUtils.stringToBytes(sparkYarnExecutorMemoryOverhead)) - val severityDriverMemoryOverhead = configurationHeuristic.sparkOverheadMemoryThreshold.severityOf(MemoryFormatUtils.stringToBytes(sparkYarnDriverMemoryOverhead)) - //Severity for the configuration thresholds - val severityConfThresholds: Severity = Severity.max(severityDriverCores, severityDriverMemory, severityExecutorCores, severityExecutorMemory, - severityMinExecutors, severityMaxExecutors, jarsSeverity, severityExecutorMemoryOverhead, severityDriverMemoryOverhead) + val severityConfThresholds: Severity = Severity.max(severityExecutorCores, severityExecutorMemory, + severityMinExecutors, severityMaxExecutors, jarsSeverity, severityExecutorMemoryOverhead) /** * The following logic computes severity based on shuffle service and dynamic allocation flags. diff --git a/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala new file mode 100644 index 000000000..b52356b07 --- /dev/null +++ b/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala @@ -0,0 +1,190 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.linkedin.drelephant.spark.heuristics + +import java.util.ArrayList + +import scala.collection.JavaConverters +import scala.util.Try +import com.linkedin.drelephant.analysis._ +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData +import com.linkedin.drelephant.spark.data.SparkApplicationData +import com.linkedin.drelephant.util.MemoryFormatUtils +import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary + +/** + * A heuristic based the driver's configurations and memory used. + * It checks whether the configuration values specified are within the threshold range. + * It also analyses the peak JVM memory used and time spent in GC by the job. + */ +class DriverHeuristic(private val heuristicConfigurationData: HeuristicConfigurationData) + extends Heuristic[SparkApplicationData] { + + import DriverHeuristic._ + import JavaConverters._ + + val gcSeverityThresholds: SeverityThresholds = + SeverityThresholds.parse(heuristicConfigurationData.getParamMap.get(GC_SEVERITY_THRESHOLDS_KEY), ascending = true) + .getOrElse(DEFAULT_GC_SEVERITY_THRESHOLDS) + + val sparkOverheadMemoryThreshold: SeverityThresholds = SeverityThresholds.parse(heuristicConfigurationData.getParamMap.get(SPARK_OVERHEAD_MEMORY_THRESHOLD_KEY), ascending = true) + .getOrElse(DEFAULT_SPARK_OVERHEAD_MEMORY_THRESHOLDS) + + val sparkExecutorMemoryThreshold: String = heuristicConfigurationData.getParamMap.getOrDefault(SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY, DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD) + + override def getHeuristicConfData(): HeuristicConfigurationData = heuristicConfigurationData + + lazy val driverPeakJvmMemoryThresholdString: String = heuristicConfigurationData.getParamMap.get(MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLD_KEY) + + override def apply(data: SparkApplicationData): HeuristicResult = { + val evaluator = new Evaluator(this, data) + + def formatProperty(property: Option[String]): String = + property.getOrElse("Not presented. Using default.") + + var resultDetails = Seq( + new HeuristicResultDetails( + SPARK_DRIVER_MEMORY_KEY, + formatProperty(evaluator.driverMemoryBytes.map(MemoryFormatUtils.bytesToString)) + ), + new HeuristicResultDetails( + "Ratio of time spent in GC to total time", evaluator.ratio.toString + ), + new HeuristicResultDetails( + SPARK_DRIVER_CORES_KEY, + formatProperty(evaluator.driverCores.map(_.toString)) + ), + new HeuristicResultDetails( + SPARK_YARN_DRIVER_MEMORY_OVERHEAD, + evaluator.sparkYarnDriverMemoryOverhead + ), + new HeuristicResultDetails("Max driver peak JVM used memory", MemoryFormatUtils.bytesToString(evaluator.maxDriverPeakJvmUsedMemory)) + ) + if(evaluator.severityJvmUsedMemory != Severity.NONE) { + resultDetails = resultDetails :+ new HeuristicResultDetails("Driver Peak JVM used Memory", "The allocated memory for the driver (in " + SPARK_DRIVER_MEMORY_KEY + ") is much more than the peak JVM used memory by the driver.") + } + if (evaluator.severityGc != Severity.NONE) { + resultDetails = resultDetails :+ new HeuristicResultDetails("Gc ratio high", "The driver is spending too much time on GC. We recommend increasing the driver memory.") + } + if(evaluator.severityDriverCores != Severity.NONE) { + resultDetails = resultDetails :+ new HeuristicResultDetails("Driver Cores", "Please do not specify excessive number of driver cores. Change it in the field : " + SPARK_DRIVER_CORES_KEY) + } + if(evaluator.severityDriverMemoryOverhead != Severity.NONE) { + resultDetails = resultDetails :+ new HeuristicResultDetails("Driver Overhead Memory", "Please do not specify excessive amount of overhead memory for Driver. Change it in the field " + SPARK_YARN_DRIVER_MEMORY_OVERHEAD) + } + if(evaluator.severityDriverMemory != Severity.NONE) { + resultDetails = resultDetails :+ new HeuristicResultDetails("Spark Driver Memory", "Please do not specify excessive amount of memory for Driver. Change it in the field " + SPARK_DRIVER_MEMORY_KEY) + } + + // Constructing a mutable ArrayList for resultDetails, otherwise addResultDetail method HeuristicResult cannot be used. + val mutableResultDetailsArrayList = new ArrayList(resultDetails.asJava) + val result = new HeuristicResult( + heuristicConfigurationData.getClassName, + heuristicConfigurationData.getHeuristicName, + evaluator.severity, + 0, + mutableResultDetailsArrayList + ) + result + } +} + +object DriverHeuristic { + + val SPARK_DRIVER_MEMORY_KEY = "spark.driver.memory" + val SPARK_DRIVER_CORES_KEY = "spark.driver.cores" + val SPARK_YARN_DRIVER_MEMORY_OVERHEAD = "spark.yarn.driver.memoryOverhead" + val SPARK_OVERHEAD_MEMORY_THRESHOLD_KEY = "spark.overheadMemory.thresholds.key" + val SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY = "spark_executor_memory_threshold_key" + val EXECUTION_MEMORY = "executionMemory" + val STORAGE_MEMORY = "storageMemory" + val JVM_USED_MEMORY = "jvmUsedMemory" + + // 300 * FileUtils.ONE_MB (300 * 1024 * 1024) + val reservedMemory : Long = 314572800 + val MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLD_KEY = "peak_jvm_memory_threshold" + val GC_SEVERITY_THRESHOLDS_KEY: String = "gc_severity_threshold" + val DEFAULT_GC_SEVERITY_THRESHOLDS = + SeverityThresholds(low = 0.08D, moderate = 0.09D, severe = 0.1D, critical = 0.15D, ascending = true) + + val DEFAULT_SPARK_OVERHEAD_MEMORY_THRESHOLDS = + SeverityThresholds(low = MemoryFormatUtils.stringToBytes("2G"), MemoryFormatUtils.stringToBytes("4G"), + severe = MemoryFormatUtils.stringToBytes("6G"), critical = MemoryFormatUtils.stringToBytes("8G"), ascending = true) + + val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD = "2G" + + class Evaluator(driverHeuristic: DriverHeuristic, data: SparkApplicationData) { + lazy val appConfigurationProperties: Map[String, String] = + data.appConfigurationProperties + + lazy val executorSummaries : Seq[ExecutorSummary] = data.executorSummaries + lazy val driver : ExecutorSummary = executorSummaries.find(_.id == "driver").getOrElse(null) + + if(driver == null) { + throw new Exception("No driver found!") + } + + //peakJvmMemory calculations + val maxDriverPeakJvmUsedMemory : Long = driver.peakJvmUsedMemory.getOrElse(JVM_USED_MEMORY, 0L).asInstanceOf[Number].longValue + + lazy val DEFAULT_MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS = + SeverityThresholds(low = 1.25 * (maxDriverPeakJvmUsedMemory + reservedMemory), moderate = 1.5 * (maxDriverPeakJvmUsedMemory + reservedMemory), + severe = 2 * (maxDriverPeakJvmUsedMemory + reservedMemory), critical = 3 * (maxDriverPeakJvmUsedMemory + reservedMemory), ascending = true) + + val MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS : SeverityThresholds = if(driverHeuristic.driverPeakJvmMemoryThresholdString == null) { + DEFAULT_MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS + } else { + SeverityThresholds.parse(driverHeuristic.driverPeakJvmMemoryThresholdString.split(",").map(_.toDouble * (maxDriverPeakJvmUsedMemory + reservedMemory)).toString, ascending = false).getOrElse(DEFAULT_MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS) + } + + lazy val severityJvmUsedMemory : Severity = if (driverMemoryBytes.getOrElse(0L).asInstanceOf[Number].longValue <= MemoryFormatUtils.stringToBytes(driverHeuristic.sparkExecutorMemoryThreshold)) { + Severity.NONE + } else { + MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS.severityOf(driverMemoryBytes.getOrElse(0L).asInstanceOf[Number].longValue) + } + + //Gc Calculations + val ratio : Double = driver.totalGCTime.toDouble / driver.totalDuration.toDouble + val severityGc = driverHeuristic.gcSeverityThresholds.severityOf(ratio) + + lazy val driverMemoryBytes: Option[Long] = + Try(getProperty(SPARK_DRIVER_MEMORY_KEY).map(MemoryFormatUtils.stringToBytes)).getOrElse(None) + + lazy val driverCores: Option[Int] = + Try(getProperty(SPARK_DRIVER_CORES_KEY).map(_.toInt)).getOrElse(None) + + lazy val sparkYarnDriverMemoryOverhead: String = if (getProperty(SPARK_YARN_DRIVER_MEMORY_OVERHEAD).getOrElse("0").matches("(.*)[0-9]")) + MemoryFormatUtils.bytesToString(MemoryFormatUtils.stringToBytes(getProperty(SPARK_YARN_DRIVER_MEMORY_OVERHEAD).getOrElse("0") + "MB")) else getProperty(SPARK_YARN_DRIVER_MEMORY_OVERHEAD).getOrElse("0") + + //The following thresholds are for checking if the memory and cores values (driver) are above normal. These thresholds are experimental, and may change in the future. + val DEFAULT_SPARK_MEMORY_THRESHOLDS = + SeverityThresholds(low = MemoryFormatUtils.stringToBytes("10G"), MemoryFormatUtils.stringToBytes("15G"), + severe = MemoryFormatUtils.stringToBytes("20G"), critical = MemoryFormatUtils.stringToBytes("25G"), ascending = true) + val DEFAULT_SPARK_CORES_THRESHOLDS = + SeverityThresholds(low = 4, moderate = 6, severe = 8, critical = 10, ascending = true) + + val severityDriverMemory = DEFAULT_SPARK_MEMORY_THRESHOLDS.severityOf(driverMemoryBytes.getOrElse(0).asInstanceOf[Number].longValue) + val severityDriverCores = DEFAULT_SPARK_CORES_THRESHOLDS.severityOf(driverCores.getOrElse(0).asInstanceOf[Number].intValue) + val severityDriverMemoryOverhead = driverHeuristic.sparkOverheadMemoryThreshold.severityOf(MemoryFormatUtils.stringToBytes(sparkYarnDriverMemoryOverhead)) + + //Severity for the configuration thresholds + val severityConfThresholds: Severity = Severity.max(severityDriverCores, severityDriverMemory, severityDriverMemoryOverhead) + lazy val severity: Severity = Severity.max(severityConfThresholds, severityGc, severityJvmUsedMemory) + private def getProperty(key: String): Option[String] = appConfigurationProperties.get(key) + } + +} diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala index d4ef405dc..7b8929ed7 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala @@ -21,6 +21,8 @@ import com.linkedin.drelephant.spark.fetchers.statusapiv1._ import com.linkedin.drelephant.analysis._ import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData import com.linkedin.drelephant.spark.data.SparkApplicationData +import com.linkedin.drelephant.math.Statistics + import scala.collection.JavaConverters @@ -47,8 +49,8 @@ class ExecutorGcHeuristic(private val heuristicConfigurationData: HeuristicConfi val evaluator = new Evaluator(this, data) var resultDetails = Seq( new HeuristicResultDetails("GC time to Executor Run time ratio", evaluator.ratio.toString), - new HeuristicResultDetails("Total GC time", evaluator.jvmTime.toString), - new HeuristicResultDetails("Total Executor Runtime", evaluator.executorRunTimeTotal.toString) + new HeuristicResultDetails("Total GC time", evaluator.msecToString(evaluator.jvmTime)), + new HeuristicResultDetails("Total Executor Runtime", evaluator.msecToString(evaluator.executorRunTimeTotal)) ) //adding recommendations to the result, severityTimeA corresponds to the ascending severity calculation @@ -98,7 +100,7 @@ object ExecutorGcHeuristic { if (executorSummaries.isEmpty) { throw new Exception("No executor information available.") } - + lazy val appConfigurationProperties: Map[String, String] = data.appConfigurationProperties var (jvmTime, executorRunTimeTotal) = getTimeValues(executorSummaries) @@ -122,6 +124,25 @@ object ExecutorGcHeuristic { }) (jvmGcTimeTotal, executorRunTimeTotal) } + + //convert millisec to units + def msecToString (milliSec: Long): String = { + var value : Long = milliSec + if(value < Statistics.SECOND_IN_MS){ + return value.toString + " msec" + } else if(value < Statistics.MINUTE_IN_MS) { + return (value/Statistics.SECOND_IN_MS).toString + " Seconds" + } else if(value < Statistics.HOUR_IN_MS) { + return (value/Statistics.MINUTE_IN_MS).toString + " Minutes" + }else { + var minutes = (value % Statistics.HOUR_IN_MS)/Statistics.MINUTE_IN_MS + if(minutes == 0) { + return (value/Statistics.HOUR_IN_MS).toString + " Hours" + } else { + return (value/Statistics.HOUR_IN_MS).toString + " Hours " + minutes.toString + " Minutes" + } + } + } } } diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala index 7001c8e27..90571ed75 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristic.scala @@ -97,13 +97,20 @@ object ExecutorStorageSpillHeuristic { class Evaluator(executorStorageSpillHeuristic: ExecutorStorageSpillHeuristic, data: SparkApplicationData) { lazy val executorAndDriverSummaries: Seq[ExecutorSummary] = data.executorSummaries + if (executorAndDriverSummaries == null) { + throw new Exception("Executors Summary is null.") + } lazy val executorSummaries: Seq[ExecutorSummary] = executorAndDriverSummaries.filterNot(_.id.equals("driver")) + if (executorSummaries.isEmpty) { + throw new Exception("No executor information available.") + } lazy val appConfigurationProperties: Map[String, String] = data.appConfigurationProperties val maxTasks: Int = executorSummaries.head.maxTasks val maxMemorySpilled: Long = executorSummaries.map(_.totalMemoryBytesSpilled).max val meanMemorySpilled = executorSummaries.map(_.totalMemoryBytesSpilled).sum / executorSummaries.size - val totalMemorySpilledPerTask = totalMemorySpilled/(executorSummaries.map(_.totalTasks).sum) + lazy val totalTasks = Integer.max(executorSummaries.map(_.totalTasks).sum, 1) + val totalMemorySpilledPerTask = totalMemorySpilled/totalTasks lazy val totalMemorySpilled = executorSummaries.map(_.totalMemoryBytesSpilled).sum val fractionOfExecutorsHavingBytesSpilled: Double = executorSummaries.count(_.totalMemoryBytesSpilled > 0).toDouble / executorSummaries.size.toDouble val severity: Severity = { diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala index dae604124..d125bc492 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala @@ -16,16 +16,16 @@ package com.linkedin.drelephant.spark.heuristics -import scala.collection.JavaConverters -import scala.collection.mutable.ArrayBuffer - -import com.linkedin.drelephant.analysis.{Heuristic, HeuristicResult, HeuristicResultDetails, Severity, SeverityThresholds} +import com.linkedin.drelephant.analysis._ import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData import com.linkedin.drelephant.math.Statistics import com.linkedin.drelephant.spark.data.SparkApplicationData import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary import com.linkedin.drelephant.util.MemoryFormatUtils +import scala.collection.JavaConverters +import scala.collection.mutable.ArrayBuffer + /** * A heuristic based on metrics for a Spark app's executors. @@ -37,6 +37,7 @@ import com.linkedin.drelephant.util.MemoryFormatUtils class ExecutorsHeuristic(private val heuristicConfigurationData: HeuristicConfigurationData) extends Heuristic[SparkApplicationData] { import ExecutorsHeuristic._ + import JavaConverters._ val maxToMedianRatioSeverityThresholds: SeverityThresholds = diff --git a/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala index f57aba948..5875151ed 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala @@ -26,7 +26,7 @@ import scala.collection.JavaConverters /** - * A heuristic based on peak JVM used memory for the spark executors and driver + * A heuristic based on peak JVM used memory for the spark executors * */ class JvmUsedMemoryHeuristic(private val heuristicConfigurationData: HeuristicConfigurationData) @@ -38,25 +38,19 @@ class JvmUsedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo override def getHeuristicConfData(): HeuristicConfigurationData = heuristicConfigurationData lazy val executorPeakJvmMemoryThresholdString: String = heuristicConfigurationData.getParamMap.get(MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLD_KEY) - lazy val driverPeakJvmMemoryThresholdString: String = heuristicConfigurationData.getParamMap.get(MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLD_KEY) + lazy val sparkExecutorMemoryThreshold: String = heuristicConfigurationData.getParamMap.getOrDefault(SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY, DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD) override def apply(data: SparkApplicationData): HeuristicResult = { val evaluator = new Evaluator(this, data) var resultDetails = Seq( new HeuristicResultDetails("Max executor peak JVM used memory", MemoryFormatUtils.bytesToString(evaluator.maxExecutorPeakJvmUsedMemory)), - new HeuristicResultDetails("Max driver peak JVM used memory", MemoryFormatUtils.bytesToString(evaluator.maxDriverPeakJvmUsedMemory)), - new HeuristicResultDetails("spark.executor.memory", MemoryFormatUtils.bytesToString(evaluator.sparkExecutorMemory)), - new HeuristicResultDetails("spark.driver.memory", MemoryFormatUtils.bytesToString(evaluator.sparkDriverMemory)) + new HeuristicResultDetails("spark.executor.memory", MemoryFormatUtils.bytesToString(evaluator.sparkExecutorMemory)) ) - if(evaluator.severityExecutor.getValue > Severity.LOW.getValue) { - resultDetails :+ new HeuristicResultDetails("Executor Memory", "The allocated memory for the executor (in " + SPARK_EXECUTOR_MEMORY +") is much more than the peak JVM used memory by executors.") - resultDetails :+ new HeuristicResultDetails("Reasonable size for executor memory", ((1+BUFFER_PERCENT.toDouble/100.0)*evaluator.maxExecutorPeakJvmUsedMemory).toString) - } - - if(evaluator.severityDriver.getValue > Severity.LOW.getValue) { - resultDetails :+ new HeuristicResultDetails("Driver Memory", "The allocated memory for the driver (in " + SPARK_DRIVER_MEMORY + ") is much more than the peak JVM used memory by the driver.") + if (evaluator.severity != Severity.NONE) { + resultDetails = resultDetails :+ new HeuristicResultDetails("Executor Memory", "The allocated memory for the executor (in " + SPARK_EXECUTOR_MEMORY + ") is much more than the peak JVM used memory by executors.") + resultDetails = resultDetails :+ new HeuristicResultDetails("Suggested spark.executor.memory", MemoryFormatUtils.roundOffMemoryStringToNextInteger((MemoryFormatUtils.bytesToString(((1 + BUFFER_FRACTION) * evaluator.maxExecutorPeakJvmUsedMemory).toLong)))) } val result = new HeuristicResult( @@ -73,52 +67,39 @@ class JvmUsedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo object JvmUsedMemoryHeuristic { val JVM_USED_MEMORY = "jvmUsedMemory" val SPARK_EXECUTOR_MEMORY = "spark.executor.memory" - val SPARK_DRIVER_MEMORY = "spark.driver.memory" + val SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY = "spark_executor_memory_threshold" + // 300 * FileUtils.ONE_MB (300 * 1024 * 1024) - val reservedMemory : Long = 314572800 - val BUFFER_PERCENT : Int = 20 + val reservedMemory: Long = 314572800 + val BUFFER_FRACTION: Double = 0.2 val MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLD_KEY = "executor_peak_jvm_memory_threshold" - val MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLD_KEY = "driver_peak_jvm_memory_threshold" + lazy val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD = "2G" class Evaluator(jvmUsedMemoryHeuristic: JvmUsedMemoryHeuristic, data: SparkApplicationData) { lazy val appConfigurationProperties: Map[String, String] = data.appConfigurationProperties lazy val executorSummaries: Seq[ExecutorSummary] = data.executorSummaries - lazy val driverSummary : Option[ExecutorSummary] = executorSummaries.find(_.id.equals("driver")) - val maxDriverPeakJvmUsedMemory : Long = driverSummary.get.peakJvmUsedMemory.getOrElse(JVM_USED_MEMORY, 0L).asInstanceOf[Number].longValue - val executorList : Seq[ExecutorSummary] = executorSummaries.filterNot(_.id.equals("driver")) - val sparkExecutorMemory : Long = (appConfigurationProperties.get(SPARK_EXECUTOR_MEMORY).map(MemoryFormatUtils.stringToBytes)).getOrElse(0L) - val sparkDriverMemory : Long = appConfigurationProperties.get(SPARK_DRIVER_MEMORY).map(MemoryFormatUtils.stringToBytes).getOrElse(0L) + val executorList: Seq[ExecutorSummary] = executorSummaries.filterNot(_.id.equals("driver")) + val sparkExecutorMemory: Long = (appConfigurationProperties.get(SPARK_EXECUTOR_MEMORY).map(MemoryFormatUtils.stringToBytes)).getOrElse(0L) lazy val maxExecutorPeakJvmUsedMemory: Long = if (executorList.isEmpty) 0L else executorList.map { _.peakJvmUsedMemory.getOrElse(JVM_USED_MEMORY, 0).asInstanceOf[Number].longValue }.max lazy val DEFAULT_MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLDS = - SeverityThresholds(low = 1.5 * (maxExecutorPeakJvmUsedMemory + reservedMemory), moderate = 2 * (maxExecutorPeakJvmUsedMemory + reservedMemory), severe = 4 * (maxExecutorPeakJvmUsedMemory + reservedMemory), critical = 8 * (maxExecutorPeakJvmUsedMemory + reservedMemory), ascending = true) - - lazy val DEFAULT_MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS = - SeverityThresholds(low = 1.5 * (maxDriverPeakJvmUsedMemory + reservedMemory), moderate = 2 * (maxDriverPeakJvmUsedMemory + reservedMemory), severe = 4 * (maxDriverPeakJvmUsedMemory + reservedMemory), critical = 8 * (maxDriverPeakJvmUsedMemory + reservedMemory), ascending = true) + SeverityThresholds(low = 1.25 * (maxExecutorPeakJvmUsedMemory + reservedMemory), moderate = 1.5 * (maxExecutorPeakJvmUsedMemory + reservedMemory), severe = 2 * (maxExecutorPeakJvmUsedMemory + reservedMemory), critical = 3 * (maxExecutorPeakJvmUsedMemory + reservedMemory), ascending = true) - val MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLDS : SeverityThresholds = if(jvmUsedMemoryHeuristic.executorPeakJvmMemoryThresholdString == null) { + val MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLDS: SeverityThresholds = if (jvmUsedMemoryHeuristic.executorPeakJvmMemoryThresholdString == null) { DEFAULT_MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLDS } else { SeverityThresholds.parse(jvmUsedMemoryHeuristic.executorPeakJvmMemoryThresholdString.split(",").map(_.toDouble * (maxExecutorPeakJvmUsedMemory + reservedMemory)).toString, ascending = false).getOrElse(DEFAULT_MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLDS) } - val MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS : SeverityThresholds = if(jvmUsedMemoryHeuristic.driverPeakJvmMemoryThresholdString == null) { - DEFAULT_MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS + lazy val severity = if (sparkExecutorMemory <= MemoryFormatUtils.stringToBytes(jvmUsedMemoryHeuristic.sparkExecutorMemoryThreshold)) { + Severity.NONE } else { - SeverityThresholds.parse(jvmUsedMemoryHeuristic.driverPeakJvmMemoryThresholdString.split(",").map(_.toDouble * (maxDriverPeakJvmUsedMemory + reservedMemory)).toString, ascending = false).getOrElse(DEFAULT_MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS) + MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLDS.severityOf(sparkExecutorMemory) } - - val severityExecutor = MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLDS.severityOf(sparkExecutorMemory) - val severityDriver = MAX_DRIVER_PEAK_JVM_USED_MEMORY_THRESHOLDS.severityOf(sparkDriverMemory) - - /** - * disabling the skew check for executors - * val severitySkew = DEFAULT_JVM_MEMORY_SKEW_THRESHOLDS.severityOf(maxExecutorPeakJvmUsedMemory) - */ - val severity : Severity = Severity.max(severityDriver, severityExecutor) } + } diff --git a/app/com/linkedin/drelephant/spark/heuristics/StagesHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/StagesHeuristic.scala index b2c36f90b..baa0426bd 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/StagesHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/StagesHeuristic.scala @@ -192,7 +192,8 @@ object StagesHeuristic { } private def averageExecutorRuntimeAndSeverityOf(stageData: StageData): (Long, Severity) = { - val averageExecutorRuntime = stageData.executorRunTime / executorInstances + val allTasks : Int = Integer.max((stageData.numActiveTasks + stageData.numCompleteTasks + stageData.numFailedTasks), 1) + val averageExecutorRuntime = stageData.executorRunTime / allTasks (averageExecutorRuntime, stageRuntimeMillisSeverityThresholds.severityOf(averageExecutorRuntime)) } } diff --git a/app/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristic.scala index 162d1021a..c062db2c1 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristic.scala @@ -43,7 +43,7 @@ class StagesWithFailedTasksHeuristic(private val heuristicConfigurationData: Heu new HeuristicResultDetails("Stages with Overhead memory errors", evaluator.stagesWithOverheadError.toString) ) if (evaluator.severityOverheadStages.getValue >= Severity.MODERATE.getValue) - resultDetails = resultDetails :+ new HeuristicResultDetails("Overhead memory errors", "Some tasks have failed due to overhead memory error. Please try increasing spark.yarn.executor.memoryOverhead by 500MB in spark.yarn.executor.memoryOverhead") + resultDetails = resultDetails :+ new HeuristicResultDetails("Overhead memory errors", "Some tasks have failed due to overhead memory error. Please try increasing spark.yarn.executor.memoryOverhead by " + increaseMemoryBy +" in spark.yarn.executor.memoryOverhead") //TODO: refine recommendations if (evaluator.severityOOMStages.getValue >= Severity.MODERATE.getValue) resultDetails = resultDetails :+ new HeuristicResultDetails("OOM errors", "Some tasks have failed due to OOM error. Try increasing spark.executor.memory or decreasing spark.memory.fraction (take a look at unified memory heuristic) or decreasing number of cores.") @@ -63,6 +63,7 @@ object StagesWithFailedTasksHeuristic { val OOM_ERROR = "java.lang.OutOfMemoryError" val OVERHEAD_MEMORY_ERROR = "killed by YARN for exceeding memory limits" val ratioThreshold: Double = 2 + val increaseMemoryBy: String = "1G" class Evaluator(memoryFractionHeuristic: StagesWithFailedTasksHeuristic, data: SparkApplicationData) { lazy val stagesWithFailedTasks: Seq[StageData] = data.stagesWithFailedTasks diff --git a/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala index 3b6f54cfb..53d261d9b 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala @@ -39,6 +39,7 @@ class UnifiedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo override def getHeuristicConfData(): HeuristicConfigurationData = heuristicConfigurationData lazy val peakUnifiedMemoryThresholdString: String = heuristicConfigurationData.getParamMap.get(PEAK_UNIFIED_MEMORY_THRESHOLD_KEY) + val sparkExecutorMemoryThreshold: String = heuristicConfigurationData.getParamMap.getOrDefault(SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY, DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD) override def apply(data: SparkApplicationData): HeuristicResult = { val evaluator = new Evaluator(this, data) @@ -69,6 +70,8 @@ object UnifiedMemoryHeuristic { val SPARK_EXECUTOR_MEMORY_KEY = "spark.executor.memory" val SPARK_MEMORY_FRACTION_KEY = "spark.memory.fraction" val PEAK_UNIFIED_MEMORY_THRESHOLD_KEY = "peak_unified_memory_threshold" + val SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY = "spark_executor_memory_threshold" + val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD = "2G" class Evaluator(unifiedMemoryHeuristic: UnifiedMemoryHeuristic, data: SparkApplicationData) { lazy val appConfigurationProperties: Map[String, String] = @@ -95,39 +98,28 @@ object UnifiedMemoryHeuristic { SeverityThresholds.parse(unifiedMemoryHeuristic.peakUnifiedMemoryThresholdString.split(",").map(_.toDouble * maxMemory).toString, ascending = false).getOrElse(DEFAULT_PEAK_UNIFIED_MEMORY_THRESHOLD) } - def getPeakUnifiedMemoryExecutorSeverity(executorSummary: ExecutorSummary): Severity = { - return PEAK_UNIFIED_MEMORY_THRESHOLDS.severityOf(executorSummary.peakUnifiedMemory.getOrElse(EXECUTION_MEMORY, 0).asInstanceOf[Number].longValue - + executorSummary.peakUnifiedMemory.getOrElse(STORAGE_MEMORY, 0).asInstanceOf[Number].longValue) - } - val sparkExecutorMemory: Long = (appConfigurationProperties.get(SPARK_EXECUTOR_MEMORY_KEY).map(MemoryFormatUtils.stringToBytes)).getOrElse(0L) - val sparkMemoryFraction: Double = appConfigurationProperties.getOrElse(SPARK_MEMORY_FRACTION_KEY, 0.6D).asInstanceOf[Number].doubleValue + lazy val sparkMemoryFraction: Double = appConfigurationProperties.getOrElse(SPARK_MEMORY_FRACTION_KEY, "0.6").toDouble lazy val meanUnifiedMemory: Long = (executorList.map { executorSummary => { - executorSummary.peakUnifiedMemory.getOrElse(EXECUTION_MEMORY, 0).asInstanceOf[Number].longValue - +executorSummary.peakUnifiedMemory.getOrElse(STORAGE_MEMORY, 0).asInstanceOf[Number].longValue + (executorSummary.peakUnifiedMemory.getOrElse(EXECUTION_MEMORY, 0).asInstanceOf[Number].longValue + + executorSummary.peakUnifiedMemory.getOrElse(STORAGE_MEMORY, 0).asInstanceOf[Number].longValue) } }.sum) / executorList.size lazy val maxUnifiedMemory: Long = executorList.map { executorSummary => { - executorSummary.peakUnifiedMemory.getOrElse(EXECUTION_MEMORY, 0).asInstanceOf[Number].longValue - +executorSummary.peakUnifiedMemory.getOrElse(STORAGE_MEMORY, 0).asInstanceOf[Number].longValue + (executorSummary.peakUnifiedMemory.getOrElse(EXECUTION_MEMORY, 0).asInstanceOf[Number].longValue + + executorSummary.peakUnifiedMemory.getOrElse(STORAGE_MEMORY, 0).asInstanceOf[Number].longValue) } }.max - lazy val severity: Severity = { - var severityPeakUnifiedMemoryVariable: Severity = Severity.NONE - for (executorSummary <- executorList) { - var peakUnifiedMemoryExecutorSeverity: Severity = getPeakUnifiedMemoryExecutorSeverity(executorSummary) - if (peakUnifiedMemoryExecutorSeverity.getValue > severityPeakUnifiedMemoryVariable.getValue) { - severityPeakUnifiedMemoryVariable = peakUnifiedMemoryExecutorSeverity - } - } - severityPeakUnifiedMemoryVariable + lazy val severity: Severity = if (sparkExecutorMemory <= MemoryFormatUtils.stringToBytes(unifiedMemoryHeuristic.sparkExecutorMemoryThreshold)) { + Severity.NONE + } else { + PEAK_UNIFIED_MEMORY_THRESHOLDS.severityOf(maxUnifiedMemory) } } - } diff --git a/app/com/linkedin/drelephant/util/MemoryFormatUtils.java b/app/com/linkedin/drelephant/util/MemoryFormatUtils.java index 8ed49fbc0..0d7a8f584 100644 --- a/app/com/linkedin/drelephant/util/MemoryFormatUtils.java +++ b/app/com/linkedin/drelephant/util/MemoryFormatUtils.java @@ -90,7 +90,7 @@ public static String bytesToString(long value) { * @return The bytes value */ public static long stringToBytes(String formattedString) { - if (formattedString == null) { + if (formattedString == null || formattedString.isEmpty()) { return 0L; } @@ -124,4 +124,41 @@ public static long stringToBytes(String formattedString) { + "] does not match any unit. The supported units are (case-insensitive, and also the 'B' is ignorable): [" + StringUtils.join(UNITS) + "]."); } + + /** + * Given a memory value in string format, it rounds off the double value to next integer. + * @param formattedString + * @return : formatted String with int value to next integer. + */ + public static String roundOffMemoryStringToNextInteger(String formattedString) { + if (formattedString == null || formattedString.isEmpty()) { + return ""; + } + + //handling if the string has , for eg. 1,000MB + formattedString = formattedString.replace(",", ""); + + Matcher matcher = REGEX_MATCHER.matcher(formattedString); + if (!matcher.matches()) { + throw new IllegalArgumentException( + "The formatted string [" + formattedString + "] does not match with the regex /" + REGEX_MATCHER.toString() + + "/"); + } + if (matcher.groupCount() != 1 && matcher.groupCount() != 2) { + throw new IllegalArgumentException(); + } + + double numPart = Double.parseDouble(matcher.group(1)); + if (numPart < 0) { + throw new IllegalArgumentException("The number part of the memory cannot be less than zero: [" + numPart + "]."); + } + + int numPartInt = (int) Math.ceil(numPart); + + String unitPart = matcher.groupCount() == 2 ? matcher.group(2).toUpperCase() : ""; + if (!unitPart.endsWith("B")) { + unitPart += "B"; + } + return (numPartInt + " " + unitPart); + } } diff --git a/app/views/help/spark/helpConfigurationHeuristic.scala.html b/app/views/help/spark/helpConfigurationHeuristic.scala.html index 1d9ec968f..6bf673521 100644 --- a/app/views/help/spark/helpConfigurationHeuristic.scala.html +++ b/app/views/help/spark/helpConfigurationHeuristic.scala.html @@ -14,8 +14,7 @@ * the License. *@

The results from this heuristic primarily inform you about key app -configuration settings, including driver memory, driver cores, executor cores, -executor instances, executor memory, and the serializer.

-

It also checks the values of dynamically allocated min and max executors, the specified yarn jars, executor and driver memory overhead and whether other configuration values are within threshold.

+configuration settings, including executor cores, executor instances, executor memory, and the serializer.

+

It also checks the values of dynamically allocated min and max executors, the specified yarn jars, executor memory overhead and whether other configuration values are within threshold.

Suggestions

Suggestions based on the configurations you have set are given in the heuristic result itself.

\ No newline at end of file diff --git a/app/views/help/spark/helpDriverHeuristic.scala.html b/app/views/help/spark/helpDriverHeuristic.scala.html new file mode 100644 index 000000000..83f65a674 --- /dev/null +++ b/app/views/help/spark/helpDriverHeuristic.scala.html @@ -0,0 +1,23 @@ +@* +* Copyright 2016 LinkedIn Corp. +* +* Licensed under the Apache License, Version 2.0 (the "License"); you may not +* use this file except in compliance with the License. You may obtain a copy of +* the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*@ +

This is a heuristic for checking whether the driver is well tuned and the configurations are set to a good value.

+

It checks the following properties

+

Driver Max Peak JVM Used Memory

+

This analyses whether the driver memory is set to a good value. To avoid wasted memory, it checks if the peak JVM used memory by the driver is reasonably close to the user allocated driver memory which is specified in spark.driver.memory. If the peak JVM memory is much smaller, then the driver memory should be reduced.

+

Time spent by driver in GC

+

This checks if your job spends too much time in GC. We recommend increasing spark.driver.memory if it does.

+

Checking configuration thresholds

+

The values of spark.driver.memory, spark.driver.cores and spark.yarn.driver.memoryOverhead are checked to verify if they are within threshold values.

\ No newline at end of file diff --git a/app/views/help/spark/helpJvmUsedMemoryHeuristic.scala.html b/app/views/help/spark/helpJvmUsedMemoryHeuristic.scala.html index f58462821..8db78a999 100644 --- a/app/views/help/spark/helpJvmUsedMemoryHeuristic.scala.html +++ b/app/views/help/spark/helpJvmUsedMemoryHeuristic.scala.html @@ -15,6 +15,5 @@ *@

This is a heuristic for peak JVM used memory.

Executor Max Peak JVM Used Memory

-

This is to analyse whether the executor memory is set to a good value. To avoid wasted memory, it checks if the peak JVM used memory by the executor is reasonably close to the blocked executor memory which is specified in spark.executor.memory. If the peak JVM memory is much smaller, then the executor memory should be reduced.

-

Driver Max Peak JVM Used Memory

-

This is to analyse whether the driver memory is set to a good value. To avoid wasted memory, it checks if the peak JVM used memory by the driver is reasonably close to the blocked driver memory which is specified in spark.driver.memory. If the peak JVM memory is much smaller, then the driver memory should be reduced.

+

This is to analyse whether the executor memory is set to a good value. To avoid wasted memory, it checks if the peak JVM used memory by the executor is reasonably close to the user allocated executor memory which is specified in spark.executor.memory. If the peak JVM memory is much smaller, then the executor memory should be reduced.

+

Note: Please note that for calculation purposes Dr. Elephant considers 1024 Bytes in 1 KB whereas the spark history server considers 1000 Bytes. So please don't get confused if you find discrepancy in values from these two places.

\ No newline at end of file diff --git a/app/views/help/spark/helpUnifiedMemoryHeuristic.scala.html b/app/views/help/spark/helpUnifiedMemoryHeuristic.scala.html index 9ea156004..352e9e3ff 100644 --- a/app/views/help/spark/helpUnifiedMemoryHeuristic.scala.html +++ b/app/views/help/spark/helpUnifiedMemoryHeuristic.scala.html @@ -22,3 +22,4 @@

Action Items

2. If your job's Executor Memory is high, then we recommend reducing the spark.executor.memory itself which will lower the Allocated Unified Memory space.

Note:

spark.memory.fraction: This is the fraction of JVM Used Memory (Executor memory - Reserved memory) dedicated to the unified memory region (execution + storage). It basically partitions user memory from execution and storage memory.

+

Note: Please note that for calculation purposes Dr. Elephant considers 1024 Bytes in 1 KB whereas the spark history server considers 1000 Bytes. So please don't get confused if you find discrepancy in values from these two places.

\ No newline at end of file diff --git a/test/com/linkedin/drelephant/spark/fetchers/SparkRestClientTest.scala b/test/com/linkedin/drelephant/spark/fetchers/SparkRestClientTest.scala index 947fc8e7b..c20223fb8 100644 --- a/test/com/linkedin/drelephant/spark/fetchers/SparkRestClientTest.scala +++ b/test/com/linkedin/drelephant/spark/fetchers/SparkRestClientTest.scala @@ -303,7 +303,7 @@ object SparkRestClientTest { @Path("applications/{appId}/{attemptId}/stages") def getStages(): StagesResource = new StagesResource() - @Path("applications/{appId}/{attemptId}/executors") + @Path("applications/{appId}/{attemptId}/allexecutors") def getExecutors(): ExecutorsResource = new ExecutorsResource() @Path("applications/{appId}/{attemptId}/logs") @@ -385,7 +385,7 @@ object SparkRestClientTest { @Path("applications/{appId}/stages") def getStages(): StagesResource = new StagesResource() - @Path("applications/{appId}/executors") + @Path("applications/{appId}/allexecutors") def getExecutors(): ExecutorsResource = new ExecutorsResource() @Path("applications/{appId}/logs") diff --git a/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala index f04777d5b..f22b5379e 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala @@ -27,7 +27,9 @@ import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} import java.util.Date - +/** + * Test class for Configuration Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ class ConfigurationHeuristicTest extends FunSpec with Matchers { import ConfigurationHeuristicTest._ @@ -60,67 +62,50 @@ class ConfigurationHeuristicTest extends FunSpec with Matchers { val heuristicResultDetails = heuristicResult.getHeuristicResultDetails it("returns the size of result details") { - heuristicResultDetails.size() should be(9) + heuristicResultDetails.size() should be(8) } it("returns the severity") { heuristicResult.getSeverity should be(Severity.NONE) } - it("returns the driver memory") { - val details = heuristicResultDetails.get(0) - details.getName should include("spark.driver.memory") - details.getValue should be("2 GB") - } - it("returns the executor memory") { - val details = heuristicResultDetails.get(1) + val details = heuristicResultDetails.get(0) details.getName should include("spark.executor.memory") details.getValue should be("1 GB") } it("returns the executor instances") { - val details = heuristicResultDetails.get(2) + val details = heuristicResultDetails.get(1) details.getName should include("spark.executor.instances") details.getValue should be("900") } it("returns the executor cores") { - val details = heuristicResultDetails.get(3) + val details = heuristicResultDetails.get(2) details.getName should include("spark.executor.cores") details.getValue should include("default") } it("returns the application duration") { - val details = heuristicResultDetails.get(4) + val details = heuristicResultDetails.get(3) details.getName should include("spark.application.duration") details.getValue should include("10") } it("returns the dynamic allocation flag") { - val details = heuristicResultDetails.get(5) + val details = heuristicResultDetails.get(4) details.getName should include("spark.dynamicAllocation.enabled") details.getValue should be("true") } - - it("returns the driver cores") { - val details = heuristicResultDetails.get(6) - details.getName should include("spark.driver.cores") - details.getValue should include("default") - } - - it("returns the driver overhead memory") { - val details = heuristicResultDetails.get(7) - details.getName should include("spark.yarn.driver.memoryOverhead") - details.getValue should include("500 MB") - } } describe("apply with Severity") { val configurationProperties = Map( "spark.serializer" -> "dummySerializer", "spark.shuffle.service.enabled" -> "false", - "spark.dynamicAllocation.enabled" -> "true" + "spark.dynamicAllocation.enabled" -> "true", + "spark.executor.cores" -> "5" ) val data = newFakeSparkApplicationData(configurationProperties) @@ -136,24 +121,30 @@ class ConfigurationHeuristicTest extends FunSpec with Matchers { } it("returns the dynamic allocation flag") { - val details = heuristicResultDetails.get(5) + val details = heuristicResultDetails.get(4) details.getName should include("spark.dynamicAllocation.enabled") details.getValue should be("true") } it("returns the serializer") { - val details = heuristicResultDetails.get(9) + val details = heuristicResultDetails.get(8) details.getName should include("spark.serializer") details.getValue should be("dummySerializer") details.getDetails should be("KyroSerializer is Not Enabled.") } it("returns the shuffle service flag") { - val details = heuristicResultDetails.get(10) + val details = heuristicResultDetails.get(9) details.getName should include("spark.shuffle.service.enabled") details.getValue should be("false") details.getDetails should be("Spark shuffle service is not enabled.") } + + it("returns executor cores") { + val details = heuristicResultDetails.get(10) + details.getName should include("Executor cores") + details.getValue should be("The number of executor cores should be <=4. Please change it in the field spark.executor.cores") + } } describe(".Evaluator") { @@ -163,16 +154,6 @@ class ConfigurationHeuristicTest extends FunSpec with Matchers { new Evaluator(configurationHeuristic, newFakeSparkApplicationData(configurationProperties)) } - it("has the driver memory bytes when they're present") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.driver.memory" -> "2G")) - evaluator.driverMemoryBytes should be(Some(2L * 1024 * 1024 * 1024)) - } - - it("has no driver memory bytes when they're absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) - evaluator.driverMemoryBytes should be(None) - } - it("has the executor memory bytes when they're present") { val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.executor.memory" -> "1g")) evaluator.executorMemoryBytes should be(Some(1L * 1024 * 1024 * 1024)) @@ -198,21 +179,11 @@ class ConfigurationHeuristicTest extends FunSpec with Matchers { evaluator.executorCores should be(Some(2)) } - it("has the driver cores when they're present") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.driver.cores" -> "3")) - evaluator.driverCores should be(Some(3)) - } - it("has no executor cores when they're absent") { val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) evaluator.executorCores should be(None) } - it("has no driver cores when they're absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) - evaluator.driverCores should be(None) - } - it("has the serializer when it's present") { val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer")) evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer")) diff --git a/test/com/linkedin/drelephant/spark/heuristics/DriverHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/DriverHeuristicTest.scala new file mode 100644 index 000000000..fe0da4ec6 --- /dev/null +++ b/test/com/linkedin/drelephant/spark/heuristics/DriverHeuristicTest.scala @@ -0,0 +1,104 @@ +package com.linkedin.drelephant.spark.heuristics + +import com.linkedin.drelephant.analysis.{ApplicationType, Severity} +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData +import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkRestDerivedData} +import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl} +import com.linkedin.drelephant.spark.heuristics.DriverHeuristic.Evaluator +import org.scalatest.{FunSpec, Matchers} + +import scala.collection.JavaConverters +import scala.concurrent.duration.Duration + +/** + * Test class for Driver Metrics Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ +class DriverHeuristicTest extends FunSpec with Matchers { + + import DriverHeuristicTest._ + + val heuristicConfigurationData = newFakeHeuristicConfigurationData() + + val driverHeuristic = new DriverHeuristic(heuristicConfigurationData) + + val executorData = Seq( + newDummyExecutorData("1", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 94567), null, 0, 0), + newDummyExecutorData("2", 400000, Map("executionMemory" -> 200000, "storageMemory" -> 34568), null, 0, 0), + newDummyExecutorData("3", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 34569), null, 0, 0), + newDummyExecutorData("4", 400000, Map("executionMemory" -> 20000, "storageMemory" -> 3456), null, 0, 0), + newDummyExecutorData("5", 400000, Map("executionMemory" -> 200000, "storageMemory" -> 34564), null, 0, 0), + newDummyExecutorData("driver", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 94561), Map("jvmUsedMemory" -> 394567123), + totalGCTime = Duration("2min").toMillis, totalDuration = Duration("15min").toMillis) + ) + describe(".apply") { + val data = newFakeSparkApplicationData(executorData) + val heuristicResult = driverHeuristic.apply(data) + val heuristicResultDetails = heuristicResult.getHeuristicResultDetails + + it("has severity") { + heuristicResult.getSeverity should be(Severity.SEVERE) + } + + describe("Evaluator") { + val evaluator = new Evaluator(driverHeuristic, data) + it("has max driver peak JVM memory") { + evaluator.maxDriverPeakJvmUsedMemory should be(394567123) + } + it("ratio of time spend in Gc to total duration") { + evaluator.ratio should be(0.13333333333333333) + } + } + } +} + +object DriverHeuristicTest { + + import JavaConverters._ + + def newFakeHeuristicConfigurationData(params: Map[String, String] = Map.empty): HeuristicConfigurationData = + new HeuristicConfigurationData("heuristic", "class", "view", new ApplicationType("type"), params.asJava) + + def newDummyExecutorData( + id: String, + maxMemory: Long, + peakUnifiedMemory: Map[String, Long], + peakJvmUsedMemory: Map[String, Long], + totalGCTime: Long, + totalDuration: Long + ): ExecutorSummaryImpl = new ExecutorSummaryImpl( + id, + hostPort = "", + rddBlocks = 0, + memoryUsed = 0, + diskUsed = 0, + activeTasks = 0, + failedTasks = 0, + completedTasks = 0, + totalTasks = 0, + maxTasks = 0, + totalDuration, + totalInputBytes = 0, + totalShuffleRead = 0, + totalShuffleWrite = 0, + maxMemory, + totalGCTime, + totalMemoryBytesSpilled = 0, + executorLogs = Map.empty, + peakJvmUsedMemory, + peakUnifiedMemory + ) + + def newFakeSparkApplicationData(executorSummaries: Seq[ExecutorSummaryImpl]): SparkApplicationData = { + val appId = "application_1" + val restDerivedData = SparkRestDerivedData( + new ApplicationInfoImpl(appId, name = "app", Seq.empty), + jobDatas = Seq.empty, + stageDatas = Seq.empty, + executorSummaries = executorSummaries, + stagesWithFailedTasks = Seq.empty + ) + + SparkApplicationData(appId, restDerivedData, logDerivedData = None) + } +} + diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala index 8bec2b3fa..2204922c1 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala @@ -26,18 +26,14 @@ import org.scalatest.{FunSpec, Matchers} import scala.concurrent.duration.Duration - +/** + * Test class for Executor GC Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ class ExecutorGcHeuristicTest extends FunSpec with Matchers { import ExecutorGcHeuristicTest._ describe("ExecutorGcHeuristic") { - val heuristicConfigurationData = newFakeHeuristicConfigurationData( - Map( - "max_to_median_ratio_severity_thresholds" -> "1.414,2,4,16", - "ignore_max_bytes_less_than_threshold" -> "4000000", - "ignore_max_millis_less_than_threshold" -> "4000001" - ) - ) + val heuristicConfigurationData = newFakeHeuristicConfigurationData() val executorGcHeuristic = new ExecutorGcHeuristic(heuristicConfigurationData) val executorSummaries = Seq( @@ -63,10 +59,21 @@ class ExecutorGcHeuristicTest extends FunSpec with Matchers { ) ) + val executorSummaries1 = Seq( + newFakeExecutorSummary( + id = "1", + totalGCTime = 500, + totalDuration = 700 + ) + ) + describe(".apply") { - val data1 = newFakeSparkApplicationData(executorSummaries) - val heuristicResult = executorGcHeuristic.apply(data1) + val data = newFakeSparkApplicationData(executorSummaries) + val data1 = newFakeSparkApplicationData(executorSummaries1) + val heuristicResult = executorGcHeuristic.apply(data) + val heuristicResult1 = executorGcHeuristic.apply(data1) val heuristicResultDetails = heuristicResult.getHeuristicResultDetails + val heuristicResultDetails1 = heuristicResult1.getHeuristicResultDetails it("returns the severity") { heuristicResult.getSeverity should be(Severity.CRITICAL) @@ -81,13 +88,25 @@ class ExecutorGcHeuristicTest extends FunSpec with Matchers { it("returns the total GC time") { val details = heuristicResultDetails.get(1) details.getName should include("Total GC time") - details.getValue should be("1200000") + details.getValue should be("20 Minutes") } it("returns the executor's run time") { val details = heuristicResultDetails.get(2) details.getName should include("Total Executor Runtime") - details.getValue should be("4740000") + details.getValue should be("1 Hours 19 Minutes") + } + + it("returns total Gc Time in millisec") { + val details = heuristicResultDetails1.get(1) + details.getName should include("Total GC time") + details.getValue should be("500 msec") + } + + it("returns executor run Time in millisec") { + val details = heuristicResultDetails1.get(2) + details.getName should include("Total Executor Runtime") + details.getValue should be("700 msec") } } } diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala index 6b083c5d5..e02e4f2fb 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala @@ -21,10 +21,13 @@ import com.linkedin.drelephant.analysis.{ApplicationType, Severity, SeverityThre import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl, StageDataImpl} +import com.linkedin.drelephant.spark.heuristics.ExecutorStorageSpillHeuristic.Evaluator import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} - +/** + * Test class for Executor Storage Spill Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ class ExecutorStorageSpillHeuristicTest extends FunSpec with Matchers { import ExecutorStorageSpillHeuristicTest._ @@ -59,6 +62,7 @@ class ExecutorStorageSpillHeuristicTest extends FunSpec with Matchers { val data1 = newFakeSparkApplicationData(executorSummaries, appConfigurationProperties) val heuristicResult = executorStorageSpillHeuristic.apply(data1) val heuristicResultDetails = heuristicResult.getHeuristicResultDetails + val evaluator = new Evaluator(executorStorageSpillHeuristic, data1) it("returns the severity") { heuristicResult.getSeverity should be(Severity.SEVERE) @@ -81,6 +85,10 @@ class ExecutorStorageSpillHeuristicTest extends FunSpec with Matchers { details.getName should include("Mean memory spilled") details.getValue should be("195.31 KB") } + + it("has the memory spilled per task") { + evaluator.totalMemorySpilledPerTask should be(800000) + } } } } @@ -103,7 +111,7 @@ object ExecutorStorageSpillHeuristicTest { activeTasks = 0, failedTasks = 0, completedTasks = 0, - totalTasks = 10, + totalTasks = 0, maxTasks = 10, totalDuration=0, totalInputBytes=0, diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala index 2ef2580e9..c1996fe41 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala @@ -25,7 +25,9 @@ import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} - +/** + * Test class for Executors Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ class ExecutorsHeuristicTest extends FunSpec with Matchers { import ExecutorsHeuristicTest._ diff --git a/test/com/linkedin/drelephant/spark/heuristics/JobsHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/JobsHeuristicTest.scala index c112a312e..435eddcdb 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/JobsHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/JobsHeuristicTest.scala @@ -26,7 +26,9 @@ import org.apache.spark.JobExecutionStatus import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} - +/** + * Test class for Jobs Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ class JobsHeuristicTest extends FunSpec with Matchers { import JobsHeuristicTest._ diff --git a/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala index d9b2e2106..e0dc6641f 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala @@ -9,6 +9,9 @@ import org.scalatest.{FunSpec, Matchers} import scala.collection.JavaConverters +/** + * Test class for JVM Used memory. It checks whether all the values used in the heuristic are calculated correctly. + */ class JvmUsedMemoryHeuristicTest extends FunSpec with Matchers { import JvmUsedMemoryHeuristicTest._ @@ -17,44 +20,49 @@ class JvmUsedMemoryHeuristicTest extends FunSpec with Matchers { val peakJvmUsedMemoryHeuristic = new JvmUsedMemoryHeuristic(heuristicConfigurationData) - val appConfigurationProperties = Map("spark.driver.memory"->"40000000000", "spark.executor.memory"->"500000000") + val appConfigurationProperties = Map("spark.driver.memory"->"40000000000", "spark.executor.memory"->"50000000000") val executorData = Seq( newDummyExecutorData("1", Map("jvmUsedMemory" -> 394567123)), - newDummyExecutorData("2", Map("jvmUsedMemory" -> 23456834)), - newDummyExecutorData("3", Map("jvmUsedMemory" -> 334569)), - newDummyExecutorData("4", Map("jvmUsedMemory" -> 134563)), - newDummyExecutorData("5", Map("jvmUsedMemory" -> 234564)), + newDummyExecutorData("2", Map("jvmUsedMemory" -> 2834)), + newDummyExecutorData("3", Map("jvmUsedMemory" -> 3945667)), + newDummyExecutorData("4", Map("jvmUsedMemory" -> 16367890)), + newDummyExecutorData("5", Map("jvmUsedMemory" -> 2345647)), newDummyExecutorData("driver", Map("jvmUsedMemory" -> 394561)) ) describe(".apply") { val data = newFakeSparkApplicationData(appConfigurationProperties, executorData) val heuristicResult = peakJvmUsedMemoryHeuristic.apply(data) + val heuristicResultDetails = heuristicResult.getHeuristicResultDetails it("has severity") { heuristicResult.getSeverity should be(Severity.CRITICAL) } + it("has all the details") { + heuristicResultDetails.size() should be(4) + } + describe(".Evaluator") { import JvmUsedMemoryHeuristic.Evaluator val data = newFakeSparkApplicationData(appConfigurationProperties, executorData) + val heuristicResult = peakJvmUsedMemoryHeuristic.apply(data) + val heuristicResultDetails = heuristicResult.getHeuristicResultDetails val evaluator = new Evaluator(peakJvmUsedMemoryHeuristic, data) it("has severity executor") { - evaluator.severityExecutor should be(Severity.NONE) - } - - it("has severity driver") { - evaluator.severityDriver should be(Severity.CRITICAL) + evaluator.severity should be(Severity.CRITICAL) } it("has max peak jvm memory") { evaluator.maxExecutorPeakJvmUsedMemory should be (394567123) } - it("has max driver peak jvm memory") { - evaluator.maxDriverPeakJvmUsedMemory should be (394561) + it("has reasonable size") { + val details = heuristicResultDetails.get(3) + details.getName should be ("Suggested spark.executor.memory") + details.getValue should be ("452 MB") } } } diff --git a/test/com/linkedin/drelephant/spark/heuristics/StagesHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/StagesHeuristicTest.scala index 723f69250..e6aae4fe1 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/StagesHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/StagesHeuristicTest.scala @@ -27,6 +27,9 @@ import com.linkedin.drelephant.spark.fetchers.statusapiv1.StageStatus import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} +/** + * Test class for Stages Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ class StagesHeuristicTest extends FunSpec with Matchers { import StagesHeuristicTest._ @@ -81,13 +84,6 @@ class StagesHeuristicTest extends FunSpec with Matchers { |stage 4, attempt 0 (task failure rate: 0.800)""".stripMargin ) } - - it("returns the list of stages with long runtimes") { - heuristicResultDetails.get(4).getValue should be( - s"""|stage 8, attempt 0 (runtime: 45 min) - |stage 9, attempt 0 (runtime: 1 hr)""".stripMargin - ) - } } describe(".Evaluator") { @@ -113,15 +109,6 @@ class StagesHeuristicTest extends FunSpec with Matchers { evaluator.stagesWithHighTaskFailureRates.map { case (stageData, taskFailureRate) => (stageData.stageId, taskFailureRate) } stageIdsAndTaskFailureRates should contain theSameElementsInOrderAs(Seq((3, 0.6D), (4, 0.8D))) } - - it("has the list of stages with long average executor runtimes") { - val stageIdsAndRuntimes = - evaluator.stagesWithLongAverageExecutorRuntimes.map { case (stageData, runtime) => (stageData.stageId, runtime) } - stageIdsAndRuntimes should contain theSameElementsInOrderAs( - Seq((8, Duration("45min").toMillis), (9, Duration("60min").toMillis)) - ) - } - it("computes the overall severity") { evaluator.severity should be(Severity.CRITICAL) } diff --git a/test/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristicTest.scala index 219e3ec05..cdfdc11ea 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/StagesWithFailedTasksHeuristicTest.scala @@ -28,7 +28,9 @@ import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationDa import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkRestDerivedData} import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, StageDataImpl, TaskDataImpl} - +/** + * Test class for Stages With Failed Tasks Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ class StagesWithFailedTasksHeuristicTest extends FunSpec with Matchers { import StagesWithFailedTasksHeuristicTest._ diff --git a/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala index 746b32741..68f668efe 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala @@ -2,36 +2,89 @@ package com.linkedin.drelephant.spark.heuristics import com.linkedin.drelephant.analysis.{ApplicationType, Severity} import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData -import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkRestDerivedData} +import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl} +import com.linkedin.drelephant.spark.heuristics.UnifiedMemoryHeuristic.Evaluator +import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} import scala.collection.JavaConverters +/** + * Test class for Unified Memory Heuristic. It checks whether all the values used in the heuristic are calculated correctly. + */ class UnifiedMemoryHeuristicTest extends FunSpec with Matchers { import UnifiedMemoryHeuristicTest._ val heuristicConfigurationData = newFakeHeuristicConfigurationData() - - val memoryFractionHeuristic = new UnifiedMemoryHeuristic(heuristicConfigurationData) + val unifiedMemoryHeuristic = new UnifiedMemoryHeuristic(heuristicConfigurationData) + val appConfigurationProperties = Map("spark.executor.memory"->"3147483647") + val appConfigurationProperties1 = Map("spark.executor.memory"->"214567874847") val executorData = Seq( - newDummyExecutorData("1", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 94567)), + newDummyExecutorData("1", 999999999, Map("executionMemory" -> 300000, "storageMemory" -> 94567)), newDummyExecutorData("2", 400000, Map("executionMemory" -> 200000, "storageMemory" -> 34568)), newDummyExecutorData("3", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 34569)), newDummyExecutorData("4", 400000, Map("executionMemory" -> 20000, "storageMemory" -> 3456)), newDummyExecutorData("5", 400000, Map("executionMemory" -> 200000, "storageMemory" -> 34564)), newDummyExecutorData("6", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 94561)) ) + + val executorData1 = Seq( + newDummyExecutorData("driver", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 94567)), + newDummyExecutorData("2", 999999999, Map("executionMemory" -> 200, "storageMemory" -> 200)) + ) + + val executorData2 = Seq( + newDummyExecutorData("driver", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 94567)), + newDummyExecutorData("2", 999999999, Map("executionMemory" -> 999999990, "storageMemory" -> 9)) + ) + describe(".apply") { - val data = newFakeSparkApplicationData(executorData) - val heuristicResult = memoryFractionHeuristic.apply(data) - val heuristicResultDetails = heuristicResult.getHeuristicResultDetails + val data = newFakeSparkApplicationData(appConfigurationProperties, executorData) + val data1 = newFakeSparkApplicationData(appConfigurationProperties1, executorData1) + val data2 = newFakeSparkApplicationData(appConfigurationProperties1, executorData2) + val heuristicResult = unifiedMemoryHeuristic.apply(data) + val heuristicResult1 = unifiedMemoryHeuristic.apply(data1) + val heuristicResult2 = unifiedMemoryHeuristic.apply(data2) + val evaluator = new Evaluator(unifiedMemoryHeuristic, data1) it("has severity") { heuristicResult.getSeverity should be(Severity.CRITICAL) } + + it("has max value") { + val details = heuristicResult.getHeuristicResultDetails.get(2) + details.getName should be("Max peak unified memory") + details.getValue should be("385.32 KB") + } + + it("has mean value") { + val details = heuristicResult.getHeuristicResultDetails.get(1) + details.getName should be("Mean peak unified memory") + details.getValue should be("263.07 KB") + } + + it("data1 has severity") { + heuristicResult1.getSeverity should be(Severity.CRITICAL) + } + + it("data1 has maxMemory") { + evaluator.maxMemory should be(999999999) + } + + it("data1 has max memory") { + evaluator.maxUnifiedMemory should be(400) + } + + it("data1 has mean memory") { + evaluator.meanUnifiedMemory should be(400) + } + + it("has no severity when max and allocated memory are the same") { + heuristicResult2.getSeverity should be(Severity.NONE) + } } } @@ -69,7 +122,10 @@ object UnifiedMemoryHeuristicTest { peakUnifiedMemory ) - def newFakeSparkApplicationData(executorSummaries: Seq[ExecutorSummaryImpl]): SparkApplicationData = { + def newFakeSparkApplicationData( + appConfigurationProperties: Map[String, String], + executorSummaries: Seq[ExecutorSummaryImpl]): SparkApplicationData = + { val appId = "application_1" val restDerivedData = SparkRestDerivedData( new ApplicationInfoImpl(appId, name = "app", Seq.empty), @@ -78,7 +134,9 @@ object UnifiedMemoryHeuristicTest { executorSummaries = executorSummaries, stagesWithFailedTasks = Seq.empty ) - - SparkApplicationData(appId, restDerivedData, logDerivedData = None) + val logDerivedData = SparkLogDerivedData( + SparkListenerEnvironmentUpdate(Map("Spark Properties" -> appConfigurationProperties.toSeq)) + ) + SparkApplicationData(appId, restDerivedData, Some(logDerivedData)) } } diff --git a/test/com/linkedin/drelephant/util/MemoryFormatUtilsTest.java b/test/com/linkedin/drelephant/util/MemoryFormatUtilsTest.java index 0ae064ebc..067f7427d 100644 --- a/test/com/linkedin/drelephant/util/MemoryFormatUtilsTest.java +++ b/test/com/linkedin/drelephant/util/MemoryFormatUtilsTest.java @@ -105,4 +105,16 @@ public void testStringToBytes() { } } } + + public void testRoundOffMemoryStringToNextInteger() { + assertEquals("157 MB", MemoryFormatUtils.roundOffMemoryStringToNextInteger("156.1 MB")); + assertEquals("155 MB", MemoryFormatUtils.roundOffMemoryStringToNextInteger("155.0 MB")); + assertEquals("0 MB", MemoryFormatUtils.roundOffMemoryStringToNextInteger("0 MB")); + assertEquals("156 GB", MemoryFormatUtils.roundOffMemoryStringToNextInteger("155.1 G")); + assertEquals("", MemoryFormatUtils.roundOffMemoryStringToNextInteger(null)); + assertEquals("500 MB", MemoryFormatUtils.roundOffMemoryStringToNextInteger("500M")); + assertEquals("600 GB", MemoryFormatUtils.roundOffMemoryStringToNextInteger("600 gb")); + assertEquals("600 GB", MemoryFormatUtils.roundOffMemoryStringToNextInteger("600 g")); + assertEquals("", MemoryFormatUtils.roundOffMemoryStringToNextInteger("")); + } } From e15343b883bb880cef4ead3024fd6b590a755daa Mon Sep 17 00:00:00 2001 From: skakker Date: Fri, 6 Apr 2018 16:46:17 +0530 Subject: [PATCH 08/15] Removing blocking keyword so as to prevent a large number of threads being spawned (#362) --- app/com/linkedin/drelephant/spark/fetchers/SparkLogClient.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/com/linkedin/drelephant/spark/fetchers/SparkLogClient.scala b/app/com/linkedin/drelephant/spark/fetchers/SparkLogClient.scala index 2a6464627..71023fab4 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/SparkLogClient.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/SparkLogClient.scala @@ -62,7 +62,7 @@ class SparkLogClient(hadoopConfiguration: Configuration, sparkConf: SparkConf, e val (eventLogPath, eventLogCodec) = sparkUtils.pathAndCodecforEventLog(sparkConf, eventLogFileSystem, baseEventLogPath, appId, attemptId) - Future { + Future { sparkUtils.withEventLog(eventLogFileSystem, eventLogPath, eventLogCodec)(findDerivedData(_)) } } From 8d7b64d5974448c73ec543318714f359d655f338 Mon Sep 17 00:00:00 2001 From: akrai Date: Thu, 14 Jun 2018 09:54:47 +0530 Subject: [PATCH 09/15] Revert "Dr. Elephant Tez Support working patch (#313)" This reverts commit a0470a3e6874527291bcc546a39cce28fc9cdd3e. --- app-conf/AggregatorConf.xml | 4 - app-conf/FetcherConf.xml | 7 - app-conf/HeuristicConf.xml | 117 ------ app-conf/JobTypeConf.xml | 6 - .../fetchers/MapReduceFetcherHadoop2.java | 84 +++- .../drelephant/tez/TezMetricsAggregator.java | 109 ----- .../tez/TezTaskLevelAggregatedMetrics.java | 154 ------- .../tez/data/TezApplicationData.java | 137 ------ .../drelephant/tez/data/TezCounterData.java | 195 --------- .../drelephant/tez/data/TezTaskData.java | 137 ------ .../drelephant/tez/fetchers/TezFetcher.java | 391 ------------------ .../heuristics/GenericDataSkewHeuristic.java | 215 ---------- .../tez/heuristics/GenericGCHeuristic.java | 142 ------- .../heuristics/GenericMemoryHeuristic.java | 185 --------- .../heuristics/MapperDataSkewHeuristic.java | 47 --- .../tez/heuristics/MapperGCHeuristic.java | 51 --- .../tez/heuristics/MapperMemoryHeuristic.java | 45 -- .../tez/heuristics/MapperSpeedHeuristic.java | 165 -------- .../tez/heuristics/MapperSpillHeuristic.java | 142 ------- .../tez/heuristics/MapperTimeHeuristic.java | 182 -------- .../heuristics/ReducerDataSkewHeuristic.java | 42 -- .../tez/heuristics/ReducerGCHeuristic.java | 41 -- .../heuristics/ReducerMemoryHeuristic.java | 47 --- .../tez/heuristics/ReducerTimeHeuristic.java | 167 -------- .../drelephant/util/InfoExtractor.java | 11 - .../drelephant/util/ThreadContextMR2.java | 92 ----- .../fetchers/MapReduceFetcherHadoop2Test.java | 2 - .../TezTaskLevelAggregatedMetricsTest.java | 62 --- .../tez/fetchers/TezFetcherTest.java | 35 -- .../MapperDataSkewHeuristicTest.java | 146 ------- .../tez/heuristics/MapperGCHeuristicTest.java | 84 ---- .../heuristics/MapperMemoryHeuristicTest.java | 94 ----- .../heuristics/MapperSpeedHeuristicTest.java | 100 ----- .../heuristics/MapperSpillHeuristicTest.java | 90 ---- .../heuristics/MapperTimeHeuristicTest.java | 111 ----- .../ReducerDataSkewHeuristicTest.java | 142 ------- .../heuristics/ReducerGCHeuristicTest.java | 83 ---- .../ReducerMemoryHeuristicTest.java | 93 ----- .../heuristics/ReducerTimeHeuristicTest.java | 97 ----- 39 files changed, 83 insertions(+), 3971 deletions(-) delete mode 100644 app/com/linkedin/drelephant/tez/TezMetricsAggregator.java delete mode 100644 app/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetrics.java delete mode 100644 app/com/linkedin/drelephant/tez/data/TezApplicationData.java delete mode 100644 app/com/linkedin/drelephant/tez/data/TezCounterData.java delete mode 100644 app/com/linkedin/drelephant/tez/data/TezTaskData.java delete mode 100644 app/com/linkedin/drelephant/tez/fetchers/TezFetcher.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/GenericDataSkewHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/GenericGCHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java delete mode 100644 app/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristic.java delete mode 100644 app/com/linkedin/drelephant/util/ThreadContextMR2.java delete mode 100644 test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java delete mode 100644 test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java delete mode 100644 test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java diff --git a/app-conf/AggregatorConf.xml b/app-conf/AggregatorConf.xml index 1536b2d96..23586d587 100644 --- a/app-conf/AggregatorConf.xml +++ b/app-conf/AggregatorConf.xml @@ -33,10 +33,6 @@ mapreduce com.linkedin.drelephant.mapreduce.MapReduceMetricsAggregator - - tez - com.linkedin.drelephant.tez.TezMetricsAggregator - spark com.linkedin.drelephant.spark.SparkMetricsAggregator diff --git a/app-conf/FetcherConf.xml b/app-conf/FetcherConf.xml index 89dafc887..1f813cf15 100644 --- a/app-conf/FetcherConf.xml +++ b/app-conf/FetcherConf.xml @@ -29,13 +29,6 @@ --> - - - tez - com.linkedin.drelephant.tez.fetchers.TezFetcher - - - - - tez - Mapper Data Skew - com.linkedin.drelephant.tez.heuristics.MapperDataSkewHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Mapper GC - com.linkedin.drelephant.tez.heuristics.MapperGCHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Mapper Time - com.linkedin.drelephant.tez.heuristics.MapperTimeHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Mapper Speed - com.linkedin.drelephant.tez.heuristics.MapperSpeedHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Mapper Memory - com.linkedin.drelephant.tez.heuristics.MapperMemoryHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Mapper Spill - com.linkedin.drelephant.tez.heuristics.MapperSpillHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Reducer Data Skew - com.linkedin.drelephant.tez.heuristics.ReducerDataSkewHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Reducer GC - com.linkedin.drelephant.tez.heuristics.ReducerGCHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Reducer Time - com.linkedin.drelephant.tez.heuristics.ReducerTimeHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - tez - Reducer Memory - com.linkedin.drelephant.tez.heuristics.ReducerMemoryHeuristic - views.html.help.mapreduce.helpMapperSpill - - - diff --git a/app-conf/JobTypeConf.xml b/app-conf/JobTypeConf.xml index 9dc4aa5ba..8a4cae3eb 100644 --- a/app-conf/JobTypeConf.xml +++ b/app-conf/JobTypeConf.xml @@ -42,12 +42,6 @@ mapreduce pig.script - - Tez - tez - hive.mapred.mode - - Hive mapreduce diff --git a/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2.java b/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2.java index 5a8a64425..4165971aa 100644 --- a/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2.java +++ b/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2.java @@ -20,23 +20,30 @@ import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; +import com.linkedin.drelephant.math.Statistics; import com.linkedin.drelephant.configurations.fetcher.FetcherConfigurationData; -import com.linkedin.drelephant.util.ThreadContextMR2; import com.linkedin.drelephant.util.Utils; import java.io.IOException; +import java.lang.Integer; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.List; import java.util.Properties; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.log4j.Logger; import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; /** @@ -380,3 +387,78 @@ private JsonNode getTaskFirstFailedAttempt(URL taskAllAttemptsUrl) throws IOExce } } } + +final class ThreadContextMR2 { + private static final Logger logger = Logger.getLogger(ThreadContextMR2.class); + private static final AtomicInteger THREAD_ID = new AtomicInteger(1); + + private static final ThreadLocal _LOCAL_THREAD_ID = new ThreadLocal() { + @Override + public Integer initialValue() { + return THREAD_ID.getAndIncrement(); + } + }; + + private static final ThreadLocal _LOCAL_LAST_UPDATED = new ThreadLocal(); + private static final ThreadLocal _LOCAL_UPDATE_INTERVAL = new ThreadLocal(); + + private static final ThreadLocal _LOCAL_DIAGNOSTIC_PATTERN = new ThreadLocal() { + @Override + public Pattern initialValue() { + // Example: "Task task_1443068695259_9143_m_000475 failed 1 times" + return Pattern.compile( + ".*[\\s\\u00A0]+(task_[0-9]+_[0-9]+_[m|r]_[0-9]+)[\\s\\u00A0]+.*"); + } + }; + + private static final ThreadLocal _LOCAL_AUTH_TOKEN = + new ThreadLocal() { + @Override + public AuthenticatedURL.Token initialValue() { + _LOCAL_LAST_UPDATED.set(System.currentTimeMillis()); + // Random an interval for each executor to avoid update token at the same time + _LOCAL_UPDATE_INTERVAL.set(Statistics.MINUTE_IN_MS * 30 + new Random().nextLong() + % (3 * Statistics.MINUTE_IN_MS)); + logger.info("Executor " + _LOCAL_THREAD_ID.get() + " update interval " + _LOCAL_UPDATE_INTERVAL.get() * 1.0 + / Statistics.MINUTE_IN_MS); + return new AuthenticatedURL.Token(); + } + }; + + private static final ThreadLocal _LOCAL_AUTH_URL = new ThreadLocal() { + @Override + public AuthenticatedURL initialValue() { + return new AuthenticatedURL(); + } + }; + + private static final ThreadLocal _LOCAL_MAPPER = new ThreadLocal() { + @Override + public ObjectMapper initialValue() { + return new ObjectMapper(); + } + }; + + private ThreadContextMR2() { + // Empty on purpose + } + + public static Matcher getDiagnosticMatcher(String diagnosticInfo) { + return _LOCAL_DIAGNOSTIC_PATTERN.get().matcher(diagnosticInfo); + } + + public static JsonNode readJsonNode(URL url) throws IOException, AuthenticationException { + HttpURLConnection conn = _LOCAL_AUTH_URL.get().openConnection(url, _LOCAL_AUTH_TOKEN.get()); + return _LOCAL_MAPPER.get().readTree(conn.getInputStream()); + } + + public static void updateAuthToken() { + long curTime = System.currentTimeMillis(); + if (curTime - _LOCAL_LAST_UPDATED.get() > _LOCAL_UPDATE_INTERVAL.get()) { + logger.info("Executor " + _LOCAL_THREAD_ID.get() + " updates its AuthenticatedToken."); + _LOCAL_AUTH_TOKEN.set(new AuthenticatedURL.Token()); + _LOCAL_AUTH_URL.set(new AuthenticatedURL()); + _LOCAL_LAST_UPDATED.set(curTime); + } + } +} diff --git a/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java b/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java deleted file mode 100644 index c04edd7fd..000000000 --- a/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez; - -import com.linkedin.drelephant.analysis.*; -import com.linkedin.drelephant.configurations.aggregator.AggregatorConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; - -/** - * Aggregates task level metrics to application - */ - -public class TezMetricsAggregator implements HadoopMetricsAggregator { - - private static final Logger logger = Logger.getLogger(TezMetricsAggregator.class); - - private static final String TEZ_CONTAINER_CONFIG = "hive.tez.container.size"; - private static final String MAP_CONTAINER_CONFIG = "mapreduce.map.memory.mb"; - private static final String REDUCER_CONTAINER_CONFIG = "mapreduce.reduce.memory.mb"; - private static final String REDUCER_SLOW_START_CONFIG = "mapreduce.job.reduce.slowstart.completedmaps"; - private static final long CONTAINER_MEMORY_DEFAULT_BYTES = 2048L * FileUtils.ONE_MB; - - private HadoopAggregatedData _hadoopAggregatedData = null; - private TezTaskLevelAggregatedMetrics _mapTasks; - private TezTaskLevelAggregatedMetrics _reduceTasks; - - private AggregatorConfigurationData _aggregatorConfigurationData; - - public TezMetricsAggregator(AggregatorConfigurationData _aggregatorConfigurationData) { - this._aggregatorConfigurationData = _aggregatorConfigurationData; - _hadoopAggregatedData = new HadoopAggregatedData(); - } - - @Override - public void aggregate(HadoopApplicationData hadoopData) { - - TezApplicationData data = (TezApplicationData) hadoopData; - - long mapTaskContainerSize = getMapContainerSize(data); - long reduceTaskContainerSize = getReducerContainerSize(data); - - int reduceTaskSlowStartPercentage = - (int) (Double.parseDouble(data.getConf().getProperty(REDUCER_SLOW_START_CONFIG)) * 100); - - - //overwrite reduceTaskSlowStartPercentage to 100%. TODO: make use of the slow start percent - reduceTaskSlowStartPercentage = 100; - - _mapTasks = new TezTaskLevelAggregatedMetrics(data.getMapTaskData(), mapTaskContainerSize, data.getStartTime()); - - long reduceIdealStartTime = _mapTasks.getNthPercentileFinishTime(reduceTaskSlowStartPercentage); - - // Mappers list is empty - if(reduceIdealStartTime == -1) { - // ideal start time for reducer is infinite since it cannot start - reduceIdealStartTime = Long.MAX_VALUE; - } - - _reduceTasks = new TezTaskLevelAggregatedMetrics(data.getReduceTaskData(), reduceTaskContainerSize, reduceIdealStartTime); - - _hadoopAggregatedData.setResourceUsed(_mapTasks.getResourceUsed() + _reduceTasks.getResourceUsed()); - _hadoopAggregatedData.setTotalDelay(_mapTasks.getDelay() + _reduceTasks.getDelay()); - _hadoopAggregatedData.setResourceWasted(_mapTasks.getResourceWasted() + _reduceTasks.getResourceWasted()); - } - - @Override - public HadoopAggregatedData getResult() { - return _hadoopAggregatedData; - } - - private long getMapContainerSize(HadoopApplicationData data) { - try { - long mapContainerSize = Long.parseLong(data.getConf().getProperty(TEZ_CONTAINER_CONFIG)); - if (mapContainerSize > 0) - return mapContainerSize; - else - return Long.parseLong(data.getConf().getProperty(MAP_CONTAINER_CONFIG)); - } catch ( NumberFormatException ex) { - return CONTAINER_MEMORY_DEFAULT_BYTES; - } - } - - private long getReducerContainerSize(HadoopApplicationData data) { - try { - long reducerContainerSize = Long.parseLong(data.getConf().getProperty(TEZ_CONTAINER_CONFIG)); - if (reducerContainerSize > 0) - return reducerContainerSize; - else - return Long.parseLong(data.getConf().getProperty(REDUCER_CONTAINER_CONFIG)); - } catch ( NumberFormatException ex) { - return CONTAINER_MEMORY_DEFAULT_BYTES; - } - } -} diff --git a/app/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetrics.java b/app/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetrics.java deleted file mode 100644 index 0f5eaa5f6..000000000 --- a/app/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetrics.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez; - -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.math.Statistics; -import java.util.ArrayList; -import java.util.List; -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; - -/** - * Aggregation functionality for task level metrics - */ - -public class TezTaskLevelAggregatedMetrics { - - private static final Logger logger = Logger.getLogger(TezTaskLevelAggregatedMetrics.class); - - private long _delay = 0; - private long _resourceWasted = 0; - private long _resourceUsed = 0; - - private List finishTimes = new ArrayList(); - private List durations = new ArrayList(); - - private static final double MEMORY_BUFFER = 1.5; - private static final double CLUSTER_MEMORY_FACTOR = 2.1; - - /** - * Returns the nth percentile finish job - * @param percentile The percentile of finish job to return - * @return The nth percentile finish job - */ - public long getNthPercentileFinishTime(int percentile) - { - if(finishTimes == null || finishTimes.size() == 0 ) { - return -1; - } - return Statistics.percentile(finishTimes, percentile); - } - - /** - * Constructor for TaskLevelAggregatedMetrics - * @param taskData Array containing the task data for mappers and/or reducers - * @param containerSize The container size of the tasks - * @param idealStartTime The ideal start time for the task. For mappers it is the submit time, for - * reducers, it is the time when the number of completed maps become more than - * the slow start time. - */ - public TezTaskLevelAggregatedMetrics(TezTaskData[] taskData, long containerSize, long idealStartTime) { - compute(taskData, containerSize, idealStartTime); - } - - /** - * Returns the overall delay for the tasks. - * @return The delay of the tasks. - */ - public long getDelay() { - return _delay; - } - - /** - * Retruns the resources wasted by all the tasks in MB Seconds - * @return The wasted resources of all the tasks in MB Seconds - */ - public long getResourceWasted() { - return _resourceWasted; - } - - /** - * Returns the resource used by all the tasks in MB Seconds - * @return The total resources used by all tasks in MB Seconds - */ - public long getResourceUsed() { - return _resourceUsed; - } - - /** - * Computes the aggregated metrics -> peakMemory, delay, total task duration, wasted resources and memory usage. - * @param taskDatas - * @param containerSize - * @param idealStartTime - */ - private void compute(TezTaskData[] taskDatas, long containerSize, long idealStartTime) { - - long peakMemoryNeed = 0; - long taskFinishTimeMax = 0; - long taskDurationMax = 0; - - // if there are zero tasks, then nothing to compute. - if(taskDatas == null || taskDatas.length == 0) { - return; - } - - for (TezTaskData taskData: taskDatas) { - if (!taskData.isSampled()) { - continue; - } - long taskMemory = taskData.getCounters().get(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES)/ FileUtils.ONE_MB; // MB - long taskVM = taskData.getCounters().get(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES)/ FileUtils.ONE_MB; // MB - long taskDuration = taskData.getFinishTime() - taskData.getStartTime(); // Milliseconds - long taskCost = (containerSize) * (taskDuration / Statistics.SECOND_IN_MS); // MB Seconds - - durations.add(taskDuration); - finishTimes.add(taskData.getFinishTime()); - - //peak Memory usage - long memoryRequiredForVM = (long) (taskVM/CLUSTER_MEMORY_FACTOR); - long biggerMemoryRequirement = memoryRequiredForVM > taskMemory ? memoryRequiredForVM : taskMemory; - peakMemoryNeed = biggerMemoryRequirement > peakMemoryNeed ? biggerMemoryRequirement : peakMemoryNeed; - - if(taskFinishTimeMax < taskData.getFinishTime()) { - taskFinishTimeMax = taskData.getFinishTime(); - } - - if(taskDurationMax < taskDuration) { - taskDurationMax = taskDuration; - } - _resourceUsed += taskCost; - } - - // Compute the delay in starting the task. - _delay = taskFinishTimeMax - (idealStartTime + taskDurationMax); - - // invalid delay - if(_delay < 0) { - _delay = 0; - } - - // wastedResources - long wastedMemory = containerSize - (long) (peakMemoryNeed * MEMORY_BUFFER); - if(wastedMemory > 0) { - for (long duration : durations) { - _resourceWasted += (wastedMemory) * (duration / Statistics.SECOND_IN_MS); // MB Seconds - } - } - } - -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/data/TezApplicationData.java b/app/com/linkedin/drelephant/tez/data/TezApplicationData.java deleted file mode 100644 index f38f527bc..000000000 --- a/app/com/linkedin/drelephant/tez/data/TezApplicationData.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.data; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HadoopApplicationData; - -import java.util.Properties; - -/** - * Tez Application level data structure which hold all task data - */ -public class TezApplicationData implements HadoopApplicationData { - - private static final ApplicationType APPLICATION_TYPE = new ApplicationType("TEZ"); - - private String _appId = ""; - private Properties _conf; - private boolean _succeeded = true; - private TezTaskData[] _reduceTasks; - private TezTaskData[] _mapTasks; - private TezCounterData _counterHolder; - - private long _submitTime = 0; - private long _startTime = 0; - private long _finishTime = 0; - - public boolean getSucceeded() { - return _succeeded; - } - - @Override - public String getAppId() { - return _appId; - } - - @Override - public Properties getConf() { - return _conf; - } - - @Override - public ApplicationType getApplicationType() { - return APPLICATION_TYPE; - } - - @Override - public boolean isEmpty() { - return _succeeded && getMapTaskData().length == 0 && getReduceTaskData().length == 0; - } - - public TezTaskData[] getReduceTaskData() { - return _reduceTasks; - } - - public TezTaskData[] getMapTaskData() { - return _mapTasks; - } - - public long getSubmitTime() { - return _submitTime; - } - - public long getStartTime() { - return _startTime; - } - - public long getFinishTime() { - return _finishTime; - } - - public TezCounterData getCounters() { - return _counterHolder; - } - - public TezApplicationData setCounters(TezCounterData counterHolder) { - this._counterHolder = counterHolder; - return this; - } - - public TezApplicationData setAppId(String appId) { - this._appId = appId; - return this; - } - - public TezApplicationData setConf(Properties conf) { - this._conf = conf; - return this; - } - - public TezApplicationData setSucceeded(boolean succeeded) { - this._succeeded = succeeded; - return this; - } - - public TezApplicationData setReduceTaskData(TezTaskData[] reduceTasks) { - this._reduceTasks = reduceTasks; - return this; - } - - public TezApplicationData setMapTaskData(TezTaskData[] mapTasks) { - this._mapTasks = mapTasks; - return this; - } - - public TezApplicationData setSubmitTime(long submitTime) { - this._submitTime = submitTime; - return this; - } - - public TezApplicationData setStartTime(long startTime) { - this._startTime = startTime; - return this; - } - - public TezApplicationData setFinishTime(long finishTime) { - this._finishTime = finishTime; - return this; - } - - public String toString(){ - return APPLICATION_TYPE.toString() + " " + _appId; - } -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/data/TezCounterData.java b/app/com/linkedin/drelephant/tez/data/TezCounterData.java deleted file mode 100644 index 9aac00117..000000000 --- a/app/com/linkedin/drelephant/tez/data/TezCounterData.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.data; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -/** - * Tez Counter Data defining data structure. - */ -public class TezCounterData { - // Map to group counters into DAG, Task and application levels. - private final Map> _pubCounters; - - public String toString() { - return _pubCounters.toString(); - } - - public TezCounterData() { - _pubCounters = new HashMap>(8); - } - - public long get(CounterName counterName) { - for(Map counterGrp : _pubCounters.values()) { - if(counterGrp.containsKey(counterName._name)) { - return counterGrp.get(counterName._name); - } - } - return 0; - } - - public void set(CounterName counterName, long value) { - set(counterName.getGroupName(), counterName.getName(), value); - } - - public void set(String groupName, String counterName, long value) { - Map counterMap = _pubCounters.get(groupName); - if (counterMap == null) { - counterMap = new HashMap(4); - _pubCounters.put(groupName, counterMap); - } - counterMap.put(counterName, value); - } - - public Set getGroupNames() { - Set groupNames = _pubCounters.keySet(); - return Collections.unmodifiableSet(groupNames); - } - - public Map getAllCountersInGroup(String groupName) { - Map counterMap = _pubCounters.get(groupName); - if (counterMap == null) { - counterMap = new HashMap(1); - } - return counterMap; - } - - public static enum GroupName { - FileSystemCounters("org.apache.tez.common.counters.FileSystemCounter"), - TezTask("org.apache.tez.common.counters.TaskCounter"), - TezDag("org.apache.tez.common.counters.DAGCounter"); - - - String _name; - GroupName(String name) { - _name = name; - } - } - - public static enum CounterName { - - NUM_SUCCEEDED_TASKS(GroupName.TezDag, "NUM_SUCCEEDED_TASKS", "NUM_SUCCEEDED_TASKS"), - TOTAL_LAUNCHED_TASKS(GroupName.TezDag, "TOTAL_LAUNCHED_TASKS", "TOTAL_LAUNCHED_TASKS"), - RACK_LOCAL_TASKS(GroupName.TezDag, "RACK_LOCAL_TASKS", "RACK_LOCAL_TASKS"), - AM_CPU_MILLISECONDS(GroupName.TezDag, "AM_CPU_MILLISECONDS", "AM_CPU_MILLISECONDS"), - AM_GC_TIME_MILLIS(GroupName.TezDag, "AM_GC_TIME_MILLIS", "AM_GC_TIME_MILLIS"), - - FILE_BYTES_READ(GroupName.FileSystemCounters, "FILE_BYTES_READ", "FILE_BYTES_READ"), - FILE_BYTES_WRITTEN(GroupName.FileSystemCounters, "FILE_BYTES_WRITTEN", "FILE_BYTES_WRITTEN"), - FILE_READ_OPS(GroupName.FileSystemCounters, "FILE_READ_OPS", "FILE_READ_OPS"), - FILE_LARGE_READ_OPS(GroupName.FileSystemCounters, "FILE_LARGE_READ_OPS", "FILE_LARGE_READ_OPS"), - FILE_WRITE_OPS(GroupName.FileSystemCounters, "FILE_WRITE_OPS", "FILE_WRITE_OPS"), - HDFS_BYTES_READ(GroupName.FileSystemCounters, "HDFS_BYTES_READ", "HDFS_BYTES_READ"), - HDFS_BYTES_WRITTEN(GroupName.FileSystemCounters, "HDFS_BYTES_WRITTEN", "HDFS_BYTES_WRITTEN"), - HDFS_READ_OPS(GroupName.FileSystemCounters, "HDFS_READ_OPS", "HDFS_READ_OPS"), - HDFS_LARGE_READ_OPS(GroupName.FileSystemCounters, "HDFS_LARGE_READ_OPS", "HDFS_LARGE_READ_OPS"), - HDFS_WRITE_OPS(GroupName.FileSystemCounters, "HDFS_WRITE_OPS", "HDFS_WRITE_OPS"), - S3A_BYTES_READ(GroupName.FileSystemCounters, "S3A_BYTES_READ", "S3A_BYTES_READ"), - S3A_BYTES_WRITTEN(GroupName.FileSystemCounters, "S3A_BYTES_WRITTEN", "S3A_BYTES_WRITTEN"), - S3A_READ_OPS(GroupName.FileSystemCounters, "S3A_READ_OPS", "S3A_READ_OPS"), - S3A_LARGE_READ_OPS(GroupName.FileSystemCounters, "S3A_LARGE_READ_OPS", "S3A_LARGE_READ_OPS"), - S3A_WRITE_OPS(GroupName.FileSystemCounters, "S3A_WRITE_OPS", "S3_WRITE_OPS"), - S3N_BYTES_READ(GroupName.FileSystemCounters, "S3N_BYTES_READ", "S3N_BYTES_READ"), - S3N_BYTES_WRITTEN(GroupName.FileSystemCounters, "S3N_BYTES_WRITTEN", "S3N_BYTES_WRITTEN"), - S3N_READ_OPS(GroupName.FileSystemCounters, "S3N_READ_OPS", "S3N_READ_OPS"), - S3N_LARGE_READ_OPS(GroupName.FileSystemCounters, "S3N_LARGE_READ_OPS", "S3N_LARGE_READ_OPS"), - S3N_WRITE_OPS(GroupName.FileSystemCounters, "S3N_WRITE_OPS", "S3N_WRITE_OPS"), - - REDUCE_INPUT_GROUPS(GroupName.TezTask, "REDUCE_INPUT_GROUPS", "REDUCE_INPUT_GROUPS"), - REDUCE_INPUT_RECORDS(GroupName.TezTask, "REDUCE_INPUT_RECORDS", "REDUCE_INPUT_RECORDS"), - COMBINE_INPUT_RECORDS(GroupName.TezTask, "COMBINE_INPUT_RECORDS", "COMBINE_INPUT_RECORDS"), - SPILLED_RECORDS(GroupName.TezTask, "SPILLED_RECORDS", "SPILLED_RECORDS"), - NUM_SHUFFLED_INPUTS(GroupName.TezTask, "NUM_SHUFFLED_INPUTS", "NUM_SHUFFLED_INPUTS"), - NUM_SKIPPED_INPUTS(GroupName.TezTask, "NUM_SKIPPED_INPUTS", "NUM_SKIPPED_INPUTS"), - NUM_FAILED_SHUFFLE_INPUTS(GroupName.TezTask, "NUM_FAILED_SHUFFLE_INPUTS", "NUM_FAILED_SHUFFLE_INPUTS"), - MERGED_MAP_OUTPUTS(GroupName.TezTask, "MERGED_MAP_OUTPUTS", "MERGED_MAP_OUTPUTS"), - GC_TIME_MILLIS(GroupName.TezTask, "GC_TIME_MILLIS", "GC_TIME_MILLIS"), - COMMITTED_HEAP_BYTES(GroupName.TezTask, "COMMITTED_HEAP_BYTES", "COMMITTED_HEAP_BYTES"), - INPUT_RECORDS_PROCESSED(GroupName.TezTask, "INPUT_RECORDS_PROCESSED", "INPUT_RECORDS_PROCESSED"), - OUTPUT_RECORDS(GroupName.TezTask, "OUTPUT_RECORDS", "OUTPUT_RECORDS"), - OUTPUT_BYTES(GroupName.TezTask, "OUTPUT_BYTES", "OUTPUT_BYTES"), - OUTPUT_BYTES_WITH_OVERHEAD(GroupName.TezTask, "OUTPUT_BYTES_WITH_OVERHEAD", "OUTPUT_BYTES_WITH_OVERHEAD"), - OUTPUT_BYTES_PHYSICAL(GroupName.TezTask, "OUTPUT_BYTES_PHYSICAL", "OUTPUT_BYTES_PHYSICAL"), - ADDITIONAL_SPILLS_BYTES_WRITTEN(GroupName.TezTask, "ADDITIONAL_SPILLS_BYTES_WRITTEN", "ADDITIONAL_SPILLS_BYTES_WRITTEN"), - ADDITIONAL_SPILLS_BYTES_READ(GroupName.TezTask, "ADDITIONAL_SPILLS_BYTES_READ", "ADDITIONAL_SPILLS_BYTES_READ"), - ADDITIONAL_SPILL_COUNT(GroupName.TezTask, "ADDITIONAL_SPILL_COUNT", "ADDITIONAL_SPILL_COUNT"), - SHUFFLE_BYTES(GroupName.TezTask, "SHUFFLE_BYTES", "SHUFFLE_BYTES"), - SHUFFLE_BYTES_DECOMPRESSED(GroupName.TezTask, "SHUFFLE_BYTES_DECOMPRESSED", "SHUFFLE_BYTES_DECOMPRESSED"), - SHUFFLE_BYTES_TO_MEM(GroupName.TezTask, "SHUFFLE_BYTES_TO_MEM", "SHUFFLE_BYTES_TO_MEM"), - SHUFFLE_BYTES_TO_DISK(GroupName.TezTask, "SHUFFLE_BYTES_TO_DISK", "SHUFFLE_BYTES_TO_DISK"), - SHUFFLE_BYTES_DISK_DIRECT(GroupName.TezTask, "SHUFFLE_BYTES_DISK_DIRECT", "SHUFFLE_BYTES_DISK_DIRECT"), - NUM_MEM_TO_DISK_MERGES(GroupName.TezTask, "NUM_MEM_TO_DISK_MERGES", "NUM_MEM_TO_DISK_MERGES"), - CPU_MILLISECONDS(GroupName.TezTask,"CPU_MILLISECONDS","CPU_MILLISECONDS"), - PHYSICAL_MEMORY_BYTES(GroupName.TezTask,"PHYSICAL_MEMORY_BYTES","PHYSICAL_MEMORY_BYTES"), - VIRTUAL_MEMORY_BYTES(GroupName.TezTask,"VIRTUAL_MEMORY_BYTES","VIRTUAL_MEMORY_BYTES"), - NUM_DISK_TO_DISK_MERGES(GroupName.TezTask, "NUM_DISK_TO_DISK_MERGES", "NUM_DISK_TO_DISK_MERGES"), - SHUFFLE_PHASE_TIME(GroupName.TezTask, "SHUFFLE_PHASE_TIME", "SHUFFLE_PHASE_TIME"), - MERGE_PHASE_TIME(GroupName.TezTask, "MERGE_PHASE_TIME", "MERGE_PHASE_TIME"), - FIRST_EVENT_RECEIVED(GroupName.TezTask, "FIRST_EVENT_RECEIVED", "FIRST_EVENT_RECEIVED"), - LAST_EVENT_RECEIVED(GroupName.TezTask, "LAST_EVENT_RECEIVED", "LAST_EVENT_RECEIVED"); - - - GroupName _group; - String _name; - String _displayName; - - CounterName(GroupName group, String name, String displayName) { - this._group = group; - this._name = name; - this._displayName = displayName; - } - - static Map _counterDisplayNameMap; - static Map _counterNameMap; - static { - _counterDisplayNameMap = new HashMap(); - _counterNameMap = new HashMap(); - for (CounterName cn : CounterName.values()) { - _counterDisplayNameMap.put(cn._displayName, cn); - _counterNameMap.put(cn._name, cn); - } - } - - public static CounterName getCounterFromName(String name) { - if (_counterNameMap.containsKey(name)) { - return _counterNameMap.get(name); - } - return null; - } - - public static CounterName getCounterFromDisplayName(String displayName) { - if (_counterDisplayNameMap.containsKey(displayName)) { - return _counterDisplayNameMap.get(displayName); - } - return null; - } - - public String getName() { - return _name; - } - - public String getDisplayName() { - return _displayName; - } - - public String getGroupName() { - return _group.name(); - } - } -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/data/TezTaskData.java b/app/com/linkedin/drelephant/tez/data/TezTaskData.java deleted file mode 100644 index eb6aa832c..000000000 --- a/app/com/linkedin/drelephant/tez/data/TezTaskData.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.data; - -/** - * Tez Task Level metadata holding data structure - */ - -public class TezTaskData { - private TezCounterData _counterHolder; - private String _taskId; - // The successful attempt id - private String _attemptId; - - private long _totalTimeMs = 0; - private long _shuffleTimeMs = 0; - private long _sortTimeMs = 0; - private long _startTime = 0; - private long _finishTime = 0; - private boolean _isSampled = false; - - //Constructor used only in Test Cases , if provided to partially assign needed time values while ignoring the others - public TezTaskData(TezCounterData counterHolder, long[] time) { - if(time == null || time.length<3){ - time = new long[5]; - } - this._counterHolder = counterHolder; - this._totalTimeMs = time[0]; - this._shuffleTimeMs = time[1]; - this._sortTimeMs = time[2]; - if (time.length > 3) - this._startTime = time[3]; - if (time.length > 4) - this._finishTime = time[4]; - this._isSampled = true; - } - - public TezTaskData(TezCounterData counterHolder) { - this._counterHolder = counterHolder; - } - - public TezTaskData(String taskId, String taskAttemptId) { - this._taskId = taskId; - this._attemptId = taskAttemptId; - } - - public void setCounter(TezCounterData counterHolder) { - this._counterHolder = counterHolder; - this._isSampled = true; - } - - public void setTime(long[] time) { - //No Validation needed here as time array will always be of fixed length 5 from upstream methods. - this._totalTimeMs = time[0]; - this._shuffleTimeMs = time[1]; - this._sortTimeMs = time[2]; - this._startTime = time[3]; - this._finishTime = time[4]; - this._isSampled = true; - } - - //Used only in Test Cases - public void setTimeAndCounter(long[] time, TezCounterData counterHolder){ - if(time == null || time.length<3){ - time = new long[5]; - } - this._totalTimeMs = time[0]; - this._shuffleTimeMs = time[1]; - this._sortTimeMs = time[2]; - if (time.length > 3) - this._startTime = time[3]; - if (time.length > 4) - this._finishTime = time[4]; - this._isSampled = true; - this._counterHolder = counterHolder; - - - } - - public TezCounterData getCounters() { - return _counterHolder; - } - - public long getTotalRunTimeMs() { - return _totalTimeMs; - } - - public long getCodeExecutionTimeMs() { - return _totalTimeMs - _shuffleTimeMs - _sortTimeMs; - } - - public long getShuffleTimeMs() { - return _shuffleTimeMs; - } - - public long getSortTimeMs() { - return _sortTimeMs; - } - - public boolean isSampled() { - return _isSampled; - } - - public String getTaskId() { - return _taskId; - } - - public String getAttemptId() { - return _attemptId; - } - - public long getStartTime() { - return _startTime; - } - - public long getFinishTime() { - return _finishTime; - } - - public void setTotalTimeMs(long totalTimeMs, boolean isSampled) { - this._totalTimeMs = totalTimeMs; - this._isSampled = isSampled; - } -} diff --git a/app/com/linkedin/drelephant/tez/fetchers/TezFetcher.java b/app/com/linkedin/drelephant/tez/fetchers/TezFetcher.java deleted file mode 100644 index 294fe20e0..000000000 --- a/app/com/linkedin/drelephant/tez/fetchers/TezFetcher.java +++ /dev/null @@ -1,391 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.fetchers; - -import com.linkedin.drelephant.analysis.AnalyticJob; -import com.linkedin.drelephant.analysis.ElephantFetcher; -import com.linkedin.drelephant.configurations.fetcher.FetcherConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.util.ThreadContextMR2; -import org.apache.log4j.Logger; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.codehaus.jackson.JsonNode; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLConnection; -import java.util.*; - -/** - * Task level data mining for Tez Tasks from timeline server API - */ - -public class TezFetcher implements ElephantFetcher { - - private static final Logger logger = Logger.getLogger(TezFetcher.class); - - private static final String TIMELINE_SERVER_URL = "yarn.timeline-service.webapp.address"; - - private URLFactory _urlFactory; - private JSONFactory _jsonFactory; - private String _timelineWebAddr; - - private FetcherConfigurationData _fetcherConfigurationData; - - public TezFetcher(FetcherConfigurationData fetcherConfData) throws IOException { - this._fetcherConfigurationData = fetcherConfData; - final String applicationHistoryAddr = new Configuration().get(TIMELINE_SERVER_URL); - - //Connection validity checked using method verifyURL(_timelineWebAddr) inside URLFactory constructor; - _urlFactory = new URLFactory(applicationHistoryAddr); - logger.info("Connection success."); - - _jsonFactory = new JSONFactory(); - _timelineWebAddr = "http://" + _timelineWebAddr + "/ws/v1/timeline/"; - - } - - public TezApplicationData fetchData(AnalyticJob analyticJob) throws IOException, AuthenticationException { - - int maxSize = 0; - String appId = analyticJob.getAppId(); - TezApplicationData jobData = new TezApplicationData(); - jobData.setAppId(appId); - Properties jobConf = _jsonFactory.getProperties(_urlFactory.getApplicationURL(appId)); - jobData.setConf(jobConf); - URL dagIdsUrl = _urlFactory.getDagURLByTezApplicationId(appId); - - List dagIdsByApplicationId = _jsonFactory.getDagIdsByApplicationId(dagIdsUrl); - - List mapperListAggregate = new ArrayList(); - List reducerListAggregate = new ArrayList(); - - //Iterate over dagIds and choose the dagId with the highest no. of tasks/highest impact as settings changes can be made only at DAG level. - for(String dagId : dagIdsByApplicationId){ - try { - //set job task independent properties - - URL dagUrl = _urlFactory.getDagURL(dagId); - String state = _jsonFactory.getState(dagUrl); - - jobData.setStartTime(_jsonFactory.getDagStartTime(dagUrl)); - jobData.setFinishTime(_jsonFactory.getDagEndTime(dagUrl)); - - if (state.equals("SUCCEEDED")) { - jobData.setSucceeded(true); - - List mapperList = new ArrayList(); - List reducerList = new ArrayList(); - - // Fetch task data - URL vertexListUrl = _urlFactory.getVertexListURL(dagId); - _jsonFactory.getTaskDataAll(vertexListUrl, dagId, mapperList, reducerList); - - if(mapperList.size() + reducerList.size() > maxSize){ - mapperListAggregate = mapperList; - reducerListAggregate = reducerList; - maxSize = mapperList.size() + reducerList.size(); - } - } - if (state.equals("FAILED")) { - jobData.setSucceeded(false); - } - } - finally { - ThreadContextMR2.updateAuthToken(); - } - } - - TezTaskData[] mapperData = mapperListAggregate.toArray(new TezTaskData[mapperListAggregate.size()]); - TezTaskData[] reducerData = reducerListAggregate.toArray(new TezTaskData[reducerListAggregate.size()]); - - TezCounterData dagCounter = _jsonFactory.getDagCounter(_urlFactory.getDagURL(_jsonFactory.getDagIdsByApplicationId(dagIdsUrl).get(0))); - - jobData.setCounters(dagCounter).setMapTaskData(mapperData).setReduceTaskData(reducerData); - - return jobData; - } - - private URL getTaskListByVertexURL(String dagId, String vertexId) throws MalformedURLException { - return _urlFactory.getTaskListByVertexURL(dagId, vertexId); - } - - private URL getTaskURL(String taskId) throws MalformedURLException { - return _urlFactory.getTasksURL(taskId); - } - - private URL getTaskAttemptURL(String dagId, String taskId, String attemptId) throws MalformedURLException { - return _urlFactory.getTaskAttemptURL(dagId, taskId, attemptId); - } - - private class URLFactory { - - private String _timelineWebAddr; - - private URLFactory(String hserverAddr) throws IOException { - _timelineWebAddr = "http://" + hserverAddr + "/ws/v1/timeline"; - verifyURL(_timelineWebAddr); - } - - private void verifyURL(String url) throws IOException { - final URLConnection connection = new URL(url).openConnection(); - // Check service availability - connection.connect(); - return; - } - - private URL getDagURLByTezApplicationId(String applicationId) throws MalformedURLException { - return new URL(_timelineWebAddr + "/TEZ_DAG_ID?primaryFilter=applicationId:" + applicationId); - } - - private URL getApplicationURL(String applicationId) throws MalformedURLException { - return new URL(_timelineWebAddr + "/TEZ_APPLICATION/tez_" + applicationId); - } - - private URL getDagURL(String dagId) throws MalformedURLException { - return new URL(_timelineWebAddr + "/TEZ_DAG_ID/" + dagId); - } - - private URL getVertexListURL(String dagId) throws MalformedURLException { - return new URL(_timelineWebAddr + "/TEZ_VERTEX_ID?primaryFilter=TEZ_DAG_ID:" + dagId); - } - - private URL getTaskListByVertexURL(String dagId, String vertexId) throws MalformedURLException { - return new URL(_timelineWebAddr + "/TEZ_TASK_ID?primaryFilter=TEZ_DAG_ID:" + dagId + - "&secondaryFilter=TEZ_VERTEX_ID:" + vertexId + "&limit=500000"); - } - - private URL getTasksURL(String taskId) throws MalformedURLException { - return new URL(_timelineWebAddr + "/TEZ_TASK_ID/" + taskId); - } - - private URL getTaskAllAttemptsURL(String dagId, String taskId) throws MalformedURLException { - return new URL(_timelineWebAddr + "/TEZ_TASK_ATTEMPT_ID?primaryFilter=TEZ_DAG_ID:" + dagId + - "&secondaryFilter=TEZ_TASK_ID:" + taskId); - } - - private URL getTaskAttemptURL(String dagId, String taskId, String attemptId) throws MalformedURLException { - return new URL(_timelineWebAddr + "/TEZ_TASK_ATTEMPT_ID/" + attemptId); - } - - } - - /** - * JSONFactory class provides functionality to parse mined job data from timeline server. - */ - - private class JSONFactory { - - private String getState(URL url) throws IOException, AuthenticationException { - JsonNode rootNode = ThreadContextMR2.readJsonNode(url); - return rootNode.path("otherinfo").path("status").getTextValue(); - } - - private Properties getProperties(URL url) throws IOException, AuthenticationException { - Properties jobConf = new Properties(); - JsonNode rootNode = ThreadContextMR2.readJsonNode(url); - JsonNode configs = rootNode.path("otherinfo").path("config"); - Iterator keys = configs.getFieldNames(); - String key = ""; - String value = ""; - while (keys.hasNext()) { - key = keys.next(); - value = configs.get(key).getTextValue(); - jobConf.put(key, value); - } - return jobConf; - } - - private List getDagIdsByApplicationId(URL dagIdsUrl) throws IOException, AuthenticationException { - List dagIds = new ArrayList(); - JsonNode nodes = ThreadContextMR2.readJsonNode(dagIdsUrl).get("entities"); - - for (JsonNode node : nodes) { - String dagId = node.get("entity").getTextValue(); - dagIds.add(dagId); - } - - return dagIds; - } - - private TezCounterData getDagCounter(URL url) throws IOException, AuthenticationException { - TezCounterData holder = new TezCounterData(); - JsonNode rootNode = ThreadContextMR2.readJsonNode(url); - JsonNode groups = rootNode.path("otherinfo").path("counters").path("counterGroups"); - - for (JsonNode group : groups) { - for (JsonNode counter : group.path("counters")) { - String name = counter.get("counterName").getTextValue(); - String groupName = group.get("counterGroupName").getTextValue(); - Long value = counter.get("counterValue").getLongValue(); - holder.set(groupName, name, value); - } - } - - return holder; - } - - private long getDagStartTime(URL url) throws IOException, AuthenticationException { - JsonNode rootNode = ThreadContextMR2.readJsonNode(url); - long startTime = rootNode.path("otherinfo").get("startTime").getLongValue(); - return startTime; - } - - private long getDagEndTime(URL url) throws IOException, AuthenticationException { - JsonNode rootNode = ThreadContextMR2.readJsonNode(url); - long endTime = rootNode.path("otherinfo").get("endTime").getLongValue(); - return endTime; - } - - private void getTaskDataAll(URL vertexListUrl, String dagId, List mapperList, - List reducerList) throws IOException, AuthenticationException { - - JsonNode rootVertexNode = ThreadContextMR2.readJsonNode(vertexListUrl); - JsonNode vertices = rootVertexNode.path("entities"); - boolean isMapVertex = false; - - for (JsonNode vertex : vertices) { - String vertexId = vertex.get("entity").getTextValue(); - String vertexClass = vertex.path("otherinfo").path("processorClassName").getTextValue(); - - if (vertexClass.equals("org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor")) - isMapVertex = true; - else if (vertexClass.equals("org.apache.hadoop.hive.ql.exec.tez.ReduceTezProcessor")) - isMapVertex = false; - - URL tasksByVertexURL = getTaskListByVertexURL(dagId, vertexId); - if(isMapVertex) - getTaskDataByVertexId(tasksByVertexURL, dagId, vertexId, mapperList, true); - else - getTaskDataByVertexId(tasksByVertexURL, dagId, vertexId, reducerList, false); - } - } - - private void getTaskDataByVertexId(URL url, String dagId, String vertexId, List taskList, - boolean isMapVertex) throws IOException, AuthenticationException { - - JsonNode rootNode = ThreadContextMR2.readJsonNode(url); - JsonNode tasks = rootNode.path("entities"); - for (JsonNode task : tasks) { - String state = task.path("otherinfo").path("status").getTextValue(); - String taskId = task.get("entity").getValueAsText(); - String attemptId = task.path("otherinfo").path("successfulAttemptId").getTextValue(); - if (state.equals("SUCCEEDED")) { - attemptId = task.path("otherinfo").path("successfulAttemptId").getTextValue(); - } - else{ - JsonNode firstAttempt = getTaskFirstFailedAttempt(_urlFactory.getTaskAllAttemptsURL(dagId,taskId)); - if(firstAttempt != null){ - attemptId = firstAttempt.get("entity").getTextValue(); - } - } - - taskList.add(new TezTaskData(taskId, attemptId)); - } - - getTaskData(dagId, taskList, isMapVertex); - - } - - private JsonNode getTaskFirstFailedAttempt(URL taskAllAttemptsUrl) throws IOException, AuthenticationException { - JsonNode rootNode = ThreadContextMR2.readJsonNode(taskAllAttemptsUrl); - long firstAttemptFinishTime = Long.MAX_VALUE; - JsonNode firstAttempt = null; - JsonNode taskAttempts = rootNode.path("entities"); - for (JsonNode taskAttempt : taskAttempts) { - String state = taskAttempt.path("otherinfo").path("counters").path("status").getTextValue(); - if (state.equals("SUCCEEDED")) { - continue; - } - long finishTime = taskAttempt.path("otherinfo").path("counters").path("endTime").getLongValue(); - if( finishTime < firstAttemptFinishTime) { - firstAttempt = taskAttempt; - firstAttemptFinishTime = finishTime; - } - } - return firstAttempt; - } - - - - private void getTaskData(String dagId, List taskList, boolean isMapTask) - throws IOException, AuthenticationException { - - for(int i=0; i { - private static final Logger logger = Logger.getLogger(GenericDataSkewHeuristic.class); - - // Severity Parameters - private static final String NUM_TASKS_SEVERITY = "num_tasks_severity"; - private static final String DEVIATION_SEVERITY = "deviation_severity"; - private static final String FILES_SEVERITY = "files_severity"; - - // Default value of parameters - private double[] numTasksLimits = {10, 50, 100, 200}; // Number of map or reduce tasks - private double[] deviationLimits = {2, 4, 8, 16}; // Deviation in i/p bytes btw 2 groups - private double[] filesLimits = {1d / 8, 1d / 4, 1d / 2, 1d}; // Fraction of HDFS Block Size - - private HeuristicConfigurationData _heuristicConfData; - private List _counterNames; - - private void loadParameters() { - Map paramMap = _heuristicConfData.getParamMap(); - String heuristicName = _heuristicConfData.getHeuristicName(); - - double[] confNumTasksThreshold = Utils.getParam(paramMap.get(NUM_TASKS_SEVERITY), numTasksLimits.length); - if (confNumTasksThreshold != null) { - numTasksLimits = confNumTasksThreshold; - } - logger.info(heuristicName + " will use " + NUM_TASKS_SEVERITY + " with the following threshold settings: " - + Arrays.toString(numTasksLimits)); - - double[] confDeviationThreshold = Utils.getParam(paramMap.get(DEVIATION_SEVERITY), deviationLimits.length); - if (confDeviationThreshold != null) { - deviationLimits = confDeviationThreshold; - } - logger.info(heuristicName + " will use " + DEVIATION_SEVERITY + " with the following threshold settings: " - + Arrays.toString(deviationLimits)); - - double[] confFilesThreshold = Utils.getParam(paramMap.get(FILES_SEVERITY), filesLimits.length); - if (confFilesThreshold != null) { - filesLimits = confFilesThreshold; - } - logger.info( - heuristicName + " will use " + FILES_SEVERITY + " with the following threshold settings: " + Arrays.toString( - filesLimits)); - for (int i = 0; i < filesLimits.length; i++) { - filesLimits[i] = filesLimits[i] * HDFSContext.HDFS_BLOCK_SIZE; - } - } - - public GenericDataSkewHeuristic(List counterNames, HeuristicConfigurationData heuristicConfData) { - this._counterNames = counterNames; - this._heuristicConfData = heuristicConfData; - loadParameters(); - } - - protected abstract TezTaskData[] getTasks(TezApplicationData data); - - public HeuristicConfigurationData getHeuristicConfData() { - return _heuristicConfData; - } - - public HeuristicResult apply(TezApplicationData data) { - - if (!data.getSucceeded()) { - return null; - } - - TezTaskData[] tasks = getTasks(data); - - //Gathering data for checking time skew - List timeTaken = new ArrayList(); - - for(int i = 0; i < tasks.length; i++) { - if (tasks[i].isSampled()) { - timeTaken.add(tasks[i].getTotalRunTimeMs()); - } - } - - long[][] groupsTime = Statistics.findTwoGroups(Longs.toArray(timeTaken)); - - long timeAvg1 = Statistics.average(groupsTime[0]); - long timeAvg2 = Statistics.average(groupsTime[1]); - - //seconds are used for calculating deviation as they provide a better idea than millisecond. - long timeAvgSec1 = TimeUnit.MILLISECONDS.toSeconds(timeAvg1); - long timeAvgSec2 = TimeUnit.MILLISECONDS.toSeconds(timeAvg2); - - long minTime = Math.min(timeAvgSec1, timeAvgSec2); - long diffTime = Math.abs(timeAvgSec1 - timeAvgSec2); - - //using the same deviation limits for time skew as for data skew. It can be changed in the fututre. - Severity severityTime = getDeviationSeverity(minTime, diffTime); - - //This reduces severity if number of tasks is insignificant - severityTime = Severity.min(severityTime, - Severity.getSeverityAscending(groupsTime[0].length, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], - numTasksLimits[3])); - - //Gather data - List inputSizes = new ArrayList(); - - for (int i = 0; i < tasks.length; i++) { - if (tasks[i].isSampled()) { - - long inputByte = 0; - for (TezCounterData.CounterName counterName : _counterNames) { - inputByte += tasks[i].getCounters().get(counterName); - } - - inputSizes.add(inputByte); - } - } - - long[][] groups = Statistics.findTwoGroups(Longs.toArray(inputSizes)); - - long avg1 = Statistics.average(groups[0]); - long avg2 = Statistics.average(groups[1]); - - long min = Math.min(avg1, avg2); - long diff = Math.abs(avg2 - avg1); - - Severity severityData = getDeviationSeverity(min, diff); - - //This reduces severity if the largest file sizes are insignificant - severityData = Severity.min(severityData, getFilesSeverity(avg2)); - - //This reduces severity if number of tasks is insignificant - severityData = Severity.min(severityData, - Severity.getSeverityAscending(groups[0].length, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], - numTasksLimits[3])); - - Severity severity = Severity.max(severityData, severityTime); - - HeuristicResult result = - new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), severity, - Utils.getHeuristicScore(severityData, tasks.length)); - - result.addResultDetail("Data skew (Number of tasks)", Integer.toString(tasks.length)); - result.addResultDetail("Data skew (Group A)", - groups[0].length + " tasks @ " + FileUtils.byteCountToDisplaySize(avg1) + " avg"); - result.addResultDetail("Data skew (Group B)", - groups[1].length + " tasks @ " + FileUtils.byteCountToDisplaySize(avg2) + " avg"); - - result.addResultDetail("Time skew (Number of tasks)", Integer.toString(tasks.length)); - result.addResultDetail("Time skew (Group A)", - groupsTime[0].length + " tasks @ " + convertTimeMs(timeAvg1) + " avg"); - result.addResultDetail("Time skew (Group B)", - groupsTime[1].length + " tasks @ " + convertTimeMs(timeAvg2) + " avg"); - - return result; - } - - private String convertTimeMs(long timeMs) { - if (timeMs < 1000) { - return Long.toString(timeMs) + " msec"; - } - return DurationFormatUtils.formatDuration(timeMs, "HH:mm:ss") + " HH:MM:SS"; - } - - private Severity getDeviationSeverity(long averageMin, long averageDiff) { - if (averageMin <= 0) { - averageMin = 1; - } - long value = averageDiff / averageMin; - return Severity.getSeverityAscending(value, deviationLimits[0], deviationLimits[1], deviationLimits[2], - deviationLimits[3]); - } - - private Severity getFilesSeverity(long value) { - return Severity.getSeverityAscending(value, filesLimits[0], filesLimits[1], filesLimits[2], filesLimits[3]); - } -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/GenericGCHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/GenericGCHeuristic.java deleted file mode 100644 index d5726ef71..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/GenericGCHeuristic.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.util.Utils; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.math.Statistics; -import java.util.Map; -import org.apache.log4j.Logger; - - -/** - * Analyses garbage collection efficiency - */ -public abstract class GenericGCHeuristic implements Heuristic { - private static final Logger logger = Logger.getLogger(GenericGCHeuristic.class); - - // Severity Parameters - private static final String GC_RATIO_SEVERITY = "gc_ratio_severity"; - private static final String RUNTIME_SEVERITY = "runtime_severity_in_min"; - - // Default value of parameters - private double[] gcRatioLimits = {0.01d, 0.02d, 0.03d, 0.04d}; // Garbage Collection Time / CPU Time - private double[] runtimeLimits = {5, 10, 12, 15}; // Task Runtime in milli sec - - private HeuristicConfigurationData _heuristicConfData; - - private void loadParameters() { - Map paramMap = _heuristicConfData.getParamMap(); - String heuristicName = _heuristicConfData.getHeuristicName(); - - double[] confGcRatioThreshold = Utils.getParam(paramMap.get(GC_RATIO_SEVERITY), gcRatioLimits.length); - if (confGcRatioThreshold != null) { - gcRatioLimits = confGcRatioThreshold; - } - logger.info(heuristicName + " will use " + GC_RATIO_SEVERITY + " with the following threshold settings: " - + Arrays.toString(gcRatioLimits)); - - double[] confRuntimeThreshold = Utils.getParam(paramMap.get(RUNTIME_SEVERITY), runtimeLimits.length); - if (confRuntimeThreshold != null) { - runtimeLimits = confRuntimeThreshold; - } - logger.info(heuristicName + " will use " + RUNTIME_SEVERITY + " with the following threshold settings: " - + Arrays.toString(runtimeLimits)); - for (int i = 0; i < runtimeLimits.length; i++) { - runtimeLimits[i] = runtimeLimits[i] * Statistics.MINUTE_IN_MS; - } - } - - public GenericGCHeuristic(HeuristicConfigurationData heuristicConfData) { - this._heuristicConfData = heuristicConfData; - loadParameters(); - } - - protected abstract TezTaskData[] getTasks(TezApplicationData data); - - public HeuristicConfigurationData getHeuristicConfData() { - return _heuristicConfData; - } - - public HeuristicResult apply(TezApplicationData data) { - - if(!data.getSucceeded()) { - return null; - } - - TezTaskData[] tasks = getTasks(data) ; - List gcMs = new ArrayList(); - List cpuMs = new ArrayList(); - List runtimesMs = new ArrayList(); - - for (TezTaskData task : tasks) { - if (task.isSampled()) { - runtimesMs.add(task.getTotalRunTimeMs()); - gcMs.add(task.getCounters().get(TezCounterData.CounterName.GC_TIME_MILLIS)); - cpuMs.add(task.getCounters().get(TezCounterData.CounterName.CPU_MILLISECONDS)); - } - } - - long avgRuntimeMs = Statistics.average(runtimesMs); - long avgCpuMs = Statistics.average(cpuMs); - long avgGcMs = Statistics.average(gcMs); - double ratio = avgCpuMs != 0 ? avgGcMs*(1.0)/avgCpuMs: 0; - - Severity severity; - if (tasks.length == 0) { - severity = Severity.NONE; - } else { - severity = getGcRatioSeverity(avgRuntimeMs, avgCpuMs, avgGcMs); - } - - HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), - _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); - - result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); - result.addResultDetail("Avg task runtime (ms)", Long.toString(avgRuntimeMs)); - result.addResultDetail("Avg task CPU time (ms)", Long.toString(avgCpuMs)); - result.addResultDetail("Avg task GC time (ms)", Long.toString(avgGcMs)); - result.addResultDetail("Task GC/CPU ratio", Double.toString(ratio)); - return result; - } - - private Severity getGcRatioSeverity(long runtimeMs, long cpuMs, long gcMs) { - double gcRatio = ((double)gcMs)/cpuMs; - Severity ratioSeverity = Severity.getSeverityAscending( - gcRatio, gcRatioLimits[0], gcRatioLimits[1], gcRatioLimits[2], gcRatioLimits[3]); - - // Severity is reduced if task runtime is insignificant - Severity runtimeSeverity = getRuntimeSeverity(runtimeMs); - - return Severity.min(ratioSeverity, runtimeSeverity); - } - - private Severity getRuntimeSeverity(long runtimeMs) { - return Severity.getSeverityAscending( - runtimeMs, runtimeLimits[0], runtimeLimits[1], runtimeLimits[2], runtimeLimits[3]); - } - -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java deleted file mode 100644 index 3a575092b..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.google.common.base.Strings; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.util.Utils; -import org.apache.log4j.Logger; - -import org.apache.commons.io.FileUtils; - -import java.util.*; - -/** - * Analyzes mapper memory allocation and requirements - */ -public abstract class GenericMemoryHeuristic implements Heuristic { - - private static final Logger logger = Logger.getLogger(GenericMemoryHeuristic.class); - - //Severity Parameters - private static final String MEM_RATIO_SEVERITY = "memory_ratio_severity"; - private static final String DEFAULT_MAPPER_CONTAINER_SIZE = "2048"; - private static final String CONTAINER_MEM_DEFAULT_MB = "container_memory_default_mb"; - private String _containerMemConf; - - //Default Value of parameters - - private double [] memoryRatioLimits = {0.6d, 0.5d, 0.4d, 0.3d}; //Ratio of successful tasks - - private HeuristicConfigurationData _heuristicConfData; - - private String getContainerMemDefaultMBytes() { - Map paramMap = _heuristicConfData.getParamMap(); - if (paramMap.containsKey(CONTAINER_MEM_DEFAULT_MB)) { - String strValue = paramMap.get(CONTAINER_MEM_DEFAULT_MB); - try { - return strValue; - } - catch (NumberFormatException e) { - logger.warn(CONTAINER_MEM_DEFAULT_MB + ": expected number [" + strValue + "]"); - } - } - return DEFAULT_MAPPER_CONTAINER_SIZE; - } - - private void loadParameters() { - Map paramMap = _heuristicConfData.getParamMap(); - String heuristicName = _heuristicConfData.getHeuristicName(); - - double[] confSuccessRatioLimits = Utils.getParam(paramMap.get(MEM_RATIO_SEVERITY), memoryRatioLimits.length); - if (confSuccessRatioLimits != null) { - memoryRatioLimits = confSuccessRatioLimits; - } - logger.info(heuristicName + " will use " + MEM_RATIO_SEVERITY + " with the following threshold settings: " - + Arrays.toString(memoryRatioLimits)); - - - } - - public GenericMemoryHeuristic(String containerMemConf, HeuristicConfigurationData heuristicConfData) { - this._containerMemConf = containerMemConf; - this._heuristicConfData = heuristicConfData; - loadParameters(); - } - - public HeuristicConfigurationData getHeuristicConfData() { - return _heuristicConfData; - } - - protected abstract TezTaskData[] getTasks(TezApplicationData data); - - public HeuristicResult apply(TezApplicationData data) { - if(!data.getSucceeded()) { - return null; - } - TezTaskData[] tasks = getTasks(data); - - - List totalPhysicalMemory = new LinkedList(); - List totalVirtualMemory = new LinkedList(); - List runTime = new LinkedList(); - - for (TezTaskData task : tasks) { - - if (task.isSampled()) { - totalPhysicalMemory.add(task.getCounters().get(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES)); - totalVirtualMemory.add(task.getCounters().get(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES)); - runTime.add(task.getTotalRunTimeMs()); - } - - - } - - long averagePMem = Statistics.average(totalPhysicalMemory); - long averageVMem = Statistics.average(totalVirtualMemory); - long maxPMem; - long minPMem; - try{ - maxPMem = Collections.max(totalPhysicalMemory); - minPMem = Collections.min(totalPhysicalMemory); - - } - catch(Exception exception){ - maxPMem = 0; - minPMem = 0; - } - long averageRunTime = Statistics.average(runTime); - - String containerSizeStr; - - if(!Strings.isNullOrEmpty(data.getConf().getProperty(_containerMemConf))){ - containerSizeStr = data.getConf().getProperty(_containerMemConf); - } - else { - containerSizeStr = getContainerMemDefaultMBytes(); - } - - long containerSize = Long.valueOf(containerSizeStr) * FileUtils.ONE_MB; - - double averageMemMb = (double)((averagePMem) /FileUtils.ONE_MB) ; - - double ratio = averageMemMb / ((double)(containerSize / FileUtils.ONE_MB)); - - Severity severity ; - - if(tasks.length == 0){ - severity = Severity.NONE; - } - else{ - severity = getMemoryRatioSeverity(ratio); - } - - HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), - _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); - - result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); - result.addResultDetail("Maximum Physical Memory (MB)", - tasks.length == 0 ? "0" : Long.toString(maxPMem/FileUtils.ONE_MB)); - result.addResultDetail("Minimum Physical memory (MB)", - tasks.length == 0 ? "0" : Long.toString(minPMem/FileUtils.ONE_MB)); - result.addResultDetail("Average Physical Memory (MB)", - tasks.length == 0 ? "0" : Long.toString(averagePMem/FileUtils.ONE_MB)); - result.addResultDetail("Average Virtual Memory (MB)", - tasks.length == 0 ? "0" : Long.toString(averageVMem/FileUtils.ONE_MB)); - result.addResultDetail("Average Task RunTime", - tasks.length == 0 ? "0" : Statistics.readableTimespan(averageRunTime)); - result.addResultDetail("Requested Container Memory (MB)", - (tasks.length == 0 || containerSize == 0 || containerSize == -1) ? "0" : String.valueOf(containerSize / FileUtils.ONE_MB)); - - - return result; - - } - - private Severity getMemoryRatioSeverity(double ratio) { - return Severity.getSeverityDescending( - ratio, memoryRatioLimits[0], memoryRatioLimits[1], memoryRatioLimits[2], memoryRatioLimits[3]); - } - - - - -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristic.java deleted file mode 100644 index b17ebe3b2..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristic.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; - -import java.util.Arrays; -import org.apache.log4j.Logger; - - -/** - * This Heuristic analyses the skewness in the task input data - */ -public class MapperDataSkewHeuristic extends GenericDataSkewHeuristic { - private static final Logger logger = Logger.getLogger(MapperDataSkewHeuristic.class); - - public MapperDataSkewHeuristic(HeuristicConfigurationData heuristicConfData) { - super(Arrays.asList( - TezCounterData.CounterName.HDFS_BYTES_READ, - TezCounterData.CounterName.S3A_BYTES_READ, - TezCounterData.CounterName.S3N_BYTES_READ - ), heuristicConfData); - } - - @Override - protected TezTaskData[] getTasks(TezApplicationData data) { - return data.getMapTaskData(); - } -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristic.java deleted file mode 100644 index 3a82ae5f9..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristic.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.util.Utils; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.math.Statistics; -import java.util.Map; -import org.apache.log4j.Logger; - - -/** - * Analyses garbage collection efficiency - */ -public class MapperGCHeuristic extends GenericGCHeuristic{ - private static final Logger logger = Logger.getLogger(MapperGCHeuristic.class); - - public MapperGCHeuristic(HeuristicConfigurationData heuristicConfData) { - super(heuristicConfData); - } - - @Override - protected TezTaskData[] getTasks(TezApplicationData data) { - return data.getMapTaskData(); - } - -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java deleted file mode 100644 index 96d5bb12e..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; - -import com.linkedin.drelephant.tez.data.TezApplicationData; - -import com.linkedin.drelephant.tez.data.TezTaskData; - -import org.apache.log4j.Logger; - -/** - * Analyzes mapper memory allocation and requirements - */ -public class MapperMemoryHeuristic extends GenericMemoryHeuristic { - - private static final Logger logger = Logger.getLogger(MapperMemoryHeuristic.class); - - public static final String MAPPER_MEMORY_CONF = "mapreduce.map.memory.mb"; - - public MapperMemoryHeuristic(HeuristicConfigurationData __heuristicConfData){ - super(MAPPER_MEMORY_CONF, __heuristicConfData); - } - - @Override - protected TezTaskData[] getTasks(TezApplicationData data) { - return data.getMapTaskData(); - } - -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristic.java deleted file mode 100644 index 1694e32ea..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristic.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.util.Utils; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import com.linkedin.drelephant.analysis.HDFSContext; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.math.Statistics; - -import java.util.Map; - -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; - -/** - * Analyzes mapper task speed and efficiency - */ -public class MapperSpeedHeuristic implements Heuristic { - private static final Logger logger = Logger.getLogger(MapperSpeedHeuristic.class); - - // Severity parameters. - private static final String DISK_SPEED_SEVERITY = "disk_speed_severity"; - private static final String RUNTIME_SEVERITY = "runtime_severity_in_min"; - - // Default value of parameters - private double[] diskSpeedLimits = {1d/2, 1d/4, 1d/8, 1d/32}; // Fraction of HDFS block size - private double[] runtimeLimits = {5, 10, 15, 30}; // The Map task runtime in milli sec - - private List _counterNames = Arrays.asList( - TezCounterData.CounterName.HDFS_BYTES_READ, - TezCounterData.CounterName.S3A_BYTES_READ, - TezCounterData.CounterName.S3N_BYTES_READ - ); - - private HeuristicConfigurationData _heuristicConfData; - - private void loadParameters() { - Map paramMap = _heuristicConfData.getParamMap(); - String heuristicName = _heuristicConfData.getHeuristicName(); - - double[] confDiskSpeedThreshold = Utils.getParam(paramMap.get(DISK_SPEED_SEVERITY), diskSpeedLimits.length); - if (confDiskSpeedThreshold != null) { - diskSpeedLimits = confDiskSpeedThreshold; - } - logger.info(heuristicName + " will use " + DISK_SPEED_SEVERITY + " with the following threshold settings: " - + Arrays.toString(diskSpeedLimits)); - for (int i = 0; i < diskSpeedLimits.length; i++) { - diskSpeedLimits[i] = diskSpeedLimits[i] * HDFSContext.DISK_READ_SPEED; - } - - double[] confRuntimeThreshold = Utils.getParam(paramMap.get(RUNTIME_SEVERITY), runtimeLimits.length); - if (confRuntimeThreshold != null) { - runtimeLimits = confRuntimeThreshold; - } - logger.info(heuristicName + " will use " + RUNTIME_SEVERITY + " with the following threshold settings: " + Arrays - .toString(runtimeLimits)); - for (int i = 0; i < runtimeLimits.length; i++) { - runtimeLimits[i] = runtimeLimits[i] * Statistics.MINUTE_IN_MS; - } - } - - public MapperSpeedHeuristic(HeuristicConfigurationData heuristicConfData) { - this._heuristicConfData = heuristicConfData; - loadParameters(); - } - - public HeuristicConfigurationData getHeuristicConfData() { - return _heuristicConfData; - } - - public HeuristicResult apply(TezApplicationData data) { - - if(!data.getSucceeded()) { - return null; - } - - TezTaskData[] tasks = data.getMapTaskData(); - - List inputSizes = new ArrayList(); - List speeds = new ArrayList(); - List runtimesMs = new ArrayList(); - - for (TezTaskData task : tasks) { - - if (task.isSampled()) { - - long inputBytes = 0; - - for (TezCounterData.CounterName counterName: _counterNames) { - inputBytes += task.getCounters().get(counterName); - } - - long runtimeMs = task.getTotalRunTimeMs(); - inputSizes.add(inputBytes); - runtimesMs.add(runtimeMs); - //Speed is records per second - speeds.add((1000 * inputBytes) / (runtimeMs)); - } - } - - long medianSpeed; - long medianSize; - long medianRuntimeMs; - - if (tasks.length != 0) { - medianSpeed = Statistics.median(speeds); - medianSize = Statistics.median(inputSizes); - medianRuntimeMs = Statistics.median(runtimesMs); - } else { - medianSpeed = 0; - medianSize = 0; - medianRuntimeMs = 0; - } - - Severity severity = getDiskSpeedSeverity(medianSpeed); - - //This reduces severity if task runtime is insignificant - severity = Severity.min(severity, getRuntimeSeverity(medianRuntimeMs)); - - HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), - _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); - - result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); - result.addResultDetail("Median task input ", FileUtils.byteCountToDisplaySize(medianSize)); - result.addResultDetail("Median task runtime", Statistics.readableTimespan(medianRuntimeMs)); - result.addResultDetail("Median task speed", FileUtils.byteCountToDisplaySize(medianSpeed) + "/s"); - - return result; - } - - private Severity getDiskSpeedSeverity(long speed) { - return Severity.getSeverityDescending( - speed, diskSpeedLimits[0], diskSpeedLimits[1], diskSpeedLimits[2], diskSpeedLimits[3]); - } - - private Severity getRuntimeSeverity(long runtimeMs) { - return Severity.getSeverityAscending( - runtimeMs, runtimeLimits[0], runtimeLimits[1], runtimeLimits[2], runtimeLimits[3]); - } -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristic.java deleted file mode 100644 index f7ea997e5..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristic.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.util.Utils; -import org.apache.log4j.Logger; - - -import java.util.Arrays; -import java.util.Map; - -/** - * Analyzes mapper task data spill rates - */ -public class MapperSpillHeuristic implements Heuristic { - private static final Logger logger = Logger.getLogger(MapperSpillHeuristic.class); - private static final long THRESHOLD_SPILL_FACTOR = 10000; - - // Severity parameters. - private static final String SPILL_SEVERITY = "spill_severity"; - private static final String NUM_TASKS_SEVERITY = "num_tasks_severity"; - - // Default value of parameters - private double[] numTasksLimits = {50, 100, 500, 1000}; // Number of Map tasks. - private double[] spillLimits = {2.01d, 2.2d, 2.5d, 3.0d}; // Records spilled/total output records - - private HeuristicConfigurationData _heuristicConfData; - - private void loadParameters() { - Map paramMap = _heuristicConfData.getParamMap(); - String heuristicName = _heuristicConfData.getHeuristicName(); - - double[] confNumTasksThreshold = Utils.getParam(paramMap.get(NUM_TASKS_SEVERITY), numTasksLimits.length); - if (confNumTasksThreshold != null) { - numTasksLimits = confNumTasksThreshold; - } - logger.info(heuristicName + " will use " + NUM_TASKS_SEVERITY + " with the following threshold settings: " - + Arrays.toString(numTasksLimits)); - - double[] confSpillThreshold = Utils.getParam(paramMap.get(SPILL_SEVERITY), spillLimits.length); - if (confSpillThreshold != null) { - spillLimits = confSpillThreshold; - } - logger.info(heuristicName + " will use " + SPILL_SEVERITY + " with the following threshold settings: " + Arrays - .toString(spillLimits)); - for (int i = 0; i < spillLimits.length; i++) { - spillLimits[i] = spillLimits[i] * THRESHOLD_SPILL_FACTOR; - } - } - - public MapperSpillHeuristic(HeuristicConfigurationData heuristicConfData) { - this._heuristicConfData = heuristicConfData; - loadParameters(); - } - - @Override - public HeuristicConfigurationData getHeuristicConfData() { - return _heuristicConfData; - } - - @Override - public HeuristicResult apply(TezApplicationData data) { - if(!data.getSucceeded()) { - return null; - } - TezTaskData[] tasks = data.getMapTaskData(); - - long totalSpills = 0; - long totalOutputRecords = 0; - double ratioSpills = 0.0; - - for (TezTaskData task : tasks) { - - if (task.isSampled()) { - totalSpills += task.getCounters().get(TezCounterData.CounterName.SPILLED_RECORDS); - totalOutputRecords += task.getCounters().get(TezCounterData.CounterName.OUTPUT_RECORDS); - } - } - - //If both totalSpills and totalOutputRecords are zero then set ratioSpills to zero. - if (totalSpills == 0) { - ratioSpills = 0; - } else { - ratioSpills = (double) totalSpills / (double) totalOutputRecords; - } - - Severity severity = getSpillSeverity(ratioSpills); - - // Severity is reduced if number of tasks is small - Severity taskSeverity = getNumTasksSeverity(tasks.length); - severity = Severity.min(severity, taskSeverity); - - HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), - _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); - - result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); - result.addResultDetail("Avg spilled records per task", - tasks.length == 0 ? "0" : Long.toString(totalSpills / tasks.length)); - result.addResultDetail("Avg output records per task", - tasks.length == 0 ? "0" : Long.toString(totalOutputRecords / tasks.length)); - result.addResultDetail("Ratio of spilled records to output records", Double.toString(ratioSpills)); - - return result; - - } - - private Severity getSpillSeverity(double ratioSpills) { - - long normalizedSpillRatio = 0; - //Normalize the ratio to integer. - normalizedSpillRatio = (long) (ratioSpills * THRESHOLD_SPILL_FACTOR); - - return Severity.getSeverityAscending( - normalizedSpillRatio, spillLimits[0], spillLimits[1], spillLimits[2], spillLimits[3]); - } - - private Severity getNumTasksSeverity(long numTasks) { - return Severity.getSeverityAscending( - numTasks, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], numTasksLimits[3]); - } - - -} diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristic.java deleted file mode 100644 index 838bd7a59..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristic.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - - -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.util.Utils; -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; - -import com.linkedin.drelephant.math.Statistics; - -import java.util.*; - -/** - * Analyzes mapper task runtimes - */ -public class MapperTimeHeuristic implements Heuristic { - - private static final Logger logger = Logger.getLogger(MapperTimeHeuristic.class); - - // Severity parameters. - private static final String SHORT_RUNTIME_SEVERITY = "short_runtime_severity_in_min"; - private static final String LONG_RUNTIME_SEVERITY = "long_runtime_severity_in_min"; - private static final String NUM_TASKS_SEVERITY = "num_tasks_severity"; - - // Default value of parameters - private double[] shortRuntimeLimits = {10, 4, 2, 1}; // Limits(ms) for tasks with shorter runtime - private double[] longRuntimeLimits = {15, 30, 60, 120}; // Limits(ms) for tasks with longer runtime - private double[] numTasksLimits = {50, 101, 500, 1000}; // Number of Map tasks. - - private List _counterNames = Arrays.asList( - TezCounterData.CounterName.HDFS_BYTES_READ, - TezCounterData.CounterName.S3A_BYTES_READ, - TezCounterData.CounterName.S3N_BYTES_READ - ); - - private HeuristicConfigurationData _heuristicConfData; - - private void loadParameters() { - Map paramMap = _heuristicConfData.getParamMap(); - String heuristicName = _heuristicConfData.getHeuristicName(); - - double[] confShortThreshold = Utils.getParam(paramMap.get(SHORT_RUNTIME_SEVERITY), shortRuntimeLimits.length); - if (confShortThreshold != null) { - shortRuntimeLimits = confShortThreshold; - } - logger.info(heuristicName + " will use " + SHORT_RUNTIME_SEVERITY + " with the following threshold settings: " - + Arrays.toString(shortRuntimeLimits)); - for (int i = 0; i < shortRuntimeLimits.length; i++) { - shortRuntimeLimits[i] = shortRuntimeLimits[i] * Statistics.MINUTE_IN_MS; - } - - double[] confLongThreshold = Utils.getParam(paramMap.get(LONG_RUNTIME_SEVERITY), longRuntimeLimits.length); - if (confLongThreshold != null) { - longRuntimeLimits = confLongThreshold; - } - logger.info(heuristicName + " will use " + LONG_RUNTIME_SEVERITY + " with the following threshold settings: " - + Arrays.toString(longRuntimeLimits)); - for (int i = 0; i < longRuntimeLimits.length; i++) { - longRuntimeLimits[i] = longRuntimeLimits[i] * Statistics.MINUTE_IN_MS; - } - - double[] confNumTasksThreshold = Utils.getParam(paramMap.get(NUM_TASKS_SEVERITY), numTasksLimits.length); - if (confNumTasksThreshold != null) { - numTasksLimits = confNumTasksThreshold; - } - logger.info(heuristicName + " will use " + NUM_TASKS_SEVERITY + " with the following threshold settings: " + Arrays - .toString(numTasksLimits)); - - - } - - public MapperTimeHeuristic(HeuristicConfigurationData heuristicConfData) { - this._heuristicConfData = heuristicConfData; - loadParameters(); - } - - public HeuristicConfigurationData getHeuristicConfData() { - return _heuristicConfData; - } - - public HeuristicResult apply(TezApplicationData data) { - if(!data.getSucceeded()) { - return null; - } - TezTaskData[] tasks = data.getMapTaskData(); - - List inputSizes = new ArrayList(); - List runtimesMs = new ArrayList(); - long taskMinMs = Long.MAX_VALUE; - long taskMaxMs = 0; - - for (TezTaskData task : tasks) { - - if (task.isSampled()) { - long inputByte = 0; - for (TezCounterData.CounterName counterName: _counterNames) { - inputByte += task.getCounters().get(counterName); - } - inputSizes.add(inputByte); - long taskTime = task.getTotalRunTimeMs(); - runtimesMs.add(taskTime); - taskMinMs = Math.min(taskMinMs, taskTime); - taskMaxMs = Math.max(taskMaxMs, taskTime); - } - } - - if(taskMinMs == Long.MAX_VALUE) { - taskMinMs = 0; - } - - long averageSize = Statistics.average(inputSizes); - long averageTimeMs = Statistics.average(runtimesMs); - - Severity shortTaskSeverity = shortTaskSeverity(tasks.length, averageTimeMs); - Severity longTaskSeverity = longTaskSeverity(tasks.length, averageTimeMs); - Severity severity = Severity.max(shortTaskSeverity, longTaskSeverity); - - HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), - _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); - - result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); - result.addResultDetail("Average task input size", FileUtils.byteCountToDisplaySize(averageSize)); - result.addResultDetail("Average task runtime", Statistics.readableTimespan(averageTimeMs)); - result.addResultDetail("Max task runtime", Statistics.readableTimespan(taskMaxMs)); - result.addResultDetail("Min task runtime", Statistics.readableTimespan(taskMinMs)); - - return result; - } - - private Severity shortTaskSeverity(long numTasks, long averageTimeMs) { - // We want to identify jobs with short task runtime - Severity severity = getShortRuntimeSeverity(averageTimeMs); - // Severity is reduced if number of tasks is small. - Severity numTaskSeverity = getNumTasksSeverity(numTasks); - return Severity.min(severity, numTaskSeverity); - } - - private Severity longTaskSeverity(long numTasks, long averageTimeMs) { - // We want to identify jobs with long task runtime. Severity is NOT reduced if num of tasks is large - return getLongRuntimeSeverity(averageTimeMs); - } - - private Severity getShortRuntimeSeverity(long runtimeMs) { - return Severity.getSeverityDescending( - runtimeMs, shortRuntimeLimits[0], shortRuntimeLimits[1], shortRuntimeLimits[2], shortRuntimeLimits[3]); - } - - private Severity getLongRuntimeSeverity(long runtimeMs) { - return Severity.getSeverityAscending( - runtimeMs, longRuntimeLimits[0], longRuntimeLimits[1], longRuntimeLimits[2], longRuntimeLimits[3]); - } - - private Severity getNumTasksSeverity(long numTasks) { - return Severity.getSeverityAscending( - numTasks, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], numTasksLimits[3]); - } - - - -} diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristic.java deleted file mode 100644 index 3a01448d4..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristic.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import org.apache.log4j.Logger; - -import java.util.Arrays; - - -/** - * This Heuristic analyses the skewness in the task input data - */ -public class ReducerDataSkewHeuristic extends GenericDataSkewHeuristic { - private static final Logger logger = Logger.getLogger(ReducerDataSkewHeuristic.class); - - public ReducerDataSkewHeuristic(HeuristicConfigurationData heuristicConfData) { - super(Arrays.asList(TezCounterData.CounterName.SHUFFLE_BYTES), heuristicConfData); - } - - @Override - protected TezTaskData[] getTasks(TezApplicationData data) { - return data.getReduceTaskData(); - } -} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristic.java deleted file mode 100644 index 3ea3bdac4..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristic.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import org.apache.log4j.Logger; - - -/** - * Analyses garbage collection efficiency - */ -public class ReducerGCHeuristic extends GenericGCHeuristic { - - private static final Logger logger = Logger.getLogger(ReducerGCHeuristic.class); - - public ReducerGCHeuristic(HeuristicConfigurationData heuristicConfData) { - super(heuristicConfData); - } - - @Override - protected TezTaskData[] getTasks(TezApplicationData data) { - return data.getReduceTaskData(); - } - -} diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java deleted file mode 100644 index 41fc6fa1d..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import org.apache.log4j.Logger; - -/** - * Analyzes reducer memory allocation and requirements - */ - -public class ReducerMemoryHeuristic extends GenericMemoryHeuristic { - - private static final Logger logger = Logger.getLogger(ReducerMemoryHeuristic.class); - - public static final String REDUCER_MEMORY_CONF = "mapreduce.reduce.memory.mb"; - - public ReducerMemoryHeuristic(HeuristicConfigurationData __heuristicConfData){ - super(REDUCER_MEMORY_CONF, __heuristicConfData); - } - - @Override - protected TezTaskData[] getTasks(TezApplicationData data) { - return data.getReduceTaskData(); - } - - - - -} diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristic.java deleted file mode 100644 index 5e0dbfe65..000000000 --- a/app/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristic.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - - -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.util.Utils; -import org.apache.log4j.Logger; - -import com.linkedin.drelephant.math.Statistics; - -import java.util.*; - -/** - * Analyzes reducer task runtimes - */ - -public class ReducerTimeHeuristic implements Heuristic { - - private static final Logger logger = Logger.getLogger(ReducerTimeHeuristic.class); - - // Severity parameters. - private static final String SHORT_RUNTIME_SEVERITY = "short_runtime_severity_in_min"; - private static final String LONG_RUNTIME_SEVERITY = "long_runtime_severity_in_min"; - private static final String NUM_TASKS_SEVERITY = "num_tasks_severity"; - - // Default value of parameters - private double[] shortRuntimeLimits = {10, 4, 2, 1}; // Limits(ms) for tasks with shorter runtime - private double[] longRuntimeLimits = {15, 30, 60, 120}; // Limits(ms) for tasks with longer runtime - private double[] numTasksLimits = {50, 101, 500, 1000}; // Number of Map tasks. - - private HeuristicConfigurationData _heuristicConfData; - - private void loadParameters() { - Map paramMap = _heuristicConfData.getParamMap(); - String heuristicName = _heuristicConfData.getHeuristicName(); - - double[] confShortThreshold = Utils.getParam(paramMap.get(SHORT_RUNTIME_SEVERITY), shortRuntimeLimits.length); - if (confShortThreshold != null) { - shortRuntimeLimits = confShortThreshold; - } - logger.info(heuristicName + " will use " + SHORT_RUNTIME_SEVERITY + " with the following threshold settings: " - + Arrays.toString(shortRuntimeLimits)); - for (int i = 0; i < shortRuntimeLimits.length; i++) { - shortRuntimeLimits[i] = shortRuntimeLimits[i] * Statistics.MINUTE_IN_MS; - } - - double[] confLongThreshold = Utils.getParam(paramMap.get(LONG_RUNTIME_SEVERITY), longRuntimeLimits.length); - if (confLongThreshold != null) { - longRuntimeLimits = confLongThreshold; - } - logger.info(heuristicName + " will use " + LONG_RUNTIME_SEVERITY + " with the following threshold settings: " - + Arrays.toString(longRuntimeLimits)); - for (int i = 0; i < longRuntimeLimits.length; i++) { - longRuntimeLimits[i] = longRuntimeLimits[i] * Statistics.MINUTE_IN_MS; - } - - double[] confNumTasksThreshold = Utils.getParam(paramMap.get(NUM_TASKS_SEVERITY), numTasksLimits.length); - if (confNumTasksThreshold != null) { - numTasksLimits = confNumTasksThreshold; - } - logger.info(heuristicName + " will use " + NUM_TASKS_SEVERITY + " with the following threshold settings: " + Arrays - .toString(numTasksLimits)); - - - } - - public ReducerTimeHeuristic(HeuristicConfigurationData heuristicConfData) { - this._heuristicConfData = heuristicConfData; - loadParameters(); - } - - public HeuristicConfigurationData getHeuristicConfData() { - return _heuristicConfData; - } - - public HeuristicResult apply(TezApplicationData data) { - if(!data.getSucceeded()) { - return null; - } - TezTaskData[] tasks = data.getReduceTaskData(); - - List runtimesMs = new ArrayList(); - long taskMinMs = Long.MAX_VALUE; - long taskMaxMs = 0; - - for (TezTaskData task : tasks) { - - if (task.isSampled()) { - long taskTime = task.getTotalRunTimeMs(); - runtimesMs.add(taskTime); - taskMinMs = Math.min(taskMinMs, taskTime); - taskMaxMs = Math.max(taskMaxMs, taskTime); - } - } - - if(taskMinMs == Long.MAX_VALUE) { - taskMinMs = 0; - } - - - long averageTimeMs = Statistics.average(runtimesMs); - - Severity shortTaskSeverity = shortTaskSeverity(tasks.length, averageTimeMs); - Severity longTaskSeverity = longTaskSeverity(tasks.length, averageTimeMs); - Severity severity = Severity.max(shortTaskSeverity, longTaskSeverity); - - HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), - _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); - - result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); - result.addResultDetail("Average task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(averageTimeMs).equals("") ? "0 sec" : Statistics.readableTimespan(averageTimeMs))); - result.addResultDetail("Max task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(taskMaxMs).equals("") ? "0 sec" : Statistics.readableTimespan(taskMaxMs)) ); - result.addResultDetail("Min task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(taskMinMs).equals("") ? "0 sec" :Statistics.readableTimespan(taskMinMs))) ; - - return result; - } - - private Severity shortTaskSeverity(long numTasks, long averageTimeMs) { - // We want to identify jobs with short task runtime - Severity severity = getShortRuntimeSeverity(averageTimeMs); - // Severity is reduced if number of tasks is small. - Severity numTaskSeverity = getNumTasksSeverity(numTasks); - return Severity.min(severity, numTaskSeverity); - } - - private Severity longTaskSeverity(long numTasks, long averageTimeMs) { - // We want to identify jobs with long task runtime. Severity is NOT reduced if num of tasks is large - return getLongRuntimeSeverity(averageTimeMs); - } - - private Severity getShortRuntimeSeverity(long runtimeMs) { - return Severity.getSeverityDescending( - runtimeMs, shortRuntimeLimits[0], shortRuntimeLimits[1], shortRuntimeLimits[2], shortRuntimeLimits[3]); - } - - private Severity getLongRuntimeSeverity(long runtimeMs) { - return Severity.getSeverityAscending( - runtimeMs, longRuntimeLimits[0], longRuntimeLimits[1], longRuntimeLimits[2], longRuntimeLimits[3]); - } - - private Severity getNumTasksSeverity(long numTasks) { - return Severity.getSeverityAscending( - numTasks, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], numTasksLimits[3]); - } - -} - diff --git a/app/com/linkedin/drelephant/util/InfoExtractor.java b/app/com/linkedin/drelephant/util/InfoExtractor.java index 6ff0f4d1b..9824b79b2 100644 --- a/app/com/linkedin/drelephant/util/InfoExtractor.java +++ b/app/com/linkedin/drelephant/util/InfoExtractor.java @@ -20,10 +20,6 @@ import com.linkedin.drelephant.clients.WorkflowClient; import com.linkedin.drelephant.configurations.scheduler.SchedulerConfiguration; import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData; - -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.clients.WorkflowClient; - import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; import com.linkedin.drelephant.schedulers.Scheduler; import com.linkedin.drelephant.spark.data.SparkApplicationData; @@ -120,9 +116,6 @@ public static void loadInfo(AppResult result, HadoopApplicationData data) { } else if ( data instanceof SparkApplicationData) { properties = retrieveSparkProperties((SparkApplicationData) data); } - else if(data instanceof TezApplicationData){ - properties = retrieveTezProperties((TezApplicationData) data); - } Scheduler scheduler = getSchedulerInstance(data.getAppId(), properties); if (scheduler == null) { @@ -173,10 +166,6 @@ public static Properties retrieveMapreduceProperties(MapReduceApplicationData ap return appData.getConf(); } - public static Properties retrieveTezProperties(TezApplicationData appData) { - return appData.getConf(); - } - /** * Populates the given app result with the info from the given application data and scheduler. * diff --git a/app/com/linkedin/drelephant/util/ThreadContextMR2.java b/app/com/linkedin/drelephant/util/ThreadContextMR2.java deleted file mode 100644 index a448a24fe..000000000 --- a/app/com/linkedin/drelephant/util/ThreadContextMR2.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.linkedin.drelephant.util; - -import com.linkedin.drelephant.math.Statistics; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.log4j.Logger; -import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.map.ObjectMapper; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public final class ThreadContextMR2 { - private static final Logger logger = Logger.getLogger(com.linkedin.drelephant.util.ThreadContextMR2.class); - - private static final AtomicInteger THREAD_ID = new AtomicInteger(1); - - private static final ThreadLocal _LOCAL_THREAD_ID = new ThreadLocal() { - @Override - public Integer initialValue() { - return THREAD_ID.getAndIncrement(); - } - }; - - private static final ThreadLocal _LOCAL_LAST_UPDATED = new ThreadLocal(); - private static final ThreadLocal _LOCAL_UPDATE_INTERVAL = new ThreadLocal(); - - private static final ThreadLocal _LOCAL_DIAGNOSTIC_PATTERN = new ThreadLocal() { - @Override - public Pattern initialValue() { - // Example: "Task task_1443068695259_9143_m_000475 failed 1 times" - return Pattern.compile( - ".*[\\s\\u00A0]+(task_[0-9]+_[0-9]+_[m|r]_[0-9]+)[\\s\\u00A0]+.*"); - } - }; - - private static final ThreadLocal _LOCAL_AUTH_TOKEN = - new ThreadLocal() { - @Override - public AuthenticatedURL.Token initialValue() { - _LOCAL_LAST_UPDATED.set(System.currentTimeMillis()); - // Random an interval for each executor to avoid update token at the same time - _LOCAL_UPDATE_INTERVAL.set(Statistics.MINUTE_IN_MS * 30 + new Random().nextLong() - % (3 * Statistics.MINUTE_IN_MS)); - logger.info("Executor " + _LOCAL_THREAD_ID.get() + " update interval " + _LOCAL_UPDATE_INTERVAL.get() * 1.0 - / Statistics.MINUTE_IN_MS); - return new AuthenticatedURL.Token(); - } - }; - - private static final ThreadLocal _LOCAL_AUTH_URL = new ThreadLocal() { - @Override - public AuthenticatedURL initialValue() { - return new AuthenticatedURL(); - } - }; - - private static final ThreadLocal _LOCAL_MAPPER = new ThreadLocal() { - @Override - public ObjectMapper initialValue() { - return new ObjectMapper(); - } - }; - - private ThreadContextMR2() { - // Empty on purpose - } - - public static Matcher getDiagnosticMatcher(String diagnosticInfo) { - return _LOCAL_DIAGNOSTIC_PATTERN.get().matcher(diagnosticInfo); - } - - public static JsonNode readJsonNode(URL url) throws IOException, AuthenticationException { - HttpURLConnection conn = _LOCAL_AUTH_URL.get().openConnection(url, _LOCAL_AUTH_TOKEN.get()); - return _LOCAL_MAPPER.get().readTree(conn.getInputStream()); - } - - public static void updateAuthToken() { - long curTime = System.currentTimeMillis(); - if (curTime - _LOCAL_LAST_UPDATED.get() > _LOCAL_UPDATE_INTERVAL.get()) { - logger.info("Executor " + _LOCAL_THREAD_ID.get() + " updates its AuthenticatedToken."); - _LOCAL_AUTH_TOKEN.set(new AuthenticatedURL.Token()); - _LOCAL_AUTH_URL.set(new AuthenticatedURL()); - _LOCAL_LAST_UPDATED.set(curTime); - } - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java b/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java index 8ce84bec1..531bb3724 100644 --- a/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java +++ b/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java @@ -18,8 +18,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; - -import com.linkedin.drelephant.util.ThreadContextMR2; import org.junit.Assert; import org.junit.Test; diff --git a/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java b/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java deleted file mode 100644 index 4a42a8617..000000000 --- a/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez; - - -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import org.junit.Assert; -import org.junit.Test; - -public class TezTaskLevelAggregatedMetricsTest { - - @Test - public void testZeroTasks() { - TezTaskData taskData[] = {}; - TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(taskData, 0, 0); - Assert.assertEquals(taskMetrics.getDelay(), 0); - Assert.assertEquals(taskMetrics.getResourceUsed(), 0); - Assert.assertEquals(taskMetrics.getResourceWasted(), 0); - } - - @Test - public void testNullTaskArray() { - TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(null, 0, 0); - Assert.assertEquals(taskMetrics.getDelay(), 0); - Assert.assertEquals(taskMetrics.getResourceUsed(), 0); - Assert.assertEquals(taskMetrics.getResourceWasted(), 0); - } - - @Test - public void testTaskLevelData() { - TezTaskData taskData[] = new TezTaskData[3]; - TezCounterData counterData = new TezCounterData(); - counterData.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, 655577088L); - counterData.set(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES, 3051589632L); - long time[] = {0,0,0,1464218501117L, 1464218534148L}; - taskData[0] = new TezTaskData("task", "id"); - taskData[0].setTimeAndCounter(time,counterData); - taskData[1] = new TezTaskData("task", "id"); - taskData[1].setTimeAndCounter(new long[5],counterData); - // Non-sampled task, which does not contain time and counter data - taskData[2] = new TezTaskData("task", "id"); - TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(taskData, 4096L, 1463218501117L); - Assert.assertEquals(taskMetrics.getDelay(), 1000000000L); - Assert.assertEquals(taskMetrics.getResourceUsed(), 135168L); - Assert.assertEquals(taskMetrics.getResourceWasted(), 66627L); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java b/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java deleted file mode 100644 index bf38f10c3..000000000 --- a/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.fetchers; - -import java.util.regex.Matcher; -import org.junit.Assert; -import org.junit.Test; -import com.linkedin.drelephant.util.ThreadContextMR2; - -public class TezFetcherTest { - - @Test - public void testDiagnosticMatcher() { - Matcher matcher = ThreadContextMR2.getDiagnosticMatcher("Task task_1443068695259_9143_m_000475 failed 1 time"); - Assert.assertEquals(".*[\\s\\u00A0]+(task_[0-9]+_[0-9]+_[m|r]_[0-9]+)[\\s\\u00A0]+.*", matcher.pattern().toString()); - Assert.assertEquals(true, matcher.matches()); - Assert.assertEquals(1, matcher.groupCount()); - Assert.assertEquals("task_1443068695259_9143_m_000475", matcher.group(1)); - } - -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java deleted file mode 100644 index 8792e70f4..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.*; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - - -public class MapperDataSkewHeuristicTest extends TestCase { - - private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64; //1MB - private static final long UNITSIZETIME = 1000000; //1000sec - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperDataSkewHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(200, 200, 1 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(200, 200, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(200, 200, 20 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(200, 200, 30 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 50 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSmallFiles() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 1 * UNITSIZE, 5 * UNITSIZE)); - } - - public void testSmallTasks() throws IOException { - assertEquals(Severity.NONE, analyzeJob(5, 5, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testCriticalTime() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJobTime(200, 200, 1 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSevereTime() throws IOException { - assertEquals(Severity.SEVERE, analyzeJobTime(200, 200, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testModerateTime() throws IOException { - assertEquals(Severity.MODERATE, analyzeJobTime(200, 200, 20 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testLowTime() throws IOException { - assertEquals(Severity.LOW, analyzeJobTime(200, 200, 30 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testNoneTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(200, 200, 50 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSmallTasksTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(5, 5, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) - throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[numSmallTasks + numLargeTasks + 1]; - - TezCounterData smallCounter = new TezCounterData(); - smallCounter.set(TezCounterData.CounterName.HDFS_BYTES_READ, smallInputSize); - - TezCounterData largeCounter = new TezCounterData(); - largeCounter.set(TezCounterData.CounterName.S3A_BYTES_READ, largeInputSize); - - int i = 0; - for (; i < numSmallTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], smallCounter); - } - for (; i < numSmallTasks + numLargeTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], largeCounter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } - - private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) - throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[numSmallTasks + numLongTasks + 1]; - - int i = 0; - for (; i < numSmallTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTotalTimeMs(smallTimeTaken, true); - mappers[i].setCounter(jobCounter); - } - for (; i < numSmallTasks + numLongTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTotalTimeMs(longTimeTaken, true); - mappers[i].setCounter(jobCounter); - } - // Non-sampled task, which does not contain time data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java deleted file mode 100644 index 52cea2035..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class MapperGCHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - - private static Heuristic _heuristic = new MapperGCHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static int NUMTASKS = 100; - - public void testGCCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1000000, 50000, 2000)); - } - - public void testGCSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1000000, 50000, 1500)); - } - - public void testGCModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1000000, 50000, 1000)); - } - - public void testGCNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1000000, 50000, 300)); - } - - public void testShortTasksNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(100000, 50000, 2000)); - } - - - private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.CPU_MILLISECONDS, cpuMs); - counter.set(TezCounterData.CounterName.GC_TIME_MILLIS, gcMs); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[]{runtimeMs, 0 , 0, 0, 0}, counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } - -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java deleted file mode 100644 index 72e426743..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class MapperMemoryHeuristicTest extends TestCase{ - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperMemoryHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private int NUMTASKS = 100; - - public void testLargeContainerSizeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(2048, 8192)); - } - - public void testLargeContainerSizeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(3072, 8192)); - } - - public void testLargeContainerSizeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(4096, 8192)); - } - - public void testLargeContainerSizeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(6144, 8192)); - } - - // If the task use default container size, it should not be flagged - // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> CRITICAL - public void testDefaultContainerNone() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(256, 2048)); - } - - // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> MODERATE - public void testDefaultContainerNoneMore() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1024, 2048)); - } - - private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB* FileUtils.ONE_MB); - - Properties p = new Properties(); - p.setProperty(MapperMemoryHeuristic.MAPPER_MEMORY_CONF, Long.toString(containerMemMB)); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTime(new long[5]); - mappers[i].setCounter(counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - data.setConf(p); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java deleted file mode 100644 index 2d21f856a..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class MapperSpeedHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperSpeedHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static final long MB_IN_BYTES = FileUtils.ONE_MB; - private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS; - private static final int NUMTASKS = 100; - - public void testCritical() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.CRITICAL, analyzeJob(runtime, 1 * speed_factor)); - } - - public void testSevere() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.SEVERE, analyzeJob(runtime, 4 * speed_factor)); - } - - public void testModerate() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.MODERATE, analyzeJob(runtime, 13 * speed_factor)); - } - - public void testLow() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.LOW, analyzeJob(runtime, 50 * speed_factor)); - } - - public void testNone() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.NONE, analyzeJob(runtime, 51 * speed_factor)); - } - - public void testShortTask() throws IOException { - long runtime = 2 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.NONE, analyzeJob(runtime, 1 * speed_factor)); - } - - private Severity analyzeJob(long runtimeMs, long readBytes) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.HDFS_BYTES_READ, readBytes / 2); - counter.set(TezCounterData.CounterName.S3A_BYTES_READ, readBytes / 2); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new TezTaskData(counter, new long[] { runtimeMs, 0, 0 ,0, 0}); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java deleted file mode 100644 index 62831a727..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - - -public class MapperSpillHeuristicTest extends TestCase{ - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperSpillHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - // Spill ratio 3.0, 1000 tasks - assertEquals(Severity.CRITICAL, analyzeJob(3000, 1000, 1000)); - } - - public void testSevere() throws IOException { - // Spill ratio 2.5, 1000 tasks - assertEquals(Severity.SEVERE, analyzeJob(2500, 1000, 1000)); - } - - public void testModerate() throws IOException { - // Spill ratio 2.3, 1000 tasks - assertEquals(Severity.MODERATE, analyzeJob(2300, 1000, 1000)); - } - - public void testLow() throws IOException { - // Spill ratio 2.1, 1000 tasks - assertEquals(Severity.LOW, analyzeJob(2100, 1000, 1000)); - } - - public void testNone() throws IOException { - // Spill ratio 1.0, 1000 tasks - assertEquals(Severity.NONE, analyzeJob(1000, 1000, 1000)); - } - - public void testSmallNumTasks() throws IOException { - // Spill ratio 3.0, should be critical, but number of task is small(10), final result is NONE - assertEquals(Severity.NONE, analyzeJob(3000, 1000, 10)); - } - - private Severity analyzeJob(long spilledRecords, long mapRecords, int numTasks) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[numTasks + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.SPILLED_RECORDS, spilledRecords); - counter.set(TezCounterData.CounterName.OUTPUT_RECORDS, mapRecords); - - int i = 0; - for (; i < numTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java deleted file mode 100644 index 39c6ec891..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - - -public class MapperTimeHeuristicTest extends TestCase { - - private static final long DUMMY_INPUT_SIZE = 0; - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperTimeHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - // Test batch 1: Large runtime. Heuristic is not affected by various number of tasks */ - - public void testLongRuntimeTasksCritical() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(10, 120 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksCriticalMore() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(1000, 120 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksSevere() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.SEVERE, analyzeJob(10, 60 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksSevereMore() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.SEVERE, analyzeJob(1000, 60 * Statistics.MINUTE_IN_MS)); - } - - // Test batch 2: Short runtime and various number of tasks - - public void testShortRuntimeTasksCritical() throws IOException { - // Should increase split size and decrease number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(1000, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksSevere() throws IOException { - // Should increase split size and decrease number of tasks - assertEquals(Severity.SEVERE, analyzeJob(500, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(101, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(50, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksNone() throws IOException { - // Small file with small number of tasks and short runtime. This should be the common case. - assertEquals(Severity.NONE, analyzeJob(5, 1 * Statistics.MINUTE_IN_MS)); - } - - private Severity analyzeJob(int numTasks, long runtime) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[numTasks + 1]; - - TezCounterData taskCounter = new TezCounterData(); - taskCounter.set(TezCounterData.CounterName.S3A_BYTES_READ, DUMMY_INPUT_SIZE / 4); - - - int i = 0; - for (; i < numTasks; i++) { - mappers[i] = new TezTaskData(jobCounter,new long[] { runtime, 0, 0, 0, 0 }); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } - - -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java deleted file mode 100644 index ff95e862a..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.*; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class ReducerDataSkewHeuristicTest extends TestCase{ - - private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64; //1mb - private static final long UNITSIZETIME = 1000000; //1000sec - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerDataSkewHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(200, 200, 1 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(200, 200, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(200, 200, 20 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(200, 200, 30 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 50 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSmallFiles() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 1 * UNITSIZE, 5 * UNITSIZE)); - } - - public void testSmallTasks() throws IOException { - assertEquals(Severity.NONE, analyzeJob(5, 5, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testCriticalTime() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJobTime(200, 200, 1 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSevereTime() throws IOException { - assertEquals(Severity.SEVERE, analyzeJobTime(200, 200, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testModerateTime() throws IOException { - assertEquals(Severity.MODERATE, analyzeJobTime(200, 200, 20 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testLowTime() throws IOException { - assertEquals(Severity.LOW, analyzeJobTime(200, 200, 30 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testNoneTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(200, 200, 50 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSmallTasksTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(5, 5, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) - throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[numSmallTasks + numLargeTasks + 1]; - - TezCounterData smallCounter = new TezCounterData(); - smallCounter.set(TezCounterData.CounterName.SHUFFLE_BYTES, smallInputSize); - - TezCounterData largeCounter = new TezCounterData(); - largeCounter.set(TezCounterData.CounterName.SHUFFLE_BYTES, largeInputSize); - - int i = 0; - for (; i < numSmallTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], smallCounter); - } - for (; i < numSmallTasks + numLargeTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], largeCounter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } - - private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) - throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[numSmallTasks + numLongTasks + 1]; - - int i = 0; - for (; i < numSmallTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTotalTimeMs(smallTimeTaken, true); - reducers[i].setCounter(jobCounter); - } - for (; i < numSmallTasks + numLongTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTotalTimeMs(longTimeTaken, true); - reducers[i].setCounter(jobCounter); - } - // Non-sampled task, which does not contain time data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java deleted file mode 100644 index 4a86fbefb..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class ReducerGCHeuristicTest extends TestCase{ - - private static Map paramsMap = new HashMap(); - - private static Heuristic _heuristic = new ReducerGCHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static int NUMTASKS = 100; - - public void testGCCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1000000, 50000, 2000)); - } - - public void testGCSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1000000, 50000, 1500)); - } - - public void testGCModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1000000, 50000, 1000)); - } - - public void testGCNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1000000, 50000, 300)); - } - - public void testShortTasksNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(100000, 50000, 2000)); - } - - - private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.CPU_MILLISECONDS, cpuMs); - counter.set(TezCounterData.CounterName.GC_TIME_MILLIS, gcMs); - - int i = 0; - for (; i < NUMTASKS; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java deleted file mode 100644 index 66c3e193c..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class ReducerMemoryHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerMemoryHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private int NUMTASKS = 100; - - public void testLargeContainerSizeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(2048, 8192)); - } - - public void testLargeContainerSizeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(3072, 8192)); - } - - public void testLargeContainerSizeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(4096, 8192)); - } - - public void testLargeContainerSizeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(6144, 8192)); - } - - // If the task use default container size, it should not be flagged - // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> CRITICAL - public void testDefaultContainerNone() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(256, 2048)); - } - - // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> MODERATE - public void testDefaultContainerNoneMore() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1024, 2048)); - } - - private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB* FileUtils.ONE_MB); - - Properties p = new Properties(); - p.setProperty(com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic.REDUCER_MEMORY_CONF, Long.toString(containerMemMB)); - - int i = 0; - for (; i < NUMTASKS; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], counter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); - data.setConf(p); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java deleted file mode 100644 index ab0abbe38..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - - -public class ReducerTimeHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerTimeHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS;; - - public void testShortRunetimeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1 * MINUTE_IN_MS, 1000)); - } - - public void testShortRunetimeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1 * MINUTE_IN_MS, 500)); - } - - public void testShortRunetimeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1 * MINUTE_IN_MS, 101)); - } - - public void testShortRunetimeLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(1 * MINUTE_IN_MS, 50)); - } - - public void testShortRunetimeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1 * MINUTE_IN_MS, 2)); - } - - public void testLongRunetimeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 10)); - } - - // Long runtime severity is not affected by number of tasks - public void testLongRunetimeCriticalMore() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 1000)); - } - - public void testLongRunetimeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(60 * MINUTE_IN_MS, 10)); - } - - public void testLongRunetimeSevereMore() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(60 * MINUTE_IN_MS, 1000)); - } - - private Severity analyzeJob(long runtimeMs, int numTasks) throws IOException { - TezCounterData dummyCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[numTasks + 1]; - - int i = 0; - for (; i < numTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTime(new long[] { runtimeMs, 0, 0, 0, 0 }); - reducers[i].setCounter(dummyCounter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(dummyCounter).setReduceTaskData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file From d5476c1e005b30383a64dc6b3dda9d7d34c94ffd Mon Sep 17 00:00:00 2001 From: Chinmay Sumant Date: Thu, 14 Jun 2018 09:55:08 +0530 Subject: [PATCH 10/15] Rerevert "Dr. Elephant Tez Support working patch (#313)" including attribution. This reverts commit e3fd598d048838006f2da26fe64538bd39d746ed. Co-authored-by: Abhishek Das --- app-conf/AggregatorConf.xml | 4 + app-conf/FetcherConf.xml | 7 + app-conf/HeuristicConf.xml | 117 ++++++ app-conf/JobTypeConf.xml | 6 + .../fetchers/MapReduceFetcherHadoop2.java | 84 +--- .../drelephant/tez/TezMetricsAggregator.java | 109 +++++ .../tez/TezTaskLevelAggregatedMetrics.java | 154 +++++++ .../tez/data/TezApplicationData.java | 137 ++++++ .../drelephant/tez/data/TezCounterData.java | 195 +++++++++ .../drelephant/tez/data/TezTaskData.java | 137 ++++++ .../drelephant/tez/fetchers/TezFetcher.java | 391 ++++++++++++++++++ .../heuristics/GenericDataSkewHeuristic.java | 215 ++++++++++ .../tez/heuristics/GenericGCHeuristic.java | 142 +++++++ .../heuristics/GenericMemoryHeuristic.java | 185 +++++++++ .../heuristics/MapperDataSkewHeuristic.java | 47 +++ .../tez/heuristics/MapperGCHeuristic.java | 51 +++ .../tez/heuristics/MapperMemoryHeuristic.java | 45 ++ .../tez/heuristics/MapperSpeedHeuristic.java | 165 ++++++++ .../tez/heuristics/MapperSpillHeuristic.java | 142 +++++++ .../tez/heuristics/MapperTimeHeuristic.java | 182 ++++++++ .../heuristics/ReducerDataSkewHeuristic.java | 42 ++ .../tez/heuristics/ReducerGCHeuristic.java | 41 ++ .../heuristics/ReducerMemoryHeuristic.java | 47 +++ .../tez/heuristics/ReducerTimeHeuristic.java | 167 ++++++++ .../drelephant/util/InfoExtractor.java | 11 + .../drelephant/util/ThreadContextMR2.java | 92 +++++ .../fetchers/MapReduceFetcherHadoop2Test.java | 2 + .../TezTaskLevelAggregatedMetricsTest.java | 62 +++ .../tez/fetchers/TezFetcherTest.java | 35 ++ .../MapperDataSkewHeuristicTest.java | 146 +++++++ .../tez/heuristics/MapperGCHeuristicTest.java | 84 ++++ .../heuristics/MapperMemoryHeuristicTest.java | 94 +++++ .../heuristics/MapperSpeedHeuristicTest.java | 100 +++++ .../heuristics/MapperSpillHeuristicTest.java | 90 ++++ .../heuristics/MapperTimeHeuristicTest.java | 111 +++++ .../ReducerDataSkewHeuristicTest.java | 142 +++++++ .../heuristics/ReducerGCHeuristicTest.java | 83 ++++ .../ReducerMemoryHeuristicTest.java | 93 +++++ .../heuristics/ReducerTimeHeuristicTest.java | 97 +++++ 39 files changed, 3971 insertions(+), 83 deletions(-) create mode 100644 app/com/linkedin/drelephant/tez/TezMetricsAggregator.java create mode 100644 app/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetrics.java create mode 100644 app/com/linkedin/drelephant/tez/data/TezApplicationData.java create mode 100644 app/com/linkedin/drelephant/tez/data/TezCounterData.java create mode 100644 app/com/linkedin/drelephant/tez/data/TezTaskData.java create mode 100644 app/com/linkedin/drelephant/tez/fetchers/TezFetcher.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/GenericDataSkewHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/GenericGCHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java create mode 100644 app/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristic.java create mode 100644 app/com/linkedin/drelephant/util/ThreadContextMR2.java create mode 100644 test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java create mode 100644 test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java create mode 100644 test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java diff --git a/app-conf/AggregatorConf.xml b/app-conf/AggregatorConf.xml index 23586d587..1536b2d96 100644 --- a/app-conf/AggregatorConf.xml +++ b/app-conf/AggregatorConf.xml @@ -33,6 +33,10 @@ mapreduce com.linkedin.drelephant.mapreduce.MapReduceMetricsAggregator + + tez + com.linkedin.drelephant.tez.TezMetricsAggregator + spark com.linkedin.drelephant.spark.SparkMetricsAggregator diff --git a/app-conf/FetcherConf.xml b/app-conf/FetcherConf.xml index 1f813cf15..89dafc887 100644 --- a/app-conf/FetcherConf.xml +++ b/app-conf/FetcherConf.xml @@ -29,6 +29,13 @@ --> + + + tez + com.linkedin.drelephant.tez.fetchers.TezFetcher + + + + + tez + Mapper Data Skew + com.linkedin.drelephant.tez.heuristics.MapperDataSkewHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Mapper GC + com.linkedin.drelephant.tez.heuristics.MapperGCHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Mapper Time + com.linkedin.drelephant.tez.heuristics.MapperTimeHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Mapper Speed + com.linkedin.drelephant.tez.heuristics.MapperSpeedHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Mapper Memory + com.linkedin.drelephant.tez.heuristics.MapperMemoryHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Mapper Spill + com.linkedin.drelephant.tez.heuristics.MapperSpillHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Reducer Data Skew + com.linkedin.drelephant.tez.heuristics.ReducerDataSkewHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Reducer GC + com.linkedin.drelephant.tez.heuristics.ReducerGCHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Reducer Time + com.linkedin.drelephant.tez.heuristics.ReducerTimeHeuristic + views.html.help.mapreduce.helpMapperSpill + + + + + tez + Reducer Memory + com.linkedin.drelephant.tez.heuristics.ReducerMemoryHeuristic + views.html.help.mapreduce.helpMapperSpill + + + diff --git a/app-conf/JobTypeConf.xml b/app-conf/JobTypeConf.xml index 8a4cae3eb..9dc4aa5ba 100644 --- a/app-conf/JobTypeConf.xml +++ b/app-conf/JobTypeConf.xml @@ -42,6 +42,12 @@ mapreduce pig.script + + Tez + tez + hive.mapred.mode + + Hive mapreduce diff --git a/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2.java b/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2.java index 4165971aa..5a8a64425 100644 --- a/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2.java +++ b/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2.java @@ -20,30 +20,23 @@ import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import com.linkedin.drelephant.math.Statistics; import com.linkedin.drelephant.configurations.fetcher.FetcherConfigurationData; +import com.linkedin.drelephant.util.ThreadContextMR2; import com.linkedin.drelephant.util.Utils; import java.io.IOException; -import java.lang.Integer; -import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.util.ArrayList; import java.util.List; import java.util.Properties; -import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.AuthenticatedURL; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.log4j.Logger; import org.codehaus.jackson.JsonNode; -import org.codehaus.jackson.map.ObjectMapper; /** @@ -387,78 +380,3 @@ private JsonNode getTaskFirstFailedAttempt(URL taskAllAttemptsUrl) throws IOExce } } } - -final class ThreadContextMR2 { - private static final Logger logger = Logger.getLogger(ThreadContextMR2.class); - private static final AtomicInteger THREAD_ID = new AtomicInteger(1); - - private static final ThreadLocal _LOCAL_THREAD_ID = new ThreadLocal() { - @Override - public Integer initialValue() { - return THREAD_ID.getAndIncrement(); - } - }; - - private static final ThreadLocal _LOCAL_LAST_UPDATED = new ThreadLocal(); - private static final ThreadLocal _LOCAL_UPDATE_INTERVAL = new ThreadLocal(); - - private static final ThreadLocal _LOCAL_DIAGNOSTIC_PATTERN = new ThreadLocal() { - @Override - public Pattern initialValue() { - // Example: "Task task_1443068695259_9143_m_000475 failed 1 times" - return Pattern.compile( - ".*[\\s\\u00A0]+(task_[0-9]+_[0-9]+_[m|r]_[0-9]+)[\\s\\u00A0]+.*"); - } - }; - - private static final ThreadLocal _LOCAL_AUTH_TOKEN = - new ThreadLocal() { - @Override - public AuthenticatedURL.Token initialValue() { - _LOCAL_LAST_UPDATED.set(System.currentTimeMillis()); - // Random an interval for each executor to avoid update token at the same time - _LOCAL_UPDATE_INTERVAL.set(Statistics.MINUTE_IN_MS * 30 + new Random().nextLong() - % (3 * Statistics.MINUTE_IN_MS)); - logger.info("Executor " + _LOCAL_THREAD_ID.get() + " update interval " + _LOCAL_UPDATE_INTERVAL.get() * 1.0 - / Statistics.MINUTE_IN_MS); - return new AuthenticatedURL.Token(); - } - }; - - private static final ThreadLocal _LOCAL_AUTH_URL = new ThreadLocal() { - @Override - public AuthenticatedURL initialValue() { - return new AuthenticatedURL(); - } - }; - - private static final ThreadLocal _LOCAL_MAPPER = new ThreadLocal() { - @Override - public ObjectMapper initialValue() { - return new ObjectMapper(); - } - }; - - private ThreadContextMR2() { - // Empty on purpose - } - - public static Matcher getDiagnosticMatcher(String diagnosticInfo) { - return _LOCAL_DIAGNOSTIC_PATTERN.get().matcher(diagnosticInfo); - } - - public static JsonNode readJsonNode(URL url) throws IOException, AuthenticationException { - HttpURLConnection conn = _LOCAL_AUTH_URL.get().openConnection(url, _LOCAL_AUTH_TOKEN.get()); - return _LOCAL_MAPPER.get().readTree(conn.getInputStream()); - } - - public static void updateAuthToken() { - long curTime = System.currentTimeMillis(); - if (curTime - _LOCAL_LAST_UPDATED.get() > _LOCAL_UPDATE_INTERVAL.get()) { - logger.info("Executor " + _LOCAL_THREAD_ID.get() + " updates its AuthenticatedToken."); - _LOCAL_AUTH_TOKEN.set(new AuthenticatedURL.Token()); - _LOCAL_AUTH_URL.set(new AuthenticatedURL()); - _LOCAL_LAST_UPDATED.set(curTime); - } - } -} diff --git a/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java b/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java new file mode 100644 index 000000000..c04edd7fd --- /dev/null +++ b/app/com/linkedin/drelephant/tez/TezMetricsAggregator.java @@ -0,0 +1,109 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez; + +import com.linkedin.drelephant.analysis.*; +import com.linkedin.drelephant.configurations.aggregator.AggregatorConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +/** + * Aggregates task level metrics to application + */ + +public class TezMetricsAggregator implements HadoopMetricsAggregator { + + private static final Logger logger = Logger.getLogger(TezMetricsAggregator.class); + + private static final String TEZ_CONTAINER_CONFIG = "hive.tez.container.size"; + private static final String MAP_CONTAINER_CONFIG = "mapreduce.map.memory.mb"; + private static final String REDUCER_CONTAINER_CONFIG = "mapreduce.reduce.memory.mb"; + private static final String REDUCER_SLOW_START_CONFIG = "mapreduce.job.reduce.slowstart.completedmaps"; + private static final long CONTAINER_MEMORY_DEFAULT_BYTES = 2048L * FileUtils.ONE_MB; + + private HadoopAggregatedData _hadoopAggregatedData = null; + private TezTaskLevelAggregatedMetrics _mapTasks; + private TezTaskLevelAggregatedMetrics _reduceTasks; + + private AggregatorConfigurationData _aggregatorConfigurationData; + + public TezMetricsAggregator(AggregatorConfigurationData _aggregatorConfigurationData) { + this._aggregatorConfigurationData = _aggregatorConfigurationData; + _hadoopAggregatedData = new HadoopAggregatedData(); + } + + @Override + public void aggregate(HadoopApplicationData hadoopData) { + + TezApplicationData data = (TezApplicationData) hadoopData; + + long mapTaskContainerSize = getMapContainerSize(data); + long reduceTaskContainerSize = getReducerContainerSize(data); + + int reduceTaskSlowStartPercentage = + (int) (Double.parseDouble(data.getConf().getProperty(REDUCER_SLOW_START_CONFIG)) * 100); + + + //overwrite reduceTaskSlowStartPercentage to 100%. TODO: make use of the slow start percent + reduceTaskSlowStartPercentage = 100; + + _mapTasks = new TezTaskLevelAggregatedMetrics(data.getMapTaskData(), mapTaskContainerSize, data.getStartTime()); + + long reduceIdealStartTime = _mapTasks.getNthPercentileFinishTime(reduceTaskSlowStartPercentage); + + // Mappers list is empty + if(reduceIdealStartTime == -1) { + // ideal start time for reducer is infinite since it cannot start + reduceIdealStartTime = Long.MAX_VALUE; + } + + _reduceTasks = new TezTaskLevelAggregatedMetrics(data.getReduceTaskData(), reduceTaskContainerSize, reduceIdealStartTime); + + _hadoopAggregatedData.setResourceUsed(_mapTasks.getResourceUsed() + _reduceTasks.getResourceUsed()); + _hadoopAggregatedData.setTotalDelay(_mapTasks.getDelay() + _reduceTasks.getDelay()); + _hadoopAggregatedData.setResourceWasted(_mapTasks.getResourceWasted() + _reduceTasks.getResourceWasted()); + } + + @Override + public HadoopAggregatedData getResult() { + return _hadoopAggregatedData; + } + + private long getMapContainerSize(HadoopApplicationData data) { + try { + long mapContainerSize = Long.parseLong(data.getConf().getProperty(TEZ_CONTAINER_CONFIG)); + if (mapContainerSize > 0) + return mapContainerSize; + else + return Long.parseLong(data.getConf().getProperty(MAP_CONTAINER_CONFIG)); + } catch ( NumberFormatException ex) { + return CONTAINER_MEMORY_DEFAULT_BYTES; + } + } + + private long getReducerContainerSize(HadoopApplicationData data) { + try { + long reducerContainerSize = Long.parseLong(data.getConf().getProperty(TEZ_CONTAINER_CONFIG)); + if (reducerContainerSize > 0) + return reducerContainerSize; + else + return Long.parseLong(data.getConf().getProperty(REDUCER_CONTAINER_CONFIG)); + } catch ( NumberFormatException ex) { + return CONTAINER_MEMORY_DEFAULT_BYTES; + } + } +} diff --git a/app/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetrics.java b/app/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetrics.java new file mode 100644 index 000000000..0f5eaa5f6 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetrics.java @@ -0,0 +1,154 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez; + +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.math.Statistics; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +/** + * Aggregation functionality for task level metrics + */ + +public class TezTaskLevelAggregatedMetrics { + + private static final Logger logger = Logger.getLogger(TezTaskLevelAggregatedMetrics.class); + + private long _delay = 0; + private long _resourceWasted = 0; + private long _resourceUsed = 0; + + private List finishTimes = new ArrayList(); + private List durations = new ArrayList(); + + private static final double MEMORY_BUFFER = 1.5; + private static final double CLUSTER_MEMORY_FACTOR = 2.1; + + /** + * Returns the nth percentile finish job + * @param percentile The percentile of finish job to return + * @return The nth percentile finish job + */ + public long getNthPercentileFinishTime(int percentile) + { + if(finishTimes == null || finishTimes.size() == 0 ) { + return -1; + } + return Statistics.percentile(finishTimes, percentile); + } + + /** + * Constructor for TaskLevelAggregatedMetrics + * @param taskData Array containing the task data for mappers and/or reducers + * @param containerSize The container size of the tasks + * @param idealStartTime The ideal start time for the task. For mappers it is the submit time, for + * reducers, it is the time when the number of completed maps become more than + * the slow start time. + */ + public TezTaskLevelAggregatedMetrics(TezTaskData[] taskData, long containerSize, long idealStartTime) { + compute(taskData, containerSize, idealStartTime); + } + + /** + * Returns the overall delay for the tasks. + * @return The delay of the tasks. + */ + public long getDelay() { + return _delay; + } + + /** + * Retruns the resources wasted by all the tasks in MB Seconds + * @return The wasted resources of all the tasks in MB Seconds + */ + public long getResourceWasted() { + return _resourceWasted; + } + + /** + * Returns the resource used by all the tasks in MB Seconds + * @return The total resources used by all tasks in MB Seconds + */ + public long getResourceUsed() { + return _resourceUsed; + } + + /** + * Computes the aggregated metrics -> peakMemory, delay, total task duration, wasted resources and memory usage. + * @param taskDatas + * @param containerSize + * @param idealStartTime + */ + private void compute(TezTaskData[] taskDatas, long containerSize, long idealStartTime) { + + long peakMemoryNeed = 0; + long taskFinishTimeMax = 0; + long taskDurationMax = 0; + + // if there are zero tasks, then nothing to compute. + if(taskDatas == null || taskDatas.length == 0) { + return; + } + + for (TezTaskData taskData: taskDatas) { + if (!taskData.isSampled()) { + continue; + } + long taskMemory = taskData.getCounters().get(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES)/ FileUtils.ONE_MB; // MB + long taskVM = taskData.getCounters().get(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES)/ FileUtils.ONE_MB; // MB + long taskDuration = taskData.getFinishTime() - taskData.getStartTime(); // Milliseconds + long taskCost = (containerSize) * (taskDuration / Statistics.SECOND_IN_MS); // MB Seconds + + durations.add(taskDuration); + finishTimes.add(taskData.getFinishTime()); + + //peak Memory usage + long memoryRequiredForVM = (long) (taskVM/CLUSTER_MEMORY_FACTOR); + long biggerMemoryRequirement = memoryRequiredForVM > taskMemory ? memoryRequiredForVM : taskMemory; + peakMemoryNeed = biggerMemoryRequirement > peakMemoryNeed ? biggerMemoryRequirement : peakMemoryNeed; + + if(taskFinishTimeMax < taskData.getFinishTime()) { + taskFinishTimeMax = taskData.getFinishTime(); + } + + if(taskDurationMax < taskDuration) { + taskDurationMax = taskDuration; + } + _resourceUsed += taskCost; + } + + // Compute the delay in starting the task. + _delay = taskFinishTimeMax - (idealStartTime + taskDurationMax); + + // invalid delay + if(_delay < 0) { + _delay = 0; + } + + // wastedResources + long wastedMemory = containerSize - (long) (peakMemoryNeed * MEMORY_BUFFER); + if(wastedMemory > 0) { + for (long duration : durations) { + _resourceWasted += (wastedMemory) * (duration / Statistics.SECOND_IN_MS); // MB Seconds + } + } + } + +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/data/TezApplicationData.java b/app/com/linkedin/drelephant/tez/data/TezApplicationData.java new file mode 100644 index 000000000..f38f527bc --- /dev/null +++ b/app/com/linkedin/drelephant/tez/data/TezApplicationData.java @@ -0,0 +1,137 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.data; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.HadoopApplicationData; + +import java.util.Properties; + +/** + * Tez Application level data structure which hold all task data + */ +public class TezApplicationData implements HadoopApplicationData { + + private static final ApplicationType APPLICATION_TYPE = new ApplicationType("TEZ"); + + private String _appId = ""; + private Properties _conf; + private boolean _succeeded = true; + private TezTaskData[] _reduceTasks; + private TezTaskData[] _mapTasks; + private TezCounterData _counterHolder; + + private long _submitTime = 0; + private long _startTime = 0; + private long _finishTime = 0; + + public boolean getSucceeded() { + return _succeeded; + } + + @Override + public String getAppId() { + return _appId; + } + + @Override + public Properties getConf() { + return _conf; + } + + @Override + public ApplicationType getApplicationType() { + return APPLICATION_TYPE; + } + + @Override + public boolean isEmpty() { + return _succeeded && getMapTaskData().length == 0 && getReduceTaskData().length == 0; + } + + public TezTaskData[] getReduceTaskData() { + return _reduceTasks; + } + + public TezTaskData[] getMapTaskData() { + return _mapTasks; + } + + public long getSubmitTime() { + return _submitTime; + } + + public long getStartTime() { + return _startTime; + } + + public long getFinishTime() { + return _finishTime; + } + + public TezCounterData getCounters() { + return _counterHolder; + } + + public TezApplicationData setCounters(TezCounterData counterHolder) { + this._counterHolder = counterHolder; + return this; + } + + public TezApplicationData setAppId(String appId) { + this._appId = appId; + return this; + } + + public TezApplicationData setConf(Properties conf) { + this._conf = conf; + return this; + } + + public TezApplicationData setSucceeded(boolean succeeded) { + this._succeeded = succeeded; + return this; + } + + public TezApplicationData setReduceTaskData(TezTaskData[] reduceTasks) { + this._reduceTasks = reduceTasks; + return this; + } + + public TezApplicationData setMapTaskData(TezTaskData[] mapTasks) { + this._mapTasks = mapTasks; + return this; + } + + public TezApplicationData setSubmitTime(long submitTime) { + this._submitTime = submitTime; + return this; + } + + public TezApplicationData setStartTime(long startTime) { + this._startTime = startTime; + return this; + } + + public TezApplicationData setFinishTime(long finishTime) { + this._finishTime = finishTime; + return this; + } + + public String toString(){ + return APPLICATION_TYPE.toString() + " " + _appId; + } +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/data/TezCounterData.java b/app/com/linkedin/drelephant/tez/data/TezCounterData.java new file mode 100644 index 000000000..9aac00117 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/data/TezCounterData.java @@ -0,0 +1,195 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.data; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Tez Counter Data defining data structure. + */ +public class TezCounterData { + // Map to group counters into DAG, Task and application levels. + private final Map> _pubCounters; + + public String toString() { + return _pubCounters.toString(); + } + + public TezCounterData() { + _pubCounters = new HashMap>(8); + } + + public long get(CounterName counterName) { + for(Map counterGrp : _pubCounters.values()) { + if(counterGrp.containsKey(counterName._name)) { + return counterGrp.get(counterName._name); + } + } + return 0; + } + + public void set(CounterName counterName, long value) { + set(counterName.getGroupName(), counterName.getName(), value); + } + + public void set(String groupName, String counterName, long value) { + Map counterMap = _pubCounters.get(groupName); + if (counterMap == null) { + counterMap = new HashMap(4); + _pubCounters.put(groupName, counterMap); + } + counterMap.put(counterName, value); + } + + public Set getGroupNames() { + Set groupNames = _pubCounters.keySet(); + return Collections.unmodifiableSet(groupNames); + } + + public Map getAllCountersInGroup(String groupName) { + Map counterMap = _pubCounters.get(groupName); + if (counterMap == null) { + counterMap = new HashMap(1); + } + return counterMap; + } + + public static enum GroupName { + FileSystemCounters("org.apache.tez.common.counters.FileSystemCounter"), + TezTask("org.apache.tez.common.counters.TaskCounter"), + TezDag("org.apache.tez.common.counters.DAGCounter"); + + + String _name; + GroupName(String name) { + _name = name; + } + } + + public static enum CounterName { + + NUM_SUCCEEDED_TASKS(GroupName.TezDag, "NUM_SUCCEEDED_TASKS", "NUM_SUCCEEDED_TASKS"), + TOTAL_LAUNCHED_TASKS(GroupName.TezDag, "TOTAL_LAUNCHED_TASKS", "TOTAL_LAUNCHED_TASKS"), + RACK_LOCAL_TASKS(GroupName.TezDag, "RACK_LOCAL_TASKS", "RACK_LOCAL_TASKS"), + AM_CPU_MILLISECONDS(GroupName.TezDag, "AM_CPU_MILLISECONDS", "AM_CPU_MILLISECONDS"), + AM_GC_TIME_MILLIS(GroupName.TezDag, "AM_GC_TIME_MILLIS", "AM_GC_TIME_MILLIS"), + + FILE_BYTES_READ(GroupName.FileSystemCounters, "FILE_BYTES_READ", "FILE_BYTES_READ"), + FILE_BYTES_WRITTEN(GroupName.FileSystemCounters, "FILE_BYTES_WRITTEN", "FILE_BYTES_WRITTEN"), + FILE_READ_OPS(GroupName.FileSystemCounters, "FILE_READ_OPS", "FILE_READ_OPS"), + FILE_LARGE_READ_OPS(GroupName.FileSystemCounters, "FILE_LARGE_READ_OPS", "FILE_LARGE_READ_OPS"), + FILE_WRITE_OPS(GroupName.FileSystemCounters, "FILE_WRITE_OPS", "FILE_WRITE_OPS"), + HDFS_BYTES_READ(GroupName.FileSystemCounters, "HDFS_BYTES_READ", "HDFS_BYTES_READ"), + HDFS_BYTES_WRITTEN(GroupName.FileSystemCounters, "HDFS_BYTES_WRITTEN", "HDFS_BYTES_WRITTEN"), + HDFS_READ_OPS(GroupName.FileSystemCounters, "HDFS_READ_OPS", "HDFS_READ_OPS"), + HDFS_LARGE_READ_OPS(GroupName.FileSystemCounters, "HDFS_LARGE_READ_OPS", "HDFS_LARGE_READ_OPS"), + HDFS_WRITE_OPS(GroupName.FileSystemCounters, "HDFS_WRITE_OPS", "HDFS_WRITE_OPS"), + S3A_BYTES_READ(GroupName.FileSystemCounters, "S3A_BYTES_READ", "S3A_BYTES_READ"), + S3A_BYTES_WRITTEN(GroupName.FileSystemCounters, "S3A_BYTES_WRITTEN", "S3A_BYTES_WRITTEN"), + S3A_READ_OPS(GroupName.FileSystemCounters, "S3A_READ_OPS", "S3A_READ_OPS"), + S3A_LARGE_READ_OPS(GroupName.FileSystemCounters, "S3A_LARGE_READ_OPS", "S3A_LARGE_READ_OPS"), + S3A_WRITE_OPS(GroupName.FileSystemCounters, "S3A_WRITE_OPS", "S3_WRITE_OPS"), + S3N_BYTES_READ(GroupName.FileSystemCounters, "S3N_BYTES_READ", "S3N_BYTES_READ"), + S3N_BYTES_WRITTEN(GroupName.FileSystemCounters, "S3N_BYTES_WRITTEN", "S3N_BYTES_WRITTEN"), + S3N_READ_OPS(GroupName.FileSystemCounters, "S3N_READ_OPS", "S3N_READ_OPS"), + S3N_LARGE_READ_OPS(GroupName.FileSystemCounters, "S3N_LARGE_READ_OPS", "S3N_LARGE_READ_OPS"), + S3N_WRITE_OPS(GroupName.FileSystemCounters, "S3N_WRITE_OPS", "S3N_WRITE_OPS"), + + REDUCE_INPUT_GROUPS(GroupName.TezTask, "REDUCE_INPUT_GROUPS", "REDUCE_INPUT_GROUPS"), + REDUCE_INPUT_RECORDS(GroupName.TezTask, "REDUCE_INPUT_RECORDS", "REDUCE_INPUT_RECORDS"), + COMBINE_INPUT_RECORDS(GroupName.TezTask, "COMBINE_INPUT_RECORDS", "COMBINE_INPUT_RECORDS"), + SPILLED_RECORDS(GroupName.TezTask, "SPILLED_RECORDS", "SPILLED_RECORDS"), + NUM_SHUFFLED_INPUTS(GroupName.TezTask, "NUM_SHUFFLED_INPUTS", "NUM_SHUFFLED_INPUTS"), + NUM_SKIPPED_INPUTS(GroupName.TezTask, "NUM_SKIPPED_INPUTS", "NUM_SKIPPED_INPUTS"), + NUM_FAILED_SHUFFLE_INPUTS(GroupName.TezTask, "NUM_FAILED_SHUFFLE_INPUTS", "NUM_FAILED_SHUFFLE_INPUTS"), + MERGED_MAP_OUTPUTS(GroupName.TezTask, "MERGED_MAP_OUTPUTS", "MERGED_MAP_OUTPUTS"), + GC_TIME_MILLIS(GroupName.TezTask, "GC_TIME_MILLIS", "GC_TIME_MILLIS"), + COMMITTED_HEAP_BYTES(GroupName.TezTask, "COMMITTED_HEAP_BYTES", "COMMITTED_HEAP_BYTES"), + INPUT_RECORDS_PROCESSED(GroupName.TezTask, "INPUT_RECORDS_PROCESSED", "INPUT_RECORDS_PROCESSED"), + OUTPUT_RECORDS(GroupName.TezTask, "OUTPUT_RECORDS", "OUTPUT_RECORDS"), + OUTPUT_BYTES(GroupName.TezTask, "OUTPUT_BYTES", "OUTPUT_BYTES"), + OUTPUT_BYTES_WITH_OVERHEAD(GroupName.TezTask, "OUTPUT_BYTES_WITH_OVERHEAD", "OUTPUT_BYTES_WITH_OVERHEAD"), + OUTPUT_BYTES_PHYSICAL(GroupName.TezTask, "OUTPUT_BYTES_PHYSICAL", "OUTPUT_BYTES_PHYSICAL"), + ADDITIONAL_SPILLS_BYTES_WRITTEN(GroupName.TezTask, "ADDITIONAL_SPILLS_BYTES_WRITTEN", "ADDITIONAL_SPILLS_BYTES_WRITTEN"), + ADDITIONAL_SPILLS_BYTES_READ(GroupName.TezTask, "ADDITIONAL_SPILLS_BYTES_READ", "ADDITIONAL_SPILLS_BYTES_READ"), + ADDITIONAL_SPILL_COUNT(GroupName.TezTask, "ADDITIONAL_SPILL_COUNT", "ADDITIONAL_SPILL_COUNT"), + SHUFFLE_BYTES(GroupName.TezTask, "SHUFFLE_BYTES", "SHUFFLE_BYTES"), + SHUFFLE_BYTES_DECOMPRESSED(GroupName.TezTask, "SHUFFLE_BYTES_DECOMPRESSED", "SHUFFLE_BYTES_DECOMPRESSED"), + SHUFFLE_BYTES_TO_MEM(GroupName.TezTask, "SHUFFLE_BYTES_TO_MEM", "SHUFFLE_BYTES_TO_MEM"), + SHUFFLE_BYTES_TO_DISK(GroupName.TezTask, "SHUFFLE_BYTES_TO_DISK", "SHUFFLE_BYTES_TO_DISK"), + SHUFFLE_BYTES_DISK_DIRECT(GroupName.TezTask, "SHUFFLE_BYTES_DISK_DIRECT", "SHUFFLE_BYTES_DISK_DIRECT"), + NUM_MEM_TO_DISK_MERGES(GroupName.TezTask, "NUM_MEM_TO_DISK_MERGES", "NUM_MEM_TO_DISK_MERGES"), + CPU_MILLISECONDS(GroupName.TezTask,"CPU_MILLISECONDS","CPU_MILLISECONDS"), + PHYSICAL_MEMORY_BYTES(GroupName.TezTask,"PHYSICAL_MEMORY_BYTES","PHYSICAL_MEMORY_BYTES"), + VIRTUAL_MEMORY_BYTES(GroupName.TezTask,"VIRTUAL_MEMORY_BYTES","VIRTUAL_MEMORY_BYTES"), + NUM_DISK_TO_DISK_MERGES(GroupName.TezTask, "NUM_DISK_TO_DISK_MERGES", "NUM_DISK_TO_DISK_MERGES"), + SHUFFLE_PHASE_TIME(GroupName.TezTask, "SHUFFLE_PHASE_TIME", "SHUFFLE_PHASE_TIME"), + MERGE_PHASE_TIME(GroupName.TezTask, "MERGE_PHASE_TIME", "MERGE_PHASE_TIME"), + FIRST_EVENT_RECEIVED(GroupName.TezTask, "FIRST_EVENT_RECEIVED", "FIRST_EVENT_RECEIVED"), + LAST_EVENT_RECEIVED(GroupName.TezTask, "LAST_EVENT_RECEIVED", "LAST_EVENT_RECEIVED"); + + + GroupName _group; + String _name; + String _displayName; + + CounterName(GroupName group, String name, String displayName) { + this._group = group; + this._name = name; + this._displayName = displayName; + } + + static Map _counterDisplayNameMap; + static Map _counterNameMap; + static { + _counterDisplayNameMap = new HashMap(); + _counterNameMap = new HashMap(); + for (CounterName cn : CounterName.values()) { + _counterDisplayNameMap.put(cn._displayName, cn); + _counterNameMap.put(cn._name, cn); + } + } + + public static CounterName getCounterFromName(String name) { + if (_counterNameMap.containsKey(name)) { + return _counterNameMap.get(name); + } + return null; + } + + public static CounterName getCounterFromDisplayName(String displayName) { + if (_counterDisplayNameMap.containsKey(displayName)) { + return _counterDisplayNameMap.get(displayName); + } + return null; + } + + public String getName() { + return _name; + } + + public String getDisplayName() { + return _displayName; + } + + public String getGroupName() { + return _group.name(); + } + } +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/data/TezTaskData.java b/app/com/linkedin/drelephant/tez/data/TezTaskData.java new file mode 100644 index 000000000..eb6aa832c --- /dev/null +++ b/app/com/linkedin/drelephant/tez/data/TezTaskData.java @@ -0,0 +1,137 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.data; + +/** + * Tez Task Level metadata holding data structure + */ + +public class TezTaskData { + private TezCounterData _counterHolder; + private String _taskId; + // The successful attempt id + private String _attemptId; + + private long _totalTimeMs = 0; + private long _shuffleTimeMs = 0; + private long _sortTimeMs = 0; + private long _startTime = 0; + private long _finishTime = 0; + private boolean _isSampled = false; + + //Constructor used only in Test Cases , if provided to partially assign needed time values while ignoring the others + public TezTaskData(TezCounterData counterHolder, long[] time) { + if(time == null || time.length<3){ + time = new long[5]; + } + this._counterHolder = counterHolder; + this._totalTimeMs = time[0]; + this._shuffleTimeMs = time[1]; + this._sortTimeMs = time[2]; + if (time.length > 3) + this._startTime = time[3]; + if (time.length > 4) + this._finishTime = time[4]; + this._isSampled = true; + } + + public TezTaskData(TezCounterData counterHolder) { + this._counterHolder = counterHolder; + } + + public TezTaskData(String taskId, String taskAttemptId) { + this._taskId = taskId; + this._attemptId = taskAttemptId; + } + + public void setCounter(TezCounterData counterHolder) { + this._counterHolder = counterHolder; + this._isSampled = true; + } + + public void setTime(long[] time) { + //No Validation needed here as time array will always be of fixed length 5 from upstream methods. + this._totalTimeMs = time[0]; + this._shuffleTimeMs = time[1]; + this._sortTimeMs = time[2]; + this._startTime = time[3]; + this._finishTime = time[4]; + this._isSampled = true; + } + + //Used only in Test Cases + public void setTimeAndCounter(long[] time, TezCounterData counterHolder){ + if(time == null || time.length<3){ + time = new long[5]; + } + this._totalTimeMs = time[0]; + this._shuffleTimeMs = time[1]; + this._sortTimeMs = time[2]; + if (time.length > 3) + this._startTime = time[3]; + if (time.length > 4) + this._finishTime = time[4]; + this._isSampled = true; + this._counterHolder = counterHolder; + + + } + + public TezCounterData getCounters() { + return _counterHolder; + } + + public long getTotalRunTimeMs() { + return _totalTimeMs; + } + + public long getCodeExecutionTimeMs() { + return _totalTimeMs - _shuffleTimeMs - _sortTimeMs; + } + + public long getShuffleTimeMs() { + return _shuffleTimeMs; + } + + public long getSortTimeMs() { + return _sortTimeMs; + } + + public boolean isSampled() { + return _isSampled; + } + + public String getTaskId() { + return _taskId; + } + + public String getAttemptId() { + return _attemptId; + } + + public long getStartTime() { + return _startTime; + } + + public long getFinishTime() { + return _finishTime; + } + + public void setTotalTimeMs(long totalTimeMs, boolean isSampled) { + this._totalTimeMs = totalTimeMs; + this._isSampled = isSampled; + } +} diff --git a/app/com/linkedin/drelephant/tez/fetchers/TezFetcher.java b/app/com/linkedin/drelephant/tez/fetchers/TezFetcher.java new file mode 100644 index 000000000..294fe20e0 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/fetchers/TezFetcher.java @@ -0,0 +1,391 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.fetchers; + +import com.linkedin.drelephant.analysis.AnalyticJob; +import com.linkedin.drelephant.analysis.ElephantFetcher; +import com.linkedin.drelephant.configurations.fetcher.FetcherConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.util.ThreadContextMR2; +import org.apache.log4j.Logger; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.codehaus.jackson.JsonNode; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.util.*; + +/** + * Task level data mining for Tez Tasks from timeline server API + */ + +public class TezFetcher implements ElephantFetcher { + + private static final Logger logger = Logger.getLogger(TezFetcher.class); + + private static final String TIMELINE_SERVER_URL = "yarn.timeline-service.webapp.address"; + + private URLFactory _urlFactory; + private JSONFactory _jsonFactory; + private String _timelineWebAddr; + + private FetcherConfigurationData _fetcherConfigurationData; + + public TezFetcher(FetcherConfigurationData fetcherConfData) throws IOException { + this._fetcherConfigurationData = fetcherConfData; + final String applicationHistoryAddr = new Configuration().get(TIMELINE_SERVER_URL); + + //Connection validity checked using method verifyURL(_timelineWebAddr) inside URLFactory constructor; + _urlFactory = new URLFactory(applicationHistoryAddr); + logger.info("Connection success."); + + _jsonFactory = new JSONFactory(); + _timelineWebAddr = "http://" + _timelineWebAddr + "/ws/v1/timeline/"; + + } + + public TezApplicationData fetchData(AnalyticJob analyticJob) throws IOException, AuthenticationException { + + int maxSize = 0; + String appId = analyticJob.getAppId(); + TezApplicationData jobData = new TezApplicationData(); + jobData.setAppId(appId); + Properties jobConf = _jsonFactory.getProperties(_urlFactory.getApplicationURL(appId)); + jobData.setConf(jobConf); + URL dagIdsUrl = _urlFactory.getDagURLByTezApplicationId(appId); + + List dagIdsByApplicationId = _jsonFactory.getDagIdsByApplicationId(dagIdsUrl); + + List mapperListAggregate = new ArrayList(); + List reducerListAggregate = new ArrayList(); + + //Iterate over dagIds and choose the dagId with the highest no. of tasks/highest impact as settings changes can be made only at DAG level. + for(String dagId : dagIdsByApplicationId){ + try { + //set job task independent properties + + URL dagUrl = _urlFactory.getDagURL(dagId); + String state = _jsonFactory.getState(dagUrl); + + jobData.setStartTime(_jsonFactory.getDagStartTime(dagUrl)); + jobData.setFinishTime(_jsonFactory.getDagEndTime(dagUrl)); + + if (state.equals("SUCCEEDED")) { + jobData.setSucceeded(true); + + List mapperList = new ArrayList(); + List reducerList = new ArrayList(); + + // Fetch task data + URL vertexListUrl = _urlFactory.getVertexListURL(dagId); + _jsonFactory.getTaskDataAll(vertexListUrl, dagId, mapperList, reducerList); + + if(mapperList.size() + reducerList.size() > maxSize){ + mapperListAggregate = mapperList; + reducerListAggregate = reducerList; + maxSize = mapperList.size() + reducerList.size(); + } + } + if (state.equals("FAILED")) { + jobData.setSucceeded(false); + } + } + finally { + ThreadContextMR2.updateAuthToken(); + } + } + + TezTaskData[] mapperData = mapperListAggregate.toArray(new TezTaskData[mapperListAggregate.size()]); + TezTaskData[] reducerData = reducerListAggregate.toArray(new TezTaskData[reducerListAggregate.size()]); + + TezCounterData dagCounter = _jsonFactory.getDagCounter(_urlFactory.getDagURL(_jsonFactory.getDagIdsByApplicationId(dagIdsUrl).get(0))); + + jobData.setCounters(dagCounter).setMapTaskData(mapperData).setReduceTaskData(reducerData); + + return jobData; + } + + private URL getTaskListByVertexURL(String dagId, String vertexId) throws MalformedURLException { + return _urlFactory.getTaskListByVertexURL(dagId, vertexId); + } + + private URL getTaskURL(String taskId) throws MalformedURLException { + return _urlFactory.getTasksURL(taskId); + } + + private URL getTaskAttemptURL(String dagId, String taskId, String attemptId) throws MalformedURLException { + return _urlFactory.getTaskAttemptURL(dagId, taskId, attemptId); + } + + private class URLFactory { + + private String _timelineWebAddr; + + private URLFactory(String hserverAddr) throws IOException { + _timelineWebAddr = "http://" + hserverAddr + "/ws/v1/timeline"; + verifyURL(_timelineWebAddr); + } + + private void verifyURL(String url) throws IOException { + final URLConnection connection = new URL(url).openConnection(); + // Check service availability + connection.connect(); + return; + } + + private URL getDagURLByTezApplicationId(String applicationId) throws MalformedURLException { + return new URL(_timelineWebAddr + "/TEZ_DAG_ID?primaryFilter=applicationId:" + applicationId); + } + + private URL getApplicationURL(String applicationId) throws MalformedURLException { + return new URL(_timelineWebAddr + "/TEZ_APPLICATION/tez_" + applicationId); + } + + private URL getDagURL(String dagId) throws MalformedURLException { + return new URL(_timelineWebAddr + "/TEZ_DAG_ID/" + dagId); + } + + private URL getVertexListURL(String dagId) throws MalformedURLException { + return new URL(_timelineWebAddr + "/TEZ_VERTEX_ID?primaryFilter=TEZ_DAG_ID:" + dagId); + } + + private URL getTaskListByVertexURL(String dagId, String vertexId) throws MalformedURLException { + return new URL(_timelineWebAddr + "/TEZ_TASK_ID?primaryFilter=TEZ_DAG_ID:" + dagId + + "&secondaryFilter=TEZ_VERTEX_ID:" + vertexId + "&limit=500000"); + } + + private URL getTasksURL(String taskId) throws MalformedURLException { + return new URL(_timelineWebAddr + "/TEZ_TASK_ID/" + taskId); + } + + private URL getTaskAllAttemptsURL(String dagId, String taskId) throws MalformedURLException { + return new URL(_timelineWebAddr + "/TEZ_TASK_ATTEMPT_ID?primaryFilter=TEZ_DAG_ID:" + dagId + + "&secondaryFilter=TEZ_TASK_ID:" + taskId); + } + + private URL getTaskAttemptURL(String dagId, String taskId, String attemptId) throws MalformedURLException { + return new URL(_timelineWebAddr + "/TEZ_TASK_ATTEMPT_ID/" + attemptId); + } + + } + + /** + * JSONFactory class provides functionality to parse mined job data from timeline server. + */ + + private class JSONFactory { + + private String getState(URL url) throws IOException, AuthenticationException { + JsonNode rootNode = ThreadContextMR2.readJsonNode(url); + return rootNode.path("otherinfo").path("status").getTextValue(); + } + + private Properties getProperties(URL url) throws IOException, AuthenticationException { + Properties jobConf = new Properties(); + JsonNode rootNode = ThreadContextMR2.readJsonNode(url); + JsonNode configs = rootNode.path("otherinfo").path("config"); + Iterator keys = configs.getFieldNames(); + String key = ""; + String value = ""; + while (keys.hasNext()) { + key = keys.next(); + value = configs.get(key).getTextValue(); + jobConf.put(key, value); + } + return jobConf; + } + + private List getDagIdsByApplicationId(URL dagIdsUrl) throws IOException, AuthenticationException { + List dagIds = new ArrayList(); + JsonNode nodes = ThreadContextMR2.readJsonNode(dagIdsUrl).get("entities"); + + for (JsonNode node : nodes) { + String dagId = node.get("entity").getTextValue(); + dagIds.add(dagId); + } + + return dagIds; + } + + private TezCounterData getDagCounter(URL url) throws IOException, AuthenticationException { + TezCounterData holder = new TezCounterData(); + JsonNode rootNode = ThreadContextMR2.readJsonNode(url); + JsonNode groups = rootNode.path("otherinfo").path("counters").path("counterGroups"); + + for (JsonNode group : groups) { + for (JsonNode counter : group.path("counters")) { + String name = counter.get("counterName").getTextValue(); + String groupName = group.get("counterGroupName").getTextValue(); + Long value = counter.get("counterValue").getLongValue(); + holder.set(groupName, name, value); + } + } + + return holder; + } + + private long getDagStartTime(URL url) throws IOException, AuthenticationException { + JsonNode rootNode = ThreadContextMR2.readJsonNode(url); + long startTime = rootNode.path("otherinfo").get("startTime").getLongValue(); + return startTime; + } + + private long getDagEndTime(URL url) throws IOException, AuthenticationException { + JsonNode rootNode = ThreadContextMR2.readJsonNode(url); + long endTime = rootNode.path("otherinfo").get("endTime").getLongValue(); + return endTime; + } + + private void getTaskDataAll(URL vertexListUrl, String dagId, List mapperList, + List reducerList) throws IOException, AuthenticationException { + + JsonNode rootVertexNode = ThreadContextMR2.readJsonNode(vertexListUrl); + JsonNode vertices = rootVertexNode.path("entities"); + boolean isMapVertex = false; + + for (JsonNode vertex : vertices) { + String vertexId = vertex.get("entity").getTextValue(); + String vertexClass = vertex.path("otherinfo").path("processorClassName").getTextValue(); + + if (vertexClass.equals("org.apache.hadoop.hive.ql.exec.tez.MapTezProcessor")) + isMapVertex = true; + else if (vertexClass.equals("org.apache.hadoop.hive.ql.exec.tez.ReduceTezProcessor")) + isMapVertex = false; + + URL tasksByVertexURL = getTaskListByVertexURL(dagId, vertexId); + if(isMapVertex) + getTaskDataByVertexId(tasksByVertexURL, dagId, vertexId, mapperList, true); + else + getTaskDataByVertexId(tasksByVertexURL, dagId, vertexId, reducerList, false); + } + } + + private void getTaskDataByVertexId(URL url, String dagId, String vertexId, List taskList, + boolean isMapVertex) throws IOException, AuthenticationException { + + JsonNode rootNode = ThreadContextMR2.readJsonNode(url); + JsonNode tasks = rootNode.path("entities"); + for (JsonNode task : tasks) { + String state = task.path("otherinfo").path("status").getTextValue(); + String taskId = task.get("entity").getValueAsText(); + String attemptId = task.path("otherinfo").path("successfulAttemptId").getTextValue(); + if (state.equals("SUCCEEDED")) { + attemptId = task.path("otherinfo").path("successfulAttemptId").getTextValue(); + } + else{ + JsonNode firstAttempt = getTaskFirstFailedAttempt(_urlFactory.getTaskAllAttemptsURL(dagId,taskId)); + if(firstAttempt != null){ + attemptId = firstAttempt.get("entity").getTextValue(); + } + } + + taskList.add(new TezTaskData(taskId, attemptId)); + } + + getTaskData(dagId, taskList, isMapVertex); + + } + + private JsonNode getTaskFirstFailedAttempt(URL taskAllAttemptsUrl) throws IOException, AuthenticationException { + JsonNode rootNode = ThreadContextMR2.readJsonNode(taskAllAttemptsUrl); + long firstAttemptFinishTime = Long.MAX_VALUE; + JsonNode firstAttempt = null; + JsonNode taskAttempts = rootNode.path("entities"); + for (JsonNode taskAttempt : taskAttempts) { + String state = taskAttempt.path("otherinfo").path("counters").path("status").getTextValue(); + if (state.equals("SUCCEEDED")) { + continue; + } + long finishTime = taskAttempt.path("otherinfo").path("counters").path("endTime").getLongValue(); + if( finishTime < firstAttemptFinishTime) { + firstAttempt = taskAttempt; + firstAttemptFinishTime = finishTime; + } + } + return firstAttempt; + } + + + + private void getTaskData(String dagId, List taskList, boolean isMapTask) + throws IOException, AuthenticationException { + + for(int i=0; i { + private static final Logger logger = Logger.getLogger(GenericDataSkewHeuristic.class); + + // Severity Parameters + private static final String NUM_TASKS_SEVERITY = "num_tasks_severity"; + private static final String DEVIATION_SEVERITY = "deviation_severity"; + private static final String FILES_SEVERITY = "files_severity"; + + // Default value of parameters + private double[] numTasksLimits = {10, 50, 100, 200}; // Number of map or reduce tasks + private double[] deviationLimits = {2, 4, 8, 16}; // Deviation in i/p bytes btw 2 groups + private double[] filesLimits = {1d / 8, 1d / 4, 1d / 2, 1d}; // Fraction of HDFS Block Size + + private HeuristicConfigurationData _heuristicConfData; + private List _counterNames; + + private void loadParameters() { + Map paramMap = _heuristicConfData.getParamMap(); + String heuristicName = _heuristicConfData.getHeuristicName(); + + double[] confNumTasksThreshold = Utils.getParam(paramMap.get(NUM_TASKS_SEVERITY), numTasksLimits.length); + if (confNumTasksThreshold != null) { + numTasksLimits = confNumTasksThreshold; + } + logger.info(heuristicName + " will use " + NUM_TASKS_SEVERITY + " with the following threshold settings: " + + Arrays.toString(numTasksLimits)); + + double[] confDeviationThreshold = Utils.getParam(paramMap.get(DEVIATION_SEVERITY), deviationLimits.length); + if (confDeviationThreshold != null) { + deviationLimits = confDeviationThreshold; + } + logger.info(heuristicName + " will use " + DEVIATION_SEVERITY + " with the following threshold settings: " + + Arrays.toString(deviationLimits)); + + double[] confFilesThreshold = Utils.getParam(paramMap.get(FILES_SEVERITY), filesLimits.length); + if (confFilesThreshold != null) { + filesLimits = confFilesThreshold; + } + logger.info( + heuristicName + " will use " + FILES_SEVERITY + " with the following threshold settings: " + Arrays.toString( + filesLimits)); + for (int i = 0; i < filesLimits.length; i++) { + filesLimits[i] = filesLimits[i] * HDFSContext.HDFS_BLOCK_SIZE; + } + } + + public GenericDataSkewHeuristic(List counterNames, HeuristicConfigurationData heuristicConfData) { + this._counterNames = counterNames; + this._heuristicConfData = heuristicConfData; + loadParameters(); + } + + protected abstract TezTaskData[] getTasks(TezApplicationData data); + + public HeuristicConfigurationData getHeuristicConfData() { + return _heuristicConfData; + } + + public HeuristicResult apply(TezApplicationData data) { + + if (!data.getSucceeded()) { + return null; + } + + TezTaskData[] tasks = getTasks(data); + + //Gathering data for checking time skew + List timeTaken = new ArrayList(); + + for(int i = 0; i < tasks.length; i++) { + if (tasks[i].isSampled()) { + timeTaken.add(tasks[i].getTotalRunTimeMs()); + } + } + + long[][] groupsTime = Statistics.findTwoGroups(Longs.toArray(timeTaken)); + + long timeAvg1 = Statistics.average(groupsTime[0]); + long timeAvg2 = Statistics.average(groupsTime[1]); + + //seconds are used for calculating deviation as they provide a better idea than millisecond. + long timeAvgSec1 = TimeUnit.MILLISECONDS.toSeconds(timeAvg1); + long timeAvgSec2 = TimeUnit.MILLISECONDS.toSeconds(timeAvg2); + + long minTime = Math.min(timeAvgSec1, timeAvgSec2); + long diffTime = Math.abs(timeAvgSec1 - timeAvgSec2); + + //using the same deviation limits for time skew as for data skew. It can be changed in the fututre. + Severity severityTime = getDeviationSeverity(minTime, diffTime); + + //This reduces severity if number of tasks is insignificant + severityTime = Severity.min(severityTime, + Severity.getSeverityAscending(groupsTime[0].length, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], + numTasksLimits[3])); + + //Gather data + List inputSizes = new ArrayList(); + + for (int i = 0; i < tasks.length; i++) { + if (tasks[i].isSampled()) { + + long inputByte = 0; + for (TezCounterData.CounterName counterName : _counterNames) { + inputByte += tasks[i].getCounters().get(counterName); + } + + inputSizes.add(inputByte); + } + } + + long[][] groups = Statistics.findTwoGroups(Longs.toArray(inputSizes)); + + long avg1 = Statistics.average(groups[0]); + long avg2 = Statistics.average(groups[1]); + + long min = Math.min(avg1, avg2); + long diff = Math.abs(avg2 - avg1); + + Severity severityData = getDeviationSeverity(min, diff); + + //This reduces severity if the largest file sizes are insignificant + severityData = Severity.min(severityData, getFilesSeverity(avg2)); + + //This reduces severity if number of tasks is insignificant + severityData = Severity.min(severityData, + Severity.getSeverityAscending(groups[0].length, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], + numTasksLimits[3])); + + Severity severity = Severity.max(severityData, severityTime); + + HeuristicResult result = + new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), severity, + Utils.getHeuristicScore(severityData, tasks.length)); + + result.addResultDetail("Data skew (Number of tasks)", Integer.toString(tasks.length)); + result.addResultDetail("Data skew (Group A)", + groups[0].length + " tasks @ " + FileUtils.byteCountToDisplaySize(avg1) + " avg"); + result.addResultDetail("Data skew (Group B)", + groups[1].length + " tasks @ " + FileUtils.byteCountToDisplaySize(avg2) + " avg"); + + result.addResultDetail("Time skew (Number of tasks)", Integer.toString(tasks.length)); + result.addResultDetail("Time skew (Group A)", + groupsTime[0].length + " tasks @ " + convertTimeMs(timeAvg1) + " avg"); + result.addResultDetail("Time skew (Group B)", + groupsTime[1].length + " tasks @ " + convertTimeMs(timeAvg2) + " avg"); + + return result; + } + + private String convertTimeMs(long timeMs) { + if (timeMs < 1000) { + return Long.toString(timeMs) + " msec"; + } + return DurationFormatUtils.formatDuration(timeMs, "HH:mm:ss") + " HH:MM:SS"; + } + + private Severity getDeviationSeverity(long averageMin, long averageDiff) { + if (averageMin <= 0) { + averageMin = 1; + } + long value = averageDiff / averageMin; + return Severity.getSeverityAscending(value, deviationLimits[0], deviationLimits[1], deviationLimits[2], + deviationLimits[3]); + } + + private Severity getFilesSeverity(long value) { + return Severity.getSeverityAscending(value, filesLimits[0], filesLimits[1], filesLimits[2], filesLimits[3]); + } +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/GenericGCHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/GenericGCHeuristic.java new file mode 100644 index 000000000..d5726ef71 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/GenericGCHeuristic.java @@ -0,0 +1,142 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.util.Utils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.math.Statistics; +import java.util.Map; +import org.apache.log4j.Logger; + + +/** + * Analyses garbage collection efficiency + */ +public abstract class GenericGCHeuristic implements Heuristic { + private static final Logger logger = Logger.getLogger(GenericGCHeuristic.class); + + // Severity Parameters + private static final String GC_RATIO_SEVERITY = "gc_ratio_severity"; + private static final String RUNTIME_SEVERITY = "runtime_severity_in_min"; + + // Default value of parameters + private double[] gcRatioLimits = {0.01d, 0.02d, 0.03d, 0.04d}; // Garbage Collection Time / CPU Time + private double[] runtimeLimits = {5, 10, 12, 15}; // Task Runtime in milli sec + + private HeuristicConfigurationData _heuristicConfData; + + private void loadParameters() { + Map paramMap = _heuristicConfData.getParamMap(); + String heuristicName = _heuristicConfData.getHeuristicName(); + + double[] confGcRatioThreshold = Utils.getParam(paramMap.get(GC_RATIO_SEVERITY), gcRatioLimits.length); + if (confGcRatioThreshold != null) { + gcRatioLimits = confGcRatioThreshold; + } + logger.info(heuristicName + " will use " + GC_RATIO_SEVERITY + " with the following threshold settings: " + + Arrays.toString(gcRatioLimits)); + + double[] confRuntimeThreshold = Utils.getParam(paramMap.get(RUNTIME_SEVERITY), runtimeLimits.length); + if (confRuntimeThreshold != null) { + runtimeLimits = confRuntimeThreshold; + } + logger.info(heuristicName + " will use " + RUNTIME_SEVERITY + " with the following threshold settings: " + + Arrays.toString(runtimeLimits)); + for (int i = 0; i < runtimeLimits.length; i++) { + runtimeLimits[i] = runtimeLimits[i] * Statistics.MINUTE_IN_MS; + } + } + + public GenericGCHeuristic(HeuristicConfigurationData heuristicConfData) { + this._heuristicConfData = heuristicConfData; + loadParameters(); + } + + protected abstract TezTaskData[] getTasks(TezApplicationData data); + + public HeuristicConfigurationData getHeuristicConfData() { + return _heuristicConfData; + } + + public HeuristicResult apply(TezApplicationData data) { + + if(!data.getSucceeded()) { + return null; + } + + TezTaskData[] tasks = getTasks(data) ; + List gcMs = new ArrayList(); + List cpuMs = new ArrayList(); + List runtimesMs = new ArrayList(); + + for (TezTaskData task : tasks) { + if (task.isSampled()) { + runtimesMs.add(task.getTotalRunTimeMs()); + gcMs.add(task.getCounters().get(TezCounterData.CounterName.GC_TIME_MILLIS)); + cpuMs.add(task.getCounters().get(TezCounterData.CounterName.CPU_MILLISECONDS)); + } + } + + long avgRuntimeMs = Statistics.average(runtimesMs); + long avgCpuMs = Statistics.average(cpuMs); + long avgGcMs = Statistics.average(gcMs); + double ratio = avgCpuMs != 0 ? avgGcMs*(1.0)/avgCpuMs: 0; + + Severity severity; + if (tasks.length == 0) { + severity = Severity.NONE; + } else { + severity = getGcRatioSeverity(avgRuntimeMs, avgCpuMs, avgGcMs); + } + + HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), + _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); + + result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); + result.addResultDetail("Avg task runtime (ms)", Long.toString(avgRuntimeMs)); + result.addResultDetail("Avg task CPU time (ms)", Long.toString(avgCpuMs)); + result.addResultDetail("Avg task GC time (ms)", Long.toString(avgGcMs)); + result.addResultDetail("Task GC/CPU ratio", Double.toString(ratio)); + return result; + } + + private Severity getGcRatioSeverity(long runtimeMs, long cpuMs, long gcMs) { + double gcRatio = ((double)gcMs)/cpuMs; + Severity ratioSeverity = Severity.getSeverityAscending( + gcRatio, gcRatioLimits[0], gcRatioLimits[1], gcRatioLimits[2], gcRatioLimits[3]); + + // Severity is reduced if task runtime is insignificant + Severity runtimeSeverity = getRuntimeSeverity(runtimeMs); + + return Severity.min(ratioSeverity, runtimeSeverity); + } + + private Severity getRuntimeSeverity(long runtimeMs) { + return Severity.getSeverityAscending( + runtimeMs, runtimeLimits[0], runtimeLimits[1], runtimeLimits[2], runtimeLimits[3]); + } + +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java new file mode 100644 index 000000000..3a575092b --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/GenericMemoryHeuristic.java @@ -0,0 +1,185 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.google.common.base.Strings; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.math.Statistics; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.util.Utils; +import org.apache.log4j.Logger; + +import org.apache.commons.io.FileUtils; + +import java.util.*; + +/** + * Analyzes mapper memory allocation and requirements + */ +public abstract class GenericMemoryHeuristic implements Heuristic { + + private static final Logger logger = Logger.getLogger(GenericMemoryHeuristic.class); + + //Severity Parameters + private static final String MEM_RATIO_SEVERITY = "memory_ratio_severity"; + private static final String DEFAULT_MAPPER_CONTAINER_SIZE = "2048"; + private static final String CONTAINER_MEM_DEFAULT_MB = "container_memory_default_mb"; + private String _containerMemConf; + + //Default Value of parameters + + private double [] memoryRatioLimits = {0.6d, 0.5d, 0.4d, 0.3d}; //Ratio of successful tasks + + private HeuristicConfigurationData _heuristicConfData; + + private String getContainerMemDefaultMBytes() { + Map paramMap = _heuristicConfData.getParamMap(); + if (paramMap.containsKey(CONTAINER_MEM_DEFAULT_MB)) { + String strValue = paramMap.get(CONTAINER_MEM_DEFAULT_MB); + try { + return strValue; + } + catch (NumberFormatException e) { + logger.warn(CONTAINER_MEM_DEFAULT_MB + ": expected number [" + strValue + "]"); + } + } + return DEFAULT_MAPPER_CONTAINER_SIZE; + } + + private void loadParameters() { + Map paramMap = _heuristicConfData.getParamMap(); + String heuristicName = _heuristicConfData.getHeuristicName(); + + double[] confSuccessRatioLimits = Utils.getParam(paramMap.get(MEM_RATIO_SEVERITY), memoryRatioLimits.length); + if (confSuccessRatioLimits != null) { + memoryRatioLimits = confSuccessRatioLimits; + } + logger.info(heuristicName + " will use " + MEM_RATIO_SEVERITY + " with the following threshold settings: " + + Arrays.toString(memoryRatioLimits)); + + + } + + public GenericMemoryHeuristic(String containerMemConf, HeuristicConfigurationData heuristicConfData) { + this._containerMemConf = containerMemConf; + this._heuristicConfData = heuristicConfData; + loadParameters(); + } + + public HeuristicConfigurationData getHeuristicConfData() { + return _heuristicConfData; + } + + protected abstract TezTaskData[] getTasks(TezApplicationData data); + + public HeuristicResult apply(TezApplicationData data) { + if(!data.getSucceeded()) { + return null; + } + TezTaskData[] tasks = getTasks(data); + + + List totalPhysicalMemory = new LinkedList(); + List totalVirtualMemory = new LinkedList(); + List runTime = new LinkedList(); + + for (TezTaskData task : tasks) { + + if (task.isSampled()) { + totalPhysicalMemory.add(task.getCounters().get(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES)); + totalVirtualMemory.add(task.getCounters().get(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES)); + runTime.add(task.getTotalRunTimeMs()); + } + + + } + + long averagePMem = Statistics.average(totalPhysicalMemory); + long averageVMem = Statistics.average(totalVirtualMemory); + long maxPMem; + long minPMem; + try{ + maxPMem = Collections.max(totalPhysicalMemory); + minPMem = Collections.min(totalPhysicalMemory); + + } + catch(Exception exception){ + maxPMem = 0; + minPMem = 0; + } + long averageRunTime = Statistics.average(runTime); + + String containerSizeStr; + + if(!Strings.isNullOrEmpty(data.getConf().getProperty(_containerMemConf))){ + containerSizeStr = data.getConf().getProperty(_containerMemConf); + } + else { + containerSizeStr = getContainerMemDefaultMBytes(); + } + + long containerSize = Long.valueOf(containerSizeStr) * FileUtils.ONE_MB; + + double averageMemMb = (double)((averagePMem) /FileUtils.ONE_MB) ; + + double ratio = averageMemMb / ((double)(containerSize / FileUtils.ONE_MB)); + + Severity severity ; + + if(tasks.length == 0){ + severity = Severity.NONE; + } + else{ + severity = getMemoryRatioSeverity(ratio); + } + + HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), + _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); + + result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); + result.addResultDetail("Maximum Physical Memory (MB)", + tasks.length == 0 ? "0" : Long.toString(maxPMem/FileUtils.ONE_MB)); + result.addResultDetail("Minimum Physical memory (MB)", + tasks.length == 0 ? "0" : Long.toString(minPMem/FileUtils.ONE_MB)); + result.addResultDetail("Average Physical Memory (MB)", + tasks.length == 0 ? "0" : Long.toString(averagePMem/FileUtils.ONE_MB)); + result.addResultDetail("Average Virtual Memory (MB)", + tasks.length == 0 ? "0" : Long.toString(averageVMem/FileUtils.ONE_MB)); + result.addResultDetail("Average Task RunTime", + tasks.length == 0 ? "0" : Statistics.readableTimespan(averageRunTime)); + result.addResultDetail("Requested Container Memory (MB)", + (tasks.length == 0 || containerSize == 0 || containerSize == -1) ? "0" : String.valueOf(containerSize / FileUtils.ONE_MB)); + + + return result; + + } + + private Severity getMemoryRatioSeverity(double ratio) { + return Severity.getSeverityDescending( + ratio, memoryRatioLimits[0], memoryRatioLimits[1], memoryRatioLimits[2], memoryRatioLimits[3]); + } + + + + +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristic.java new file mode 100644 index 000000000..b17ebe3b2 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristic.java @@ -0,0 +1,47 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + + +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; + +import java.util.Arrays; +import org.apache.log4j.Logger; + + +/** + * This Heuristic analyses the skewness in the task input data + */ +public class MapperDataSkewHeuristic extends GenericDataSkewHeuristic { + private static final Logger logger = Logger.getLogger(MapperDataSkewHeuristic.class); + + public MapperDataSkewHeuristic(HeuristicConfigurationData heuristicConfData) { + super(Arrays.asList( + TezCounterData.CounterName.HDFS_BYTES_READ, + TezCounterData.CounterName.S3A_BYTES_READ, + TezCounterData.CounterName.S3N_BYTES_READ + ), heuristicConfData); + } + + @Override + protected TezTaskData[] getTasks(TezApplicationData data) { + return data.getMapTaskData(); + } +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristic.java new file mode 100644 index 000000000..3a82ae5f9 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristic.java @@ -0,0 +1,51 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + + +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.util.Utils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.math.Statistics; +import java.util.Map; +import org.apache.log4j.Logger; + + +/** + * Analyses garbage collection efficiency + */ +public class MapperGCHeuristic extends GenericGCHeuristic{ + private static final Logger logger = Logger.getLogger(MapperGCHeuristic.class); + + public MapperGCHeuristic(HeuristicConfigurationData heuristicConfData) { + super(heuristicConfData); + } + + @Override + protected TezTaskData[] getTasks(TezApplicationData data) { + return data.getMapTaskData(); + } + +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java new file mode 100644 index 000000000..96d5bb12e --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristic.java @@ -0,0 +1,45 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; + +import com.linkedin.drelephant.tez.data.TezApplicationData; + +import com.linkedin.drelephant.tez.data.TezTaskData; + +import org.apache.log4j.Logger; + +/** + * Analyzes mapper memory allocation and requirements + */ +public class MapperMemoryHeuristic extends GenericMemoryHeuristic { + + private static final Logger logger = Logger.getLogger(MapperMemoryHeuristic.class); + + public static final String MAPPER_MEMORY_CONF = "mapreduce.map.memory.mb"; + + public MapperMemoryHeuristic(HeuristicConfigurationData __heuristicConfData){ + super(MAPPER_MEMORY_CONF, __heuristicConfData); + } + + @Override + protected TezTaskData[] getTasks(TezApplicationData data) { + return data.getMapTaskData(); + } + +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristic.java new file mode 100644 index 000000000..1694e32ea --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristic.java @@ -0,0 +1,165 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + + +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.util.Utils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.linkedin.drelephant.analysis.HDFSContext; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.math.Statistics; + +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +/** + * Analyzes mapper task speed and efficiency + */ +public class MapperSpeedHeuristic implements Heuristic { + private static final Logger logger = Logger.getLogger(MapperSpeedHeuristic.class); + + // Severity parameters. + private static final String DISK_SPEED_SEVERITY = "disk_speed_severity"; + private static final String RUNTIME_SEVERITY = "runtime_severity_in_min"; + + // Default value of parameters + private double[] diskSpeedLimits = {1d/2, 1d/4, 1d/8, 1d/32}; // Fraction of HDFS block size + private double[] runtimeLimits = {5, 10, 15, 30}; // The Map task runtime in milli sec + + private List _counterNames = Arrays.asList( + TezCounterData.CounterName.HDFS_BYTES_READ, + TezCounterData.CounterName.S3A_BYTES_READ, + TezCounterData.CounterName.S3N_BYTES_READ + ); + + private HeuristicConfigurationData _heuristicConfData; + + private void loadParameters() { + Map paramMap = _heuristicConfData.getParamMap(); + String heuristicName = _heuristicConfData.getHeuristicName(); + + double[] confDiskSpeedThreshold = Utils.getParam(paramMap.get(DISK_SPEED_SEVERITY), diskSpeedLimits.length); + if (confDiskSpeedThreshold != null) { + diskSpeedLimits = confDiskSpeedThreshold; + } + logger.info(heuristicName + " will use " + DISK_SPEED_SEVERITY + " with the following threshold settings: " + + Arrays.toString(diskSpeedLimits)); + for (int i = 0; i < diskSpeedLimits.length; i++) { + diskSpeedLimits[i] = diskSpeedLimits[i] * HDFSContext.DISK_READ_SPEED; + } + + double[] confRuntimeThreshold = Utils.getParam(paramMap.get(RUNTIME_SEVERITY), runtimeLimits.length); + if (confRuntimeThreshold != null) { + runtimeLimits = confRuntimeThreshold; + } + logger.info(heuristicName + " will use " + RUNTIME_SEVERITY + " with the following threshold settings: " + Arrays + .toString(runtimeLimits)); + for (int i = 0; i < runtimeLimits.length; i++) { + runtimeLimits[i] = runtimeLimits[i] * Statistics.MINUTE_IN_MS; + } + } + + public MapperSpeedHeuristic(HeuristicConfigurationData heuristicConfData) { + this._heuristicConfData = heuristicConfData; + loadParameters(); + } + + public HeuristicConfigurationData getHeuristicConfData() { + return _heuristicConfData; + } + + public HeuristicResult apply(TezApplicationData data) { + + if(!data.getSucceeded()) { + return null; + } + + TezTaskData[] tasks = data.getMapTaskData(); + + List inputSizes = new ArrayList(); + List speeds = new ArrayList(); + List runtimesMs = new ArrayList(); + + for (TezTaskData task : tasks) { + + if (task.isSampled()) { + + long inputBytes = 0; + + for (TezCounterData.CounterName counterName: _counterNames) { + inputBytes += task.getCounters().get(counterName); + } + + long runtimeMs = task.getTotalRunTimeMs(); + inputSizes.add(inputBytes); + runtimesMs.add(runtimeMs); + //Speed is records per second + speeds.add((1000 * inputBytes) / (runtimeMs)); + } + } + + long medianSpeed; + long medianSize; + long medianRuntimeMs; + + if (tasks.length != 0) { + medianSpeed = Statistics.median(speeds); + medianSize = Statistics.median(inputSizes); + medianRuntimeMs = Statistics.median(runtimesMs); + } else { + medianSpeed = 0; + medianSize = 0; + medianRuntimeMs = 0; + } + + Severity severity = getDiskSpeedSeverity(medianSpeed); + + //This reduces severity if task runtime is insignificant + severity = Severity.min(severity, getRuntimeSeverity(medianRuntimeMs)); + + HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), + _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); + + result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); + result.addResultDetail("Median task input ", FileUtils.byteCountToDisplaySize(medianSize)); + result.addResultDetail("Median task runtime", Statistics.readableTimespan(medianRuntimeMs)); + result.addResultDetail("Median task speed", FileUtils.byteCountToDisplaySize(medianSpeed) + "/s"); + + return result; + } + + private Severity getDiskSpeedSeverity(long speed) { + return Severity.getSeverityDescending( + speed, diskSpeedLimits[0], diskSpeedLimits[1], diskSpeedLimits[2], diskSpeedLimits[3]); + } + + private Severity getRuntimeSeverity(long runtimeMs) { + return Severity.getSeverityAscending( + runtimeMs, runtimeLimits[0], runtimeLimits[1], runtimeLimits[2], runtimeLimits[3]); + } +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristic.java new file mode 100644 index 000000000..f7ea997e5 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristic.java @@ -0,0 +1,142 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.util.Utils; +import org.apache.log4j.Logger; + + +import java.util.Arrays; +import java.util.Map; + +/** + * Analyzes mapper task data spill rates + */ +public class MapperSpillHeuristic implements Heuristic { + private static final Logger logger = Logger.getLogger(MapperSpillHeuristic.class); + private static final long THRESHOLD_SPILL_FACTOR = 10000; + + // Severity parameters. + private static final String SPILL_SEVERITY = "spill_severity"; + private static final String NUM_TASKS_SEVERITY = "num_tasks_severity"; + + // Default value of parameters + private double[] numTasksLimits = {50, 100, 500, 1000}; // Number of Map tasks. + private double[] spillLimits = {2.01d, 2.2d, 2.5d, 3.0d}; // Records spilled/total output records + + private HeuristicConfigurationData _heuristicConfData; + + private void loadParameters() { + Map paramMap = _heuristicConfData.getParamMap(); + String heuristicName = _heuristicConfData.getHeuristicName(); + + double[] confNumTasksThreshold = Utils.getParam(paramMap.get(NUM_TASKS_SEVERITY), numTasksLimits.length); + if (confNumTasksThreshold != null) { + numTasksLimits = confNumTasksThreshold; + } + logger.info(heuristicName + " will use " + NUM_TASKS_SEVERITY + " with the following threshold settings: " + + Arrays.toString(numTasksLimits)); + + double[] confSpillThreshold = Utils.getParam(paramMap.get(SPILL_SEVERITY), spillLimits.length); + if (confSpillThreshold != null) { + spillLimits = confSpillThreshold; + } + logger.info(heuristicName + " will use " + SPILL_SEVERITY + " with the following threshold settings: " + Arrays + .toString(spillLimits)); + for (int i = 0; i < spillLimits.length; i++) { + spillLimits[i] = spillLimits[i] * THRESHOLD_SPILL_FACTOR; + } + } + + public MapperSpillHeuristic(HeuristicConfigurationData heuristicConfData) { + this._heuristicConfData = heuristicConfData; + loadParameters(); + } + + @Override + public HeuristicConfigurationData getHeuristicConfData() { + return _heuristicConfData; + } + + @Override + public HeuristicResult apply(TezApplicationData data) { + if(!data.getSucceeded()) { + return null; + } + TezTaskData[] tasks = data.getMapTaskData(); + + long totalSpills = 0; + long totalOutputRecords = 0; + double ratioSpills = 0.0; + + for (TezTaskData task : tasks) { + + if (task.isSampled()) { + totalSpills += task.getCounters().get(TezCounterData.CounterName.SPILLED_RECORDS); + totalOutputRecords += task.getCounters().get(TezCounterData.CounterName.OUTPUT_RECORDS); + } + } + + //If both totalSpills and totalOutputRecords are zero then set ratioSpills to zero. + if (totalSpills == 0) { + ratioSpills = 0; + } else { + ratioSpills = (double) totalSpills / (double) totalOutputRecords; + } + + Severity severity = getSpillSeverity(ratioSpills); + + // Severity is reduced if number of tasks is small + Severity taskSeverity = getNumTasksSeverity(tasks.length); + severity = Severity.min(severity, taskSeverity); + + HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), + _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); + + result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); + result.addResultDetail("Avg spilled records per task", + tasks.length == 0 ? "0" : Long.toString(totalSpills / tasks.length)); + result.addResultDetail("Avg output records per task", + tasks.length == 0 ? "0" : Long.toString(totalOutputRecords / tasks.length)); + result.addResultDetail("Ratio of spilled records to output records", Double.toString(ratioSpills)); + + return result; + + } + + private Severity getSpillSeverity(double ratioSpills) { + + long normalizedSpillRatio = 0; + //Normalize the ratio to integer. + normalizedSpillRatio = (long) (ratioSpills * THRESHOLD_SPILL_FACTOR); + + return Severity.getSeverityAscending( + normalizedSpillRatio, spillLimits[0], spillLimits[1], spillLimits[2], spillLimits[3]); + } + + private Severity getNumTasksSeverity(long numTasks) { + return Severity.getSeverityAscending( + numTasks, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], numTasksLimits[3]); + } + + +} diff --git a/app/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristic.java new file mode 100644 index 000000000..838bd7a59 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristic.java @@ -0,0 +1,182 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + + +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.util.Utils; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +import com.linkedin.drelephant.math.Statistics; + +import java.util.*; + +/** + * Analyzes mapper task runtimes + */ +public class MapperTimeHeuristic implements Heuristic { + + private static final Logger logger = Logger.getLogger(MapperTimeHeuristic.class); + + // Severity parameters. + private static final String SHORT_RUNTIME_SEVERITY = "short_runtime_severity_in_min"; + private static final String LONG_RUNTIME_SEVERITY = "long_runtime_severity_in_min"; + private static final String NUM_TASKS_SEVERITY = "num_tasks_severity"; + + // Default value of parameters + private double[] shortRuntimeLimits = {10, 4, 2, 1}; // Limits(ms) for tasks with shorter runtime + private double[] longRuntimeLimits = {15, 30, 60, 120}; // Limits(ms) for tasks with longer runtime + private double[] numTasksLimits = {50, 101, 500, 1000}; // Number of Map tasks. + + private List _counterNames = Arrays.asList( + TezCounterData.CounterName.HDFS_BYTES_READ, + TezCounterData.CounterName.S3A_BYTES_READ, + TezCounterData.CounterName.S3N_BYTES_READ + ); + + private HeuristicConfigurationData _heuristicConfData; + + private void loadParameters() { + Map paramMap = _heuristicConfData.getParamMap(); + String heuristicName = _heuristicConfData.getHeuristicName(); + + double[] confShortThreshold = Utils.getParam(paramMap.get(SHORT_RUNTIME_SEVERITY), shortRuntimeLimits.length); + if (confShortThreshold != null) { + shortRuntimeLimits = confShortThreshold; + } + logger.info(heuristicName + " will use " + SHORT_RUNTIME_SEVERITY + " with the following threshold settings: " + + Arrays.toString(shortRuntimeLimits)); + for (int i = 0; i < shortRuntimeLimits.length; i++) { + shortRuntimeLimits[i] = shortRuntimeLimits[i] * Statistics.MINUTE_IN_MS; + } + + double[] confLongThreshold = Utils.getParam(paramMap.get(LONG_RUNTIME_SEVERITY), longRuntimeLimits.length); + if (confLongThreshold != null) { + longRuntimeLimits = confLongThreshold; + } + logger.info(heuristicName + " will use " + LONG_RUNTIME_SEVERITY + " with the following threshold settings: " + + Arrays.toString(longRuntimeLimits)); + for (int i = 0; i < longRuntimeLimits.length; i++) { + longRuntimeLimits[i] = longRuntimeLimits[i] * Statistics.MINUTE_IN_MS; + } + + double[] confNumTasksThreshold = Utils.getParam(paramMap.get(NUM_TASKS_SEVERITY), numTasksLimits.length); + if (confNumTasksThreshold != null) { + numTasksLimits = confNumTasksThreshold; + } + logger.info(heuristicName + " will use " + NUM_TASKS_SEVERITY + " with the following threshold settings: " + Arrays + .toString(numTasksLimits)); + + + } + + public MapperTimeHeuristic(HeuristicConfigurationData heuristicConfData) { + this._heuristicConfData = heuristicConfData; + loadParameters(); + } + + public HeuristicConfigurationData getHeuristicConfData() { + return _heuristicConfData; + } + + public HeuristicResult apply(TezApplicationData data) { + if(!data.getSucceeded()) { + return null; + } + TezTaskData[] tasks = data.getMapTaskData(); + + List inputSizes = new ArrayList(); + List runtimesMs = new ArrayList(); + long taskMinMs = Long.MAX_VALUE; + long taskMaxMs = 0; + + for (TezTaskData task : tasks) { + + if (task.isSampled()) { + long inputByte = 0; + for (TezCounterData.CounterName counterName: _counterNames) { + inputByte += task.getCounters().get(counterName); + } + inputSizes.add(inputByte); + long taskTime = task.getTotalRunTimeMs(); + runtimesMs.add(taskTime); + taskMinMs = Math.min(taskMinMs, taskTime); + taskMaxMs = Math.max(taskMaxMs, taskTime); + } + } + + if(taskMinMs == Long.MAX_VALUE) { + taskMinMs = 0; + } + + long averageSize = Statistics.average(inputSizes); + long averageTimeMs = Statistics.average(runtimesMs); + + Severity shortTaskSeverity = shortTaskSeverity(tasks.length, averageTimeMs); + Severity longTaskSeverity = longTaskSeverity(tasks.length, averageTimeMs); + Severity severity = Severity.max(shortTaskSeverity, longTaskSeverity); + + HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), + _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); + + result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); + result.addResultDetail("Average task input size", FileUtils.byteCountToDisplaySize(averageSize)); + result.addResultDetail("Average task runtime", Statistics.readableTimespan(averageTimeMs)); + result.addResultDetail("Max task runtime", Statistics.readableTimespan(taskMaxMs)); + result.addResultDetail("Min task runtime", Statistics.readableTimespan(taskMinMs)); + + return result; + } + + private Severity shortTaskSeverity(long numTasks, long averageTimeMs) { + // We want to identify jobs with short task runtime + Severity severity = getShortRuntimeSeverity(averageTimeMs); + // Severity is reduced if number of tasks is small. + Severity numTaskSeverity = getNumTasksSeverity(numTasks); + return Severity.min(severity, numTaskSeverity); + } + + private Severity longTaskSeverity(long numTasks, long averageTimeMs) { + // We want to identify jobs with long task runtime. Severity is NOT reduced if num of tasks is large + return getLongRuntimeSeverity(averageTimeMs); + } + + private Severity getShortRuntimeSeverity(long runtimeMs) { + return Severity.getSeverityDescending( + runtimeMs, shortRuntimeLimits[0], shortRuntimeLimits[1], shortRuntimeLimits[2], shortRuntimeLimits[3]); + } + + private Severity getLongRuntimeSeverity(long runtimeMs) { + return Severity.getSeverityAscending( + runtimeMs, longRuntimeLimits[0], longRuntimeLimits[1], longRuntimeLimits[2], longRuntimeLimits[3]); + } + + private Severity getNumTasksSeverity(long numTasks) { + return Severity.getSeverityAscending( + numTasks, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], numTasksLimits[3]); + } + + + +} diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristic.java new file mode 100644 index 000000000..3a01448d4 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristic.java @@ -0,0 +1,42 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import org.apache.log4j.Logger; + +import java.util.Arrays; + + +/** + * This Heuristic analyses the skewness in the task input data + */ +public class ReducerDataSkewHeuristic extends GenericDataSkewHeuristic { + private static final Logger logger = Logger.getLogger(ReducerDataSkewHeuristic.class); + + public ReducerDataSkewHeuristic(HeuristicConfigurationData heuristicConfData) { + super(Arrays.asList(TezCounterData.CounterName.SHUFFLE_BYTES), heuristicConfData); + } + + @Override + protected TezTaskData[] getTasks(TezApplicationData data) { + return data.getReduceTaskData(); + } +} \ No newline at end of file diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristic.java new file mode 100644 index 000000000..3ea3bdac4 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristic.java @@ -0,0 +1,41 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import org.apache.log4j.Logger; + + +/** + * Analyses garbage collection efficiency + */ +public class ReducerGCHeuristic extends GenericGCHeuristic { + + private static final Logger logger = Logger.getLogger(ReducerGCHeuristic.class); + + public ReducerGCHeuristic(HeuristicConfigurationData heuristicConfData) { + super(heuristicConfData); + } + + @Override + protected TezTaskData[] getTasks(TezApplicationData data) { + return data.getReduceTaskData(); + } + +} diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java new file mode 100644 index 000000000..41fc6fa1d --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristic.java @@ -0,0 +1,47 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + + +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import org.apache.log4j.Logger; + +/** + * Analyzes reducer memory allocation and requirements + */ + +public class ReducerMemoryHeuristic extends GenericMemoryHeuristic { + + private static final Logger logger = Logger.getLogger(ReducerMemoryHeuristic.class); + + public static final String REDUCER_MEMORY_CONF = "mapreduce.reduce.memory.mb"; + + public ReducerMemoryHeuristic(HeuristicConfigurationData __heuristicConfData){ + super(REDUCER_MEMORY_CONF, __heuristicConfData); + } + + @Override + protected TezTaskData[] getTasks(TezApplicationData data) { + return data.getReduceTaskData(); + } + + + + +} diff --git a/app/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristic.java b/app/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristic.java new file mode 100644 index 000000000..5e0dbfe65 --- /dev/null +++ b/app/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristic.java @@ -0,0 +1,167 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + + +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.util.Utils; +import org.apache.log4j.Logger; + +import com.linkedin.drelephant.math.Statistics; + +import java.util.*; + +/** + * Analyzes reducer task runtimes + */ + +public class ReducerTimeHeuristic implements Heuristic { + + private static final Logger logger = Logger.getLogger(ReducerTimeHeuristic.class); + + // Severity parameters. + private static final String SHORT_RUNTIME_SEVERITY = "short_runtime_severity_in_min"; + private static final String LONG_RUNTIME_SEVERITY = "long_runtime_severity_in_min"; + private static final String NUM_TASKS_SEVERITY = "num_tasks_severity"; + + // Default value of parameters + private double[] shortRuntimeLimits = {10, 4, 2, 1}; // Limits(ms) for tasks with shorter runtime + private double[] longRuntimeLimits = {15, 30, 60, 120}; // Limits(ms) for tasks with longer runtime + private double[] numTasksLimits = {50, 101, 500, 1000}; // Number of Map tasks. + + private HeuristicConfigurationData _heuristicConfData; + + private void loadParameters() { + Map paramMap = _heuristicConfData.getParamMap(); + String heuristicName = _heuristicConfData.getHeuristicName(); + + double[] confShortThreshold = Utils.getParam(paramMap.get(SHORT_RUNTIME_SEVERITY), shortRuntimeLimits.length); + if (confShortThreshold != null) { + shortRuntimeLimits = confShortThreshold; + } + logger.info(heuristicName + " will use " + SHORT_RUNTIME_SEVERITY + " with the following threshold settings: " + + Arrays.toString(shortRuntimeLimits)); + for (int i = 0; i < shortRuntimeLimits.length; i++) { + shortRuntimeLimits[i] = shortRuntimeLimits[i] * Statistics.MINUTE_IN_MS; + } + + double[] confLongThreshold = Utils.getParam(paramMap.get(LONG_RUNTIME_SEVERITY), longRuntimeLimits.length); + if (confLongThreshold != null) { + longRuntimeLimits = confLongThreshold; + } + logger.info(heuristicName + " will use " + LONG_RUNTIME_SEVERITY + " with the following threshold settings: " + + Arrays.toString(longRuntimeLimits)); + for (int i = 0; i < longRuntimeLimits.length; i++) { + longRuntimeLimits[i] = longRuntimeLimits[i] * Statistics.MINUTE_IN_MS; + } + + double[] confNumTasksThreshold = Utils.getParam(paramMap.get(NUM_TASKS_SEVERITY), numTasksLimits.length); + if (confNumTasksThreshold != null) { + numTasksLimits = confNumTasksThreshold; + } + logger.info(heuristicName + " will use " + NUM_TASKS_SEVERITY + " with the following threshold settings: " + Arrays + .toString(numTasksLimits)); + + + } + + public ReducerTimeHeuristic(HeuristicConfigurationData heuristicConfData) { + this._heuristicConfData = heuristicConfData; + loadParameters(); + } + + public HeuristicConfigurationData getHeuristicConfData() { + return _heuristicConfData; + } + + public HeuristicResult apply(TezApplicationData data) { + if(!data.getSucceeded()) { + return null; + } + TezTaskData[] tasks = data.getReduceTaskData(); + + List runtimesMs = new ArrayList(); + long taskMinMs = Long.MAX_VALUE; + long taskMaxMs = 0; + + for (TezTaskData task : tasks) { + + if (task.isSampled()) { + long taskTime = task.getTotalRunTimeMs(); + runtimesMs.add(taskTime); + taskMinMs = Math.min(taskMinMs, taskTime); + taskMaxMs = Math.max(taskMaxMs, taskTime); + } + } + + if(taskMinMs == Long.MAX_VALUE) { + taskMinMs = 0; + } + + + long averageTimeMs = Statistics.average(runtimesMs); + + Severity shortTaskSeverity = shortTaskSeverity(tasks.length, averageTimeMs); + Severity longTaskSeverity = longTaskSeverity(tasks.length, averageTimeMs); + Severity severity = Severity.max(shortTaskSeverity, longTaskSeverity); + + HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), + _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); + + result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); + result.addResultDetail("Average task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(averageTimeMs).equals("") ? "0 sec" : Statistics.readableTimespan(averageTimeMs))); + result.addResultDetail("Max task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(taskMaxMs).equals("") ? "0 sec" : Statistics.readableTimespan(taskMaxMs)) ); + result.addResultDetail("Min task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(taskMinMs).equals("") ? "0 sec" :Statistics.readableTimespan(taskMinMs))) ; + + return result; + } + + private Severity shortTaskSeverity(long numTasks, long averageTimeMs) { + // We want to identify jobs with short task runtime + Severity severity = getShortRuntimeSeverity(averageTimeMs); + // Severity is reduced if number of tasks is small. + Severity numTaskSeverity = getNumTasksSeverity(numTasks); + return Severity.min(severity, numTaskSeverity); + } + + private Severity longTaskSeverity(long numTasks, long averageTimeMs) { + // We want to identify jobs with long task runtime. Severity is NOT reduced if num of tasks is large + return getLongRuntimeSeverity(averageTimeMs); + } + + private Severity getShortRuntimeSeverity(long runtimeMs) { + return Severity.getSeverityDescending( + runtimeMs, shortRuntimeLimits[0], shortRuntimeLimits[1], shortRuntimeLimits[2], shortRuntimeLimits[3]); + } + + private Severity getLongRuntimeSeverity(long runtimeMs) { + return Severity.getSeverityAscending( + runtimeMs, longRuntimeLimits[0], longRuntimeLimits[1], longRuntimeLimits[2], longRuntimeLimits[3]); + } + + private Severity getNumTasksSeverity(long numTasks) { + return Severity.getSeverityAscending( + numTasks, numTasksLimits[0], numTasksLimits[1], numTasksLimits[2], numTasksLimits[3]); + } + +} + diff --git a/app/com/linkedin/drelephant/util/InfoExtractor.java b/app/com/linkedin/drelephant/util/InfoExtractor.java index 9824b79b2..6ff0f4d1b 100644 --- a/app/com/linkedin/drelephant/util/InfoExtractor.java +++ b/app/com/linkedin/drelephant/util/InfoExtractor.java @@ -20,6 +20,10 @@ import com.linkedin.drelephant.clients.WorkflowClient; import com.linkedin.drelephant.configurations.scheduler.SchedulerConfiguration; import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData; + +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.clients.WorkflowClient; + import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; import com.linkedin.drelephant.schedulers.Scheduler; import com.linkedin.drelephant.spark.data.SparkApplicationData; @@ -116,6 +120,9 @@ public static void loadInfo(AppResult result, HadoopApplicationData data) { } else if ( data instanceof SparkApplicationData) { properties = retrieveSparkProperties((SparkApplicationData) data); } + else if(data instanceof TezApplicationData){ + properties = retrieveTezProperties((TezApplicationData) data); + } Scheduler scheduler = getSchedulerInstance(data.getAppId(), properties); if (scheduler == null) { @@ -166,6 +173,10 @@ public static Properties retrieveMapreduceProperties(MapReduceApplicationData ap return appData.getConf(); } + public static Properties retrieveTezProperties(TezApplicationData appData) { + return appData.getConf(); + } + /** * Populates the given app result with the info from the given application data and scheduler. * diff --git a/app/com/linkedin/drelephant/util/ThreadContextMR2.java b/app/com/linkedin/drelephant/util/ThreadContextMR2.java new file mode 100644 index 000000000..a448a24fe --- /dev/null +++ b/app/com/linkedin/drelephant/util/ThreadContextMR2.java @@ -0,0 +1,92 @@ +package com.linkedin.drelephant.util; + +import com.linkedin.drelephant.math.Statistics; +import org.apache.hadoop.security.authentication.client.AuthenticatedURL; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.log4j.Logger; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class ThreadContextMR2 { + private static final Logger logger = Logger.getLogger(com.linkedin.drelephant.util.ThreadContextMR2.class); + + private static final AtomicInteger THREAD_ID = new AtomicInteger(1); + + private static final ThreadLocal _LOCAL_THREAD_ID = new ThreadLocal() { + @Override + public Integer initialValue() { + return THREAD_ID.getAndIncrement(); + } + }; + + private static final ThreadLocal _LOCAL_LAST_UPDATED = new ThreadLocal(); + private static final ThreadLocal _LOCAL_UPDATE_INTERVAL = new ThreadLocal(); + + private static final ThreadLocal _LOCAL_DIAGNOSTIC_PATTERN = new ThreadLocal() { + @Override + public Pattern initialValue() { + // Example: "Task task_1443068695259_9143_m_000475 failed 1 times" + return Pattern.compile( + ".*[\\s\\u00A0]+(task_[0-9]+_[0-9]+_[m|r]_[0-9]+)[\\s\\u00A0]+.*"); + } + }; + + private static final ThreadLocal _LOCAL_AUTH_TOKEN = + new ThreadLocal() { + @Override + public AuthenticatedURL.Token initialValue() { + _LOCAL_LAST_UPDATED.set(System.currentTimeMillis()); + // Random an interval for each executor to avoid update token at the same time + _LOCAL_UPDATE_INTERVAL.set(Statistics.MINUTE_IN_MS * 30 + new Random().nextLong() + % (3 * Statistics.MINUTE_IN_MS)); + logger.info("Executor " + _LOCAL_THREAD_ID.get() + " update interval " + _LOCAL_UPDATE_INTERVAL.get() * 1.0 + / Statistics.MINUTE_IN_MS); + return new AuthenticatedURL.Token(); + } + }; + + private static final ThreadLocal _LOCAL_AUTH_URL = new ThreadLocal() { + @Override + public AuthenticatedURL initialValue() { + return new AuthenticatedURL(); + } + }; + + private static final ThreadLocal _LOCAL_MAPPER = new ThreadLocal() { + @Override + public ObjectMapper initialValue() { + return new ObjectMapper(); + } + }; + + private ThreadContextMR2() { + // Empty on purpose + } + + public static Matcher getDiagnosticMatcher(String diagnosticInfo) { + return _LOCAL_DIAGNOSTIC_PATTERN.get().matcher(diagnosticInfo); + } + + public static JsonNode readJsonNode(URL url) throws IOException, AuthenticationException { + HttpURLConnection conn = _LOCAL_AUTH_URL.get().openConnection(url, _LOCAL_AUTH_TOKEN.get()); + return _LOCAL_MAPPER.get().readTree(conn.getInputStream()); + } + + public static void updateAuthToken() { + long curTime = System.currentTimeMillis(); + if (curTime - _LOCAL_LAST_UPDATED.get() > _LOCAL_UPDATE_INTERVAL.get()) { + logger.info("Executor " + _LOCAL_THREAD_ID.get() + " updates its AuthenticatedToken."); + _LOCAL_AUTH_TOKEN.set(new AuthenticatedURL.Token()); + _LOCAL_AUTH_URL.set(new AuthenticatedURL()); + _LOCAL_LAST_UPDATED.set(curTime); + } + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java b/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java index 531bb3724..8ce84bec1 100644 --- a/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java +++ b/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java @@ -18,6 +18,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; + +import com.linkedin.drelephant.util.ThreadContextMR2; import org.junit.Assert; import org.junit.Test; diff --git a/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java b/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java new file mode 100644 index 000000000..4a42a8617 --- /dev/null +++ b/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez; + + +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import org.junit.Assert; +import org.junit.Test; + +public class TezTaskLevelAggregatedMetricsTest { + + @Test + public void testZeroTasks() { + TezTaskData taskData[] = {}; + TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(taskData, 0, 0); + Assert.assertEquals(taskMetrics.getDelay(), 0); + Assert.assertEquals(taskMetrics.getResourceUsed(), 0); + Assert.assertEquals(taskMetrics.getResourceWasted(), 0); + } + + @Test + public void testNullTaskArray() { + TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(null, 0, 0); + Assert.assertEquals(taskMetrics.getDelay(), 0); + Assert.assertEquals(taskMetrics.getResourceUsed(), 0); + Assert.assertEquals(taskMetrics.getResourceWasted(), 0); + } + + @Test + public void testTaskLevelData() { + TezTaskData taskData[] = new TezTaskData[3]; + TezCounterData counterData = new TezCounterData(); + counterData.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, 655577088L); + counterData.set(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES, 3051589632L); + long time[] = {0,0,0,1464218501117L, 1464218534148L}; + taskData[0] = new TezTaskData("task", "id"); + taskData[0].setTimeAndCounter(time,counterData); + taskData[1] = new TezTaskData("task", "id"); + taskData[1].setTimeAndCounter(new long[5],counterData); + // Non-sampled task, which does not contain time and counter data + taskData[2] = new TezTaskData("task", "id"); + TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(taskData, 4096L, 1463218501117L); + Assert.assertEquals(taskMetrics.getDelay(), 1000000000L); + Assert.assertEquals(taskMetrics.getResourceUsed(), 135168L); + Assert.assertEquals(taskMetrics.getResourceWasted(), 66627L); + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java b/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java new file mode 100644 index 000000000..bf38f10c3 --- /dev/null +++ b/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java @@ -0,0 +1,35 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.fetchers; + +import java.util.regex.Matcher; +import org.junit.Assert; +import org.junit.Test; +import com.linkedin.drelephant.util.ThreadContextMR2; + +public class TezFetcherTest { + + @Test + public void testDiagnosticMatcher() { + Matcher matcher = ThreadContextMR2.getDiagnosticMatcher("Task task_1443068695259_9143_m_000475 failed 1 time"); + Assert.assertEquals(".*[\\s\\u00A0]+(task_[0-9]+_[0-9]+_[m|r]_[0-9]+)[\\s\\u00A0]+.*", matcher.pattern().toString()); + Assert.assertEquals(true, matcher.matches()); + Assert.assertEquals(1, matcher.groupCount()); + Assert.assertEquals("task_1443068695259_9143_m_000475", matcher.group(1)); + } + +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java new file mode 100644 index 000000000..8792e70f4 --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java @@ -0,0 +1,146 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.*; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; + + +public class MapperDataSkewHeuristicTest extends TestCase { + + private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64; //1MB + private static final long UNITSIZETIME = 1000000; //1000sec + + private static Map paramsMap = new HashMap(); + private static Heuristic _heuristic = new MapperDataSkewHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + public void testCritical() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(200, 200, 1 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testSevere() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(200, 200, 10 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testModerate() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(200, 200, 20 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testLow() throws IOException { + assertEquals(Severity.LOW, analyzeJob(200, 200, 30 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(200, 200, 50 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testSmallFiles() throws IOException { + assertEquals(Severity.NONE, analyzeJob(200, 200, 1 * UNITSIZE, 5 * UNITSIZE)); + } + + public void testSmallTasks() throws IOException { + assertEquals(Severity.NONE, analyzeJob(5, 5, 10 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testCriticalTime() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJobTime(200, 200, 1 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testSevereTime() throws IOException { + assertEquals(Severity.SEVERE, analyzeJobTime(200, 200, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testModerateTime() throws IOException { + assertEquals(Severity.MODERATE, analyzeJobTime(200, 200, 20 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testLowTime() throws IOException { + assertEquals(Severity.LOW, analyzeJobTime(200, 200, 30 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testNoneTime() throws IOException { + assertEquals(Severity.NONE, analyzeJobTime(200, 200, 50 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testSmallTasksTime() throws IOException { + assertEquals(Severity.NONE, analyzeJobTime(5, 5, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) + throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] mappers = new TezTaskData[numSmallTasks + numLargeTasks + 1]; + + TezCounterData smallCounter = new TezCounterData(); + smallCounter.set(TezCounterData.CounterName.HDFS_BYTES_READ, smallInputSize); + + TezCounterData largeCounter = new TezCounterData(); + largeCounter.set(TezCounterData.CounterName.S3A_BYTES_READ, largeInputSize); + + int i = 0; + for (; i < numSmallTasks; i++) { + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + mappers[i].setTimeAndCounter(new long[5], smallCounter); + } + for (; i < numSmallTasks + numLargeTasks; i++) { + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + mappers[i].setTimeAndCounter(new long[5], largeCounter); + } + // Non-sampled task, which does not contain time and counter data + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + + } + + private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) + throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] mappers = new TezTaskData[numSmallTasks + numLongTasks + 1]; + + int i = 0; + for (; i < numSmallTasks; i++) { + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + mappers[i].setTotalTimeMs(smallTimeTaken, true); + mappers[i].setCounter(jobCounter); + } + for (; i < numSmallTasks + numLongTasks; i++) { + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + mappers[i].setTotalTimeMs(longTimeTaken, true); + mappers[i].setCounter(jobCounter); + } + // Non-sampled task, which does not contain time data + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java new file mode 100644 index 000000000..52cea2035 --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java @@ -0,0 +1,84 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import junit.framework.TestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class MapperGCHeuristicTest extends TestCase { + + private static Map paramsMap = new HashMap(); + + private static Heuristic _heuristic = new MapperGCHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + private static int NUMTASKS = 100; + + public void testGCCritical() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(1000000, 50000, 2000)); + } + + public void testGCSevere() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(1000000, 50000, 1500)); + } + + public void testGCModerate() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(1000000, 50000, 1000)); + } + + public void testGCNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(1000000, 50000, 300)); + } + + public void testShortTasksNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(100000, 50000, 2000)); + } + + + private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; + + TezCounterData counter = new TezCounterData(); + counter.set(TezCounterData.CounterName.CPU_MILLISECONDS, cpuMs); + counter.set(TezCounterData.CounterName.GC_TIME_MILLIS, gcMs); + + int i = 0; + for (; i < NUMTASKS; i++) { + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + mappers[i].setTimeAndCounter(new long[]{runtimeMs, 0 , 0, 0, 0}, counter); + } + // Non-sampled task, which does not contain time and counter data + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } + +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java new file mode 100644 index 000000000..72e426743 --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; +import org.apache.commons.io.FileUtils; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class MapperMemoryHeuristicTest extends TestCase{ + + private static Map paramsMap = new HashMap(); + private static Heuristic _heuristic = new MapperMemoryHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + private int NUMTASKS = 100; + + public void testLargeContainerSizeCritical() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(2048, 8192)); + } + + public void testLargeContainerSizeSevere() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(3072, 8192)); + } + + public void testLargeContainerSizeModerate() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(4096, 8192)); + } + + public void testLargeContainerSizeNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(6144, 8192)); + } + + // If the task use default container size, it should not be flagged + // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> CRITICAL + public void testDefaultContainerNone() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(256, 2048)); + } + + // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> MODERATE + public void testDefaultContainerNoneMore() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(1024, 2048)); + } + + private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; + + TezCounterData counter = new TezCounterData(); + counter.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB* FileUtils.ONE_MB); + + Properties p = new Properties(); + p.setProperty(MapperMemoryHeuristic.MAPPER_MEMORY_CONF, Long.toString(containerMemMB)); + + int i = 0; + for (; i < NUMTASKS; i++) { + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + mappers[i].setTime(new long[5]); + mappers[i].setCounter(counter); + } + // Non-sampled task, which does not contain time and counter data + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); + data.setConf(p); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java new file mode 100644 index 000000000..2d21f856a --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java @@ -0,0 +1,100 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.math.Statistics; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; +import org.apache.commons.io.FileUtils; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class MapperSpeedHeuristicTest extends TestCase { + + private static Map paramsMap = new HashMap(); + private static Heuristic _heuristic = new MapperSpeedHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + private static final long MB_IN_BYTES = FileUtils.ONE_MB; + private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS; + private static final int NUMTASKS = 100; + + public void testCritical() throws IOException { + long runtime = 120 * MINUTE_IN_MS; + long speed_factor = (runtime * MB_IN_BYTES) / 1000; + assertEquals(Severity.CRITICAL, analyzeJob(runtime, 1 * speed_factor)); + } + + public void testSevere() throws IOException { + long runtime = 120 * MINUTE_IN_MS; + long speed_factor = (runtime * MB_IN_BYTES) / 1000; + assertEquals(Severity.SEVERE, analyzeJob(runtime, 4 * speed_factor)); + } + + public void testModerate() throws IOException { + long runtime = 120 * MINUTE_IN_MS; + long speed_factor = (runtime * MB_IN_BYTES) / 1000; + assertEquals(Severity.MODERATE, analyzeJob(runtime, 13 * speed_factor)); + } + + public void testLow() throws IOException { + long runtime = 120 * MINUTE_IN_MS; + long speed_factor = (runtime * MB_IN_BYTES) / 1000; + assertEquals(Severity.LOW, analyzeJob(runtime, 50 * speed_factor)); + } + + public void testNone() throws IOException { + long runtime = 120 * MINUTE_IN_MS; + long speed_factor = (runtime * MB_IN_BYTES) / 1000; + assertEquals(Severity.NONE, analyzeJob(runtime, 51 * speed_factor)); + } + + public void testShortTask() throws IOException { + long runtime = 2 * MINUTE_IN_MS; + long speed_factor = (runtime * MB_IN_BYTES) / 1000; + assertEquals(Severity.NONE, analyzeJob(runtime, 1 * speed_factor)); + } + + private Severity analyzeJob(long runtimeMs, long readBytes) throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; + + TezCounterData counter = new TezCounterData(); + counter.set(TezCounterData.CounterName.HDFS_BYTES_READ, readBytes / 2); + counter.set(TezCounterData.CounterName.S3A_BYTES_READ, readBytes / 2); + + int i = 0; + for (; i < NUMTASKS; i++) { + mappers[i] = new TezTaskData(counter, new long[] { runtimeMs, 0, 0 ,0, 0}); + } + // Non-sampled task, which does not contain time and counter data + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java new file mode 100644 index 000000000..62831a727 --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + + +public class MapperSpillHeuristicTest extends TestCase{ + + private static Map paramsMap = new HashMap(); + private static Heuristic _heuristic = new MapperSpillHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + public void testCritical() throws IOException { + // Spill ratio 3.0, 1000 tasks + assertEquals(Severity.CRITICAL, analyzeJob(3000, 1000, 1000)); + } + + public void testSevere() throws IOException { + // Spill ratio 2.5, 1000 tasks + assertEquals(Severity.SEVERE, analyzeJob(2500, 1000, 1000)); + } + + public void testModerate() throws IOException { + // Spill ratio 2.3, 1000 tasks + assertEquals(Severity.MODERATE, analyzeJob(2300, 1000, 1000)); + } + + public void testLow() throws IOException { + // Spill ratio 2.1, 1000 tasks + assertEquals(Severity.LOW, analyzeJob(2100, 1000, 1000)); + } + + public void testNone() throws IOException { + // Spill ratio 1.0, 1000 tasks + assertEquals(Severity.NONE, analyzeJob(1000, 1000, 1000)); + } + + public void testSmallNumTasks() throws IOException { + // Spill ratio 3.0, should be critical, but number of task is small(10), final result is NONE + assertEquals(Severity.NONE, analyzeJob(3000, 1000, 10)); + } + + private Severity analyzeJob(long spilledRecords, long mapRecords, int numTasks) throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] mappers = new TezTaskData[numTasks + 1]; + + TezCounterData counter = new TezCounterData(); + counter.set(TezCounterData.CounterName.SPILLED_RECORDS, spilledRecords); + counter.set(TezCounterData.CounterName.OUTPUT_RECORDS, mapRecords); + + int i = 0; + for (; i < numTasks; i++) { + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + mappers[i].setTimeAndCounter(new long[5], counter); + } + // Non-sampled task, which does not contain time and counter data + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java new file mode 100644 index 000000000..39c6ec891 --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java @@ -0,0 +1,111 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.math.Statistics; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + + +public class MapperTimeHeuristicTest extends TestCase { + + private static final long DUMMY_INPUT_SIZE = 0; + + private static Map paramsMap = new HashMap(); + private static Heuristic _heuristic = new MapperTimeHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + // Test batch 1: Large runtime. Heuristic is not affected by various number of tasks */ + + public void testLongRuntimeTasksCritical() throws IOException { + // Should decrease split size and increase number of tasks + assertEquals(Severity.CRITICAL, analyzeJob(10, 120 * Statistics.MINUTE_IN_MS)); + } + + public void testLongRuntimeTasksCriticalMore() throws IOException { + // Should decrease split size and increase number of tasks + assertEquals(Severity.CRITICAL, analyzeJob(1000, 120 * Statistics.MINUTE_IN_MS)); + } + + public void testLongRuntimeTasksSevere() throws IOException { + // Should decrease split size and increase number of tasks + assertEquals(Severity.SEVERE, analyzeJob(10, 60 * Statistics.MINUTE_IN_MS)); + } + + public void testLongRuntimeTasksSevereMore() throws IOException { + // Should decrease split size and increase number of tasks + assertEquals(Severity.SEVERE, analyzeJob(1000, 60 * Statistics.MINUTE_IN_MS)); + } + + // Test batch 2: Short runtime and various number of tasks + + public void testShortRuntimeTasksCritical() throws IOException { + // Should increase split size and decrease number of tasks + assertEquals(Severity.CRITICAL, analyzeJob(1000, 1 * Statistics.MINUTE_IN_MS)); + } + + public void testShortRuntimeTasksSevere() throws IOException { + // Should increase split size and decrease number of tasks + assertEquals(Severity.SEVERE, analyzeJob(500, 1 * Statistics.MINUTE_IN_MS)); + } + + public void testShortRuntimeTasksModerate() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(101, 1 * Statistics.MINUTE_IN_MS)); + } + + public void testShortRuntimeTasksLow() throws IOException { + assertEquals(Severity.LOW, analyzeJob(50, 1 * Statistics.MINUTE_IN_MS)); + } + + public void testShortRuntimeTasksNone() throws IOException { + // Small file with small number of tasks and short runtime. This should be the common case. + assertEquals(Severity.NONE, analyzeJob(5, 1 * Statistics.MINUTE_IN_MS)); + } + + private Severity analyzeJob(int numTasks, long runtime) throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] mappers = new TezTaskData[numTasks + 1]; + + TezCounterData taskCounter = new TezCounterData(); + taskCounter.set(TezCounterData.CounterName.S3A_BYTES_READ, DUMMY_INPUT_SIZE / 4); + + + int i = 0; + for (; i < numTasks; i++) { + mappers[i] = new TezTaskData(jobCounter,new long[] { runtime, 0, 0, 0, 0 }); + } + // Non-sampled task, which does not contain time and counter data + mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } + + +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java new file mode 100644 index 000000000..ff95e862a --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.*; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class ReducerDataSkewHeuristicTest extends TestCase{ + + private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64; //1mb + private static final long UNITSIZETIME = 1000000; //1000sec + + private static Map paramsMap = new HashMap(); + private static Heuristic _heuristic = new ReducerDataSkewHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + public void testCritical() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(200, 200, 1 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testSevere() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(200, 200, 10 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testModerate() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(200, 200, 20 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testLow() throws IOException { + assertEquals(Severity.LOW, analyzeJob(200, 200, 30 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(200, 200, 50 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testSmallFiles() throws IOException { + assertEquals(Severity.NONE, analyzeJob(200, 200, 1 * UNITSIZE, 5 * UNITSIZE)); + } + + public void testSmallTasks() throws IOException { + assertEquals(Severity.NONE, analyzeJob(5, 5, 10 * UNITSIZE, 100 * UNITSIZE)); + } + + public void testCriticalTime() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJobTime(200, 200, 1 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testSevereTime() throws IOException { + assertEquals(Severity.SEVERE, analyzeJobTime(200, 200, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testModerateTime() throws IOException { + assertEquals(Severity.MODERATE, analyzeJobTime(200, 200, 20 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testLowTime() throws IOException { + assertEquals(Severity.LOW, analyzeJobTime(200, 200, 30 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testNoneTime() throws IOException { + assertEquals(Severity.NONE, analyzeJobTime(200, 200, 50 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + public void testSmallTasksTime() throws IOException { + assertEquals(Severity.NONE, analyzeJobTime(5, 5, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); + } + + private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) + throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] reducers = new TezTaskData[numSmallTasks + numLargeTasks + 1]; + + TezCounterData smallCounter = new TezCounterData(); + smallCounter.set(TezCounterData.CounterName.SHUFFLE_BYTES, smallInputSize); + + TezCounterData largeCounter = new TezCounterData(); + largeCounter.set(TezCounterData.CounterName.SHUFFLE_BYTES, largeInputSize); + + int i = 0; + for (; i < numSmallTasks; i++) { + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + reducers[i].setTimeAndCounter(new long[5], smallCounter); + } + for (; i < numSmallTasks + numLargeTasks; i++) { + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + reducers[i].setTimeAndCounter(new long[5], largeCounter); + } + // Non-sampled task, which does not contain time and counter data + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } + + private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) + throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] reducers = new TezTaskData[numSmallTasks + numLongTasks + 1]; + + int i = 0; + for (; i < numSmallTasks; i++) { + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + reducers[i].setTotalTimeMs(smallTimeTaken, true); + reducers[i].setCounter(jobCounter); + } + for (; i < numSmallTasks + numLongTasks; i++) { + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + reducers[i].setTotalTimeMs(longTimeTaken, true); + reducers[i].setCounter(jobCounter); + } + // Non-sampled task, which does not contain time data + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java new file mode 100644 index 000000000..4a86fbefb --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class ReducerGCHeuristicTest extends TestCase{ + + private static Map paramsMap = new HashMap(); + + private static Heuristic _heuristic = new ReducerGCHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + private static int NUMTASKS = 100; + + public void testGCCritical() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(1000000, 50000, 2000)); + } + + public void testGCSevere() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(1000000, 50000, 1500)); + } + + public void testGCModerate() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(1000000, 50000, 1000)); + } + + public void testGCNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(1000000, 50000, 300)); + } + + public void testShortTasksNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(100000, 50000, 2000)); + } + + + private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] reducers = new TezTaskData[NUMTASKS + 1]; + + TezCounterData counter = new TezCounterData(); + counter.set(TezCounterData.CounterName.CPU_MILLISECONDS, cpuMs); + counter.set(TezCounterData.CounterName.GC_TIME_MILLIS, gcMs); + + int i = 0; + for (; i < NUMTASKS; i++) { + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter); + } + // Non-sampled task, which does not contain time and counter data + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java new file mode 100644 index 000000000..66c3e193c --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java @@ -0,0 +1,93 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; +import org.apache.commons.io.FileUtils; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class ReducerMemoryHeuristicTest extends TestCase { + + private static Map paramsMap = new HashMap(); + private static Heuristic _heuristic = new ReducerMemoryHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + private int NUMTASKS = 100; + + public void testLargeContainerSizeCritical() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(2048, 8192)); + } + + public void testLargeContainerSizeSevere() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(3072, 8192)); + } + + public void testLargeContainerSizeModerate() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(4096, 8192)); + } + + public void testLargeContainerSizeNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(6144, 8192)); + } + + // If the task use default container size, it should not be flagged + // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> CRITICAL + public void testDefaultContainerNone() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(256, 2048)); + } + + // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> MODERATE + public void testDefaultContainerNoneMore() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(1024, 2048)); + } + + private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException { + TezCounterData jobCounter = new TezCounterData(); + TezTaskData[] reducers = new TezTaskData[NUMTASKS + 1]; + + TezCounterData counter = new TezCounterData(); + counter.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB* FileUtils.ONE_MB); + + Properties p = new Properties(); + p.setProperty(com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic.REDUCER_MEMORY_CONF, Long.toString(containerMemMB)); + + int i = 0; + for (; i < NUMTASKS; i++) { + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + reducers[i].setTimeAndCounter(new long[5], counter); + } + // Non-sampled task, which does not contain time and counter data + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); + data.setConf(p); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } +} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java new file mode 100644 index 000000000..ab0abbe38 --- /dev/null +++ b/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2017 Electronic Arts Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + */ +package com.linkedin.drelephant.tez.heuristics; + +import com.linkedin.drelephant.analysis.ApplicationType; +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.math.Statistics; +import com.linkedin.drelephant.tez.data.TezApplicationData; +import com.linkedin.drelephant.tez.data.TezCounterData; +import com.linkedin.drelephant.tez.data.TezTaskData; +import junit.framework.TestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + + +public class ReducerTimeHeuristicTest extends TestCase { + + private static Map paramsMap = new HashMap(); + private static Heuristic _heuristic = new ReducerTimeHeuristic(new HeuristicConfigurationData("test_heuristic", + "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); + + private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS;; + + public void testShortRunetimeCritical() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(1 * MINUTE_IN_MS, 1000)); + } + + public void testShortRunetimeSevere() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(1 * MINUTE_IN_MS, 500)); + } + + public void testShortRunetimeModerate() throws IOException { + assertEquals(Severity.MODERATE, analyzeJob(1 * MINUTE_IN_MS, 101)); + } + + public void testShortRunetimeLow() throws IOException { + assertEquals(Severity.LOW, analyzeJob(1 * MINUTE_IN_MS, 50)); + } + + public void testShortRunetimeNone() throws IOException { + assertEquals(Severity.NONE, analyzeJob(1 * MINUTE_IN_MS, 2)); + } + + public void testLongRunetimeCritical() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 10)); + } + + // Long runtime severity is not affected by number of tasks + public void testLongRunetimeCriticalMore() throws IOException { + assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 1000)); + } + + public void testLongRunetimeSevere() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(60 * MINUTE_IN_MS, 10)); + } + + public void testLongRunetimeSevereMore() throws IOException { + assertEquals(Severity.SEVERE, analyzeJob(60 * MINUTE_IN_MS, 1000)); + } + + private Severity analyzeJob(long runtimeMs, int numTasks) throws IOException { + TezCounterData dummyCounter = new TezCounterData(); + TezTaskData[] reducers = new TezTaskData[numTasks + 1]; + + int i = 0; + for (; i < numTasks; i++) { + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + reducers[i].setTime(new long[] { runtimeMs, 0, 0, 0, 0 }); + reducers[i].setCounter(dummyCounter); + } + // Non-sampled task, which does not contain time and counter data + reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); + + TezApplicationData data = new TezApplicationData().setCounters(dummyCounter).setReduceTaskData(reducers); + HeuristicResult result = _heuristic.apply(data); + return result.getSeverity(); + } +} \ No newline at end of file From 51fe80a35055e9de9167b8f60f71e035e72e0a94 Mon Sep 17 00:00:00 2001 From: Arpan Agrawal Date: Tue, 10 Jul 2018 22:55:01 -0700 Subject: [PATCH 11/15] Auto tuning: Support for parameter set multi-try (#386) --- .../tuning/AutoTuningAPIHelper.java | 562 +++++++++--------- .../tuning/AzkabanJobCompleteDetector.java | 59 +- .../tuning/BaselineComputeUtil.java | 13 +- .../drelephant/tuning/FitnessComputeUtil.java | 403 ++++++++----- .../tuning/JobCompleteDetector.java | 87 +-- .../drelephant/tuning/JobTuningInfo.java | 2 - .../drelephant/tuning/PSOParamGenerator.java | 20 +- .../drelephant/tuning/ParamGenerator.java | 183 +++--- .../linkedin/drelephant/tuning/Particle.java | 1 - .../drelephant/tuning/TuningInput.java | 48 +- app/controllers/Application.java | 41 +- app/models/FlowDefinition.java | 24 + app/models/FlowExecution.java | 24 + app/models/JobDefinition.java | 17 +- app/models/JobExecution.java | 31 +- app/models/JobSavedState.java | 14 + app/models/JobSuggestedParamSet.java | 118 ++++ app/models/JobSuggestedParamValue.java | 30 +- app/models/TuningAlgorithm.java | 15 + app/models/TuningJobDefinition.java | 16 +- ...n.java => TuningJobExecutionParamSet.java} | 60 +- app/models/TuningParameter.java | 16 +- conf/evolutions/default/2.sql | 18 +- conf/evolutions/default/3.sql | 16 + conf/evolutions/default/4.sql | 16 + conf/evolutions/default/5.sql | 109 ++-- conf/evolutions/default/6.sql | 11 - .../tuning/PSOParamGeneratorTest.java | 16 +- test/resources/AutoTuningConf.xml | 4 +- test/resources/test-init.sql | 39 +- test/resources/tunein-test1.sql | 56 +- test/rest/RestAPITest.java | 43 +- 32 files changed, 1269 insertions(+), 843 deletions(-) create mode 100644 app/models/JobSuggestedParamSet.java rename app/models/{TuningJobExecution.java => TuningJobExecutionParamSet.java} (50%) delete mode 100644 conf/evolutions/default/6.sql diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index 14346fa17..b6b5b68ef 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -19,21 +19,27 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.util.Utils; + import controllers.AutoTuningMetricsController; + +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; + import models.FlowDefinition; import models.FlowExecution; import models.JobDefinition; import models.JobExecution; import models.JobExecution.ExecutionState; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamSet.ParamSetStatus; import models.JobSuggestedParamValue; import models.TuningAlgorithm; import models.TuningJobDefinition; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; +import models.TuningJobExecutionParamSet; import models.TuningParameter; + import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; @@ -44,96 +50,46 @@ */ public class AutoTuningAPIHelper { - public static final String ALLOWED_MAX_RESOURCE_USAGE_PERCENT_DEFAULT = + private static final String ALLOWED_MAX_RESOURCE_USAGE_PERCENT_DEFAULT = "autotuning.default.allowed_max_resource_usage_percent"; - public static final String ALLOWED_MAX_EXECUTION_TIME_PERCENT_DEFAULT = + private static final String ALLOWED_MAX_EXECUTION_TIME_PERCENT_DEFAULT = "autotuning.default.allowed_max_execution_time_percent"; private static final Logger logger = Logger.getLogger(AutoTuningAPIHelper.class); /** - * For a job, returns the execution with the best parameter set if available else the one with the default parameter set. + * For a job, returns the best parameter set of the given job if it exists else the default parameter set * @param jobDefId Sting JobDefId of the job - * @return TuningJobExecution with the best parameter set if available else the one with the default parameter set. + * @return JobSuggestedParamSet the best parameter set of the given job if it exists else the default parameter set */ - - private TuningJobExecution getBestParamSetTuningJobExecution(String jobDefId) { - TuningJobExecution tuningJobExecutionBestParamSet = TuningJobExecution.find.select("*") + private JobSuggestedParamSet getBestParamSet(String jobDefId) { + JobSuggestedParamSet jobSuggestedParamSetBestParamSet = JobSuggestedParamSet.find.select("*") .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.jobDefId, - jobDefId) - .eq(TuningJobExecution.TABLE.isParamSetBest, true) + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, true) .setMaxRows(1) .findUnique(); - if (tuningJobExecutionBestParamSet == null) { - tuningJobExecutionBestParamSet = TuningJobExecution.find.select("*") + if (jobSuggestedParamSetBestParamSet == null) { + jobSuggestedParamSetBestParamSet = JobSuggestedParamSet.find.select("*") .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.jobDefId, - jobDefId) - .eq(TuningJobExecution.TABLE.isDefaultExecution, true) + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, true) .setMaxRows(1) .findUnique(); } - return tuningJobExecutionBestParamSet; + return jobSuggestedParamSetBestParamSet; } /** - * Returns the param set corresponding to a given job execution id - * @param jobExecutionId Long job execution id of the execution - * @return List: list of parameters + * Returns the param values corresponding to the given param set id + * @param paramSetId Long parameter set id + * @return List list of parameters */ - private List getJobExecutionParamSet(Long jobExecutionId) { - return JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobExecution + "." + JobExecution.TABLE.id, jobExecutionId) + private List getParamSetValues(Long paramSetId) { + List jobSuggestedParamValues = JobSuggestedParamValue.find.where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, paramSetId) .findList(); - } - - /** - * This method creates a job execution with best parameter set. This is required when there is no parameters set or tuning has been switch off for the job - * remains for suggestion. - * @param tuningJobDefinition Job definition - * @return Tuning Job Execution with best parameters - */ - private TuningJobExecution cloneBestParamSetTuningJobExecution(TuningJobDefinition tuningJobDefinition) { - logger.info("Searching for best param set for job: " + tuningJobDefinition.job.jobName); - - TuningJobExecution bestParamSetTuningJobExecution = - getBestParamSetTuningJobExecution(tuningJobDefinition.job.jobDefId); - List jobSuggestedParamValueList = - getJobExecutionParamSet(bestParamSetTuningJobExecution.jobExecution.id); - - TuningJobExecution tuningJobExecution = new TuningJobExecution(); - JobExecution jobExecution = new JobExecution(); - jobExecution.id = 0L; - jobExecution.job = bestParamSetTuningJobExecution.jobExecution.job; - jobExecution.executionState = ExecutionState.NOT_STARTED; - jobExecution.save(); - - tuningJobExecution.jobExecution = jobExecution; - tuningJobExecution.isDefaultExecution = bestParamSetTuningJobExecution.isDefaultExecution; - tuningJobExecution.tuningAlgorithm = bestParamSetTuningJobExecution.tuningAlgorithm; - tuningJobExecution.paramSetState = ParamSetStatus.CREATED; - tuningJobExecution.save(); - - logger.debug("Execution with default parameter created with execution id: " + tuningJobExecution.jobExecution.id); - - //Save default parameters corresponding to new default execution - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - JobSuggestedParamValue jobSuggestedParamValue1 = new JobSuggestedParamValue(); - jobSuggestedParamValue1.id = 0; - jobSuggestedParamValue1.jobExecution = jobExecution; - jobSuggestedParamValue1.paramValue = jobSuggestedParamValue.paramValue; - jobSuggestedParamValue1.tuningParameter = jobSuggestedParamValue.tuningParameter; - jobSuggestedParamValue1.save(); - } - - tuningJobExecution = TuningJobExecution.find.select("*") - .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.id, tuningJobExecution.jobExecution.id) - .setMaxRows(1) - .findUnique(); - - return tuningJobExecution; + return jobSuggestedParamValues; } /** @@ -158,27 +114,39 @@ private void setMaxAllowedMetricIncreasePercentage(TuningInput tuningInput) { * Sets the tuning algorithm based on the job type and optimization metric * @param tuningInput TuningInput for which tuning algorithm is to be set */ - private void setTuningAlgorithm(TuningInput tuningInput) { + private void setTuningAlgorithm(TuningInput tuningInput) throws IllegalArgumentException { //Todo: Handle algorithm version later TuningAlgorithm tuningAlgorithm = TuningAlgorithm.find.select("*") .where() .eq(TuningAlgorithm.TABLE.jobType, tuningInput.getJobType()) .eq(TuningAlgorithm.TABLE.optimizationMetric, tuningInput.getOptimizationMetric()) .findUnique(); + if (tuningAlgorithm == null) { + throw new IllegalArgumentException( + "Wrong job type or optimization metric. Job Type " + tuningInput.getJobType() + ". Optimization Metrics: " + + tuningInput.getOptimizationMetric()); + } tuningInput.setTuningAlgorithm(tuningAlgorithm); } /** - * Applies penalty to the given execution - * @param jobExecId String jobExecId of the execution to which penalty has to be applied + * Applies penalty to the param set corresponding to the given execution + * @param jobExecId String job execution id/url of the execution whose parameter set has to be penalized + * Assumption: Best param set will never be penalized */ private void applyPenalty(String jobExecId) { Integer penaltyConstant = 3; logger.info("Execution " + jobExecId + " failed/cancelled. Applying penalty"); - TuningJobExecution tuningJobExecution = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.jobExecution + '.' + JobExecution.TABLE.jobExecId, jobExecId) + + TuningJobExecutionParamSet tuningJobExecutionParamSet = TuningJobExecutionParamSet.find.where() + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.jobExecId, jobExecId) + .setMaxRows(1) .findUnique(); - JobDefinition jobDefinition = tuningJobExecution.jobExecution.job; + + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + JobDefinition jobDefinition = jobExecution.job; + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) .findUnique(); @@ -186,11 +154,12 @@ private void applyPenalty(String jobExecId) { tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; Double maxDesiredResourceUsagePerGBInput = averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - tuningJobExecution.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - tuningJobExecution.paramSetState = ParamSetStatus.FITNESS_COMPUTED; - tuningJobExecution.update(); - JobExecution jobExecution = tuningJobExecution.jobExecution; + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + jobSuggestedParamSet.paramSetState = ParamSetStatus.FITNESS_COMPUTED; + jobSuggestedParamSet.fitnessJobExecution = jobExecution; + jobSuggestedParamSet.update(); + jobExecution.resourceUsage = 0D; jobExecution.executionTime = 0D; jobExecution.inputSizeInBytes = 1D; @@ -198,128 +167,30 @@ private void applyPenalty(String jobExecId) { } /** - * Handles the api request and returns param suggestions as response - * @param tuningInput Rest api parameters - * @return Parameter Suggestion + * Returns flow definition corresponding to the given tuning input if it exists, else creates one and returns it + * @param tuningInput TuningInput containing the flow definition id corresponding to which flow definition + * is to be returned + * @return FlowDefinition flow definition */ - public Map getCurrentRunParameters(TuningInput tuningInput) { - logger.info("Parameter suggestion request for execution: " + tuningInput.getJobExecId()); - List jobSuggestedParamValues; - - if (tuningInput.getAllowedMaxExecutionTimePercent() == null - || tuningInput.getAllowedMaxResourceUsagePercent() == null) { - setMaxAllowedMetricIncreasePercentage(tuningInput); - } - setTuningAlgorithm(tuningInput); - String jobDefId = tuningInput.getJobDefId(); - - if (tuningInput.getRetry()) { - applyPenalty(tuningInput.getJobExecId()); - TuningJobExecution bestParamSetTuningJobExecution = getBestParamSetTuningJobExecution(jobDefId); - jobSuggestedParamValues = getJobExecutionParamSet(bestParamSetTuningJobExecution.jobExecution.id); - } else { - boolean isJobNewToTuning = false; - boolean isTuningEnabledForJob; - - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId) - .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) - .findUnique(); - - isTuningEnabledForJob = tuningJobDefinition != null; - - if (!isTuningEnabledForJob) { - //Tuning not enabled for the job currently. Either the job is new to tuning or tuning has been turned off for the job - //TuningJobDefinition will have a unique entry for every time a job is turned on for tuning - tuningJobDefinition = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId) - .setMaxRows(1) - .orderBy(TuningJobDefinition.TABLE.createdTs + " desc") - .findUnique(); - - isJobNewToTuning = tuningJobDefinition == null; - - if (isJobNewToTuning) { - //The job is new to tuning - logger.debug("Registering job: " + tuningInput.getJobName() + " for auto tuning tuning"); - AutoTuningMetricsController.markNewAutoTuningJob(); - tuningJobDefinition = addNewJobForTuning(tuningInput); - } - } - - TuningJobExecution tuningJobExecution; - if (isJobNewToTuning || isTuningEnabledForJob) { - logger.debug("Finding parameter suggestion for job: " + tuningJobDefinition.job.jobName); - tuningJobExecution = getNewTuningJobExecution(tuningJobDefinition); - } else { - //Tuning has been switched off for the job. Returning best param set - tuningJobExecution = cloneBestParamSetTuningJobExecution(tuningJobDefinition); - } - updateJobExecutionParameter(tuningJobExecution, tuningInput); - jobSuggestedParamValues = getJobExecutionParamSet(tuningJobExecution.jobExecution.id); - } - logger.debug("Number of output parameters : " + jobSuggestedParamValues.size()); - logger.info("Finishing getCurrentRunParameters"); - return jobSuggestedParamValueListToMap(jobSuggestedParamValues); - } - - /** - * Returns an execution with unsent parameters corresponding to the given job definition - * @param tuningJobDefinition TuningJobDefinition corresponding to which execution is to be returned - * @return TuningJobExecution corresponding to the given job definition - */ - private TuningJobExecution getNewTuningJobExecution(TuningJobDefinition tuningJobDefinition) { - TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*") - .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.id, - tuningJobDefinition.job.id) - .eq(TuningJobExecution.TABLE.paramSetState, ParamSetStatus.CREATED) - .order() - .asc(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.createdTs) - .setMaxRows(1) - .findUnique(); - - //If no new parameter set for suggestion, create a new suggestion with best parameter set - if (tuningJobExecution == null) { - logger.info( - "Returning best parameter set as no parameter suggestion found for job: " + tuningJobDefinition.job.jobName); - AutoTuningMetricsController.markParamSetNotFound(); - tuningJobExecution = cloneBestParamSetTuningJobExecution(tuningJobDefinition); - } - return tuningJobExecution; - } - - /** - * Returns the list of JobSuggestedParamValue as Map of String to Double - * @param jobSuggestedParamValues List of JobSuggestedParamValue - * @return Map of string to double containing the parameter name and corresponding value - */ - private Map jobSuggestedParamValueListToMap(List jobSuggestedParamValues) { - Map paramValues = new HashMap(); - if (jobSuggestedParamValues != null) { - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { - logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " - + jobSuggestedParamValue.paramValue); - paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); - } + private FlowDefinition getFlowDefinition(TuningInput tuningInput) { + FlowDefinition flowDefinition = + FlowDefinition.find.where().eq(FlowDefinition.TABLE.flowDefId, tuningInput.getFlowDefId()).findUnique(); + if (flowDefinition == null) { + flowDefinition = new FlowDefinition(); + flowDefinition.flowDefId = tuningInput.getFlowDefId(); + flowDefinition.flowDefUrl = tuningInput.getFlowDefUrl(); + flowDefinition.save(); } - return paramValues; + return flowDefinition; } /** - *This is to update job execution with IN_PROGRESS and parameter set with IN_PROGRESS. Also update flow_exec_id - *, flowExecURL, JobExecID and jobExecURL - * @param tuningJobExecution TuningJobExecution which is to be updated - * @param tuningInput TuningInput corresponding to the TuningJobExecution + * Returns flow execution corresponding to the given tuning input if it exists, else creates one and returns it + * @param tuningInput TuningInput containing the flow execution id corresponding to which flow execution + * is to be returned + * @return FlowExecution flow execution */ - private void updateJobExecutionParameter(TuningJobExecution tuningJobExecution, TuningInput tuningInput) { - + private FlowExecution getFlowExecution(TuningInput tuningInput) { FlowExecution flowExecution = FlowExecution.find.where().eq(FlowExecution.TABLE.flowExecId, tuningInput.getFlowExecId()).findUnique(); @@ -327,47 +198,23 @@ private void updateJobExecutionParameter(TuningJobExecution tuningJobExecution, flowExecution = new FlowExecution(); flowExecution.flowExecId = tuningInput.getFlowExecId(); flowExecution.flowExecUrl = tuningInput.getFlowExecUrl(); - flowExecution.flowDefinition = tuningJobExecution.jobExecution.job.flowDefinition; + flowExecution.flowDefinition = getFlowDefinition(tuningInput); flowExecution.save(); } - - JobExecution jobExecution = tuningJobExecution.jobExecution; - jobExecution.jobExecId = tuningInput.getJobExecId(); - jobExecution.jobExecUrl = tuningInput.getJobExecUrl(); - jobExecution.executionState = ExecutionState.IN_PROGRESS; - jobExecution.flowExecution = flowExecution; - - logger.debug("Saving job execution" + jobExecution.jobExecId); - - jobExecution.save(); - - tuningJobExecution.jobExecution = jobExecution; - tuningJobExecution.paramSetState = ParamSetStatus.SENT; - tuningJobExecution.save(); + return flowExecution; } /** - * Add new job for tuning + * Adds new job for tuning * @param tuningInput Tuning input parameters * @return Job */ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { - logger.info("Adding new job for tuning, job id: " + tuningInput.getJobDefId()); - + FlowDefinition flowDefinition = getFlowDefinition(tuningInput); JobDefinition job = JobDefinition.find.select("*").where().eq(JobDefinition.TABLE.jobDefId, tuningInput.getJobDefId()).findUnique(); - FlowDefinition flowDefinition = - FlowDefinition.find.where().eq(FlowDefinition.TABLE.flowDefId, tuningInput.getFlowDefId()).findUnique(); - - if (flowDefinition == null) { - flowDefinition = new FlowDefinition(); - flowDefinition.flowDefId = tuningInput.getFlowDefId(); - flowDefinition.flowDefUrl = tuningInput.getFlowDefUrl(); - flowDefinition.save(); - } - if (job == null) { job = new JobDefinition(); job.jobDefId = tuningInput.getJobDefId(); @@ -379,99 +226,231 @@ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { job.save(); } - String flowExecId = tuningInput.getFlowExecId(); - String jobExecId = tuningInput.getJobExecId(); - String flowExecUrl = tuningInput.getFlowExecUrl(); - String jobExecUrl = tuningInput.getJobExecUrl(); String client = tuningInput.getClient(); - String defaultParams = tuningInput.getDefaultParams(); - + Map defaultParams = null; + try { + defaultParams = tuningInput.getDefaultParams(); + } catch (IOException e) { + logger.error("Error in getting default parameters from request. ", e); + } TuningJobDefinition tuningJobDefinition = new TuningJobDefinition(); tuningJobDefinition.job = job; tuningJobDefinition.client = client; tuningJobDefinition.tuningAlgorithm = tuningInput.getTuningAlgorithm(); - tuningJobDefinition.tuningEnabled = 1; + tuningJobDefinition.tuningEnabled = true; tuningJobDefinition.allowedMaxExecutionTimePercent = tuningInput.getAllowedMaxExecutionTimePercent(); tuningJobDefinition.allowedMaxResourceUsagePercent = tuningInput.getAllowedMaxResourceUsagePercent(); tuningJobDefinition.save(); - TuningJobExecution tuningJobExecution = - insertDefaultJobExecution(job, flowExecId, jobExecId, flowExecUrl, jobExecUrl, flowDefinition, - tuningInput.getTuningAlgorithm()); - insertDefaultParameters(tuningJobExecution, defaultParams); + insertParamSet(job, tuningInput.getTuningAlgorithm(), defaultParams); logger.info("Added job: " + tuningInput.getJobDefId() + " for tuning"); return tuningJobDefinition; } /** - * Inserts default job execution in database - * @param job Job - * @param flowExecId Flow execution id - * @param jobExecId Job execution id - * @param flowExecUrl Flow execution url - * @param jobExecUrl Job execution url - * @return default job execution + * Returns the job definition corresponding to the given tuning input if it exists, else creates one and returns it + * @param tuningInput Tuning Input corresponding to which job definition is to be returned + * @return JobDefinition corresponding to the given tuning input */ - private TuningJobExecution insertDefaultJobExecution(JobDefinition job, String flowExecId, String jobExecId, - String flowExecUrl, String jobExecUrl, FlowDefinition flowDefinition, TuningAlgorithm tuningAlgorithm) { - logger.debug("Starting insertDefaultJobExecution"); + private JobDefinition getJobDefinition(TuningInput tuningInput) { - FlowExecution flowExecution = - FlowExecution.find.where().eq(FlowExecution.TABLE.flowExecId, flowExecId).findUnique(); + String jobDefId = tuningInput.getJobDefId(); - if (flowExecution == null) { - flowExecution = new FlowExecution(); - flowExecution.flowExecId = flowExecId; - flowExecution.flowExecUrl = flowExecUrl; - flowExecution.flowDefinition = flowDefinition; - flowExecution.save(); + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .setMaxRows(1) + .orderBy(TuningJobDefinition.TABLE.createdTs + " desc") + .findUnique(); + + if (tuningJobDefinition == null) { + // Job new to tuning + logger.debug("Registering job: " + tuningInput.getJobName() + " for auto tuning tuning"); + AutoTuningMetricsController.markNewAutoTuningJob(); + tuningJobDefinition = addNewJobForTuning(tuningInput); } + return tuningJobDefinition.job; + } + + /** + * Creates a new job execution entry corresponding to the given tuning input + * @param tuningInput Input corresponding to which job execution is to be created + * @return JobExecution: the newly created job execution + */ + private JobExecution addNewExecution(TuningInput tuningInput) { + JobDefinition jobDefinition = getJobDefinition(tuningInput); + FlowExecution flowExecution = getFlowExecution(tuningInput); + + JobExecution jobExecution = new JobExecution(); + jobExecution.jobExecId = tuningInput.getJobExecId(); + jobExecution.jobExecUrl = tuningInput.getJobExecUrl(); + jobExecution.job = jobDefinition; + jobExecution.executionState = ExecutionState.IN_PROGRESS; + jobExecution.flowExecution = flowExecution; + jobExecution.save(); + return jobExecution; + } - JobExecution jobExecution = JobExecution.find.where().eq(JobExecution.TABLE.jobExecId, jobExecId).findUnique(); + /** + * Returns the job execution corresponding to the given tuning input if it exists, else creates one and returns it + * @param tuningInput Tuning Input corresponding to which job execution is to be returned + * @return JobExecution corresponding to the given tuning input + */ + private JobExecution getJobExecution(TuningInput tuningInput) { + + JobExecution jobExecution = JobExecution.find.select("*") + .fetch(JobExecution.TABLE.job, "*") + .where() + .eq(JobExecution.TABLE.jobExecId, tuningInput.getJobExecId()) + .findUnique(); if (jobExecution == null) { - jobExecution = new JobExecution(); - jobExecution.job = job; - jobExecution.executionState = ExecutionState.NOT_STARTED; - jobExecution.jobExecId = jobExecId; - jobExecution.jobExecUrl = jobExecUrl; - jobExecution.flowExecution = flowExecution; - jobExecution.save(); + jobExecution = addNewExecution(tuningInput); + } + return jobExecution; + } + + /** + * Handles the api request and returns param suggestions as response + * @param tuningInput Rest api parameters + * @return Parameter Suggestion + */ + public Map getCurrentRunParameters(TuningInput tuningInput) throws Exception { + logger.info("Parameter set request received from execution: " + tuningInput.getJobExecId()); + + if (tuningInput.getAllowedMaxExecutionTimePercent() == null + || tuningInput.getAllowedMaxResourceUsagePercent() == null) { + setMaxAllowedMetricIncreasePercentage(tuningInput); } + setTuningAlgorithm(tuningInput); - TuningJobExecution tuningJobExecution = new TuningJobExecution(); - tuningJobExecution.jobExecution = jobExecution; - tuningJobExecution.tuningAlgorithm = tuningAlgorithm; - tuningJobExecution.paramSetState = ParamSetStatus.CREATED; - tuningJobExecution.isDefaultExecution = true; - tuningJobExecution.save(); + JobSuggestedParamSet jobSuggestedParamSet; + JobExecution jobExecution = getJobExecution(tuningInput); - logger.debug("Finishing insertDefaultJobExecution. Job Execution ID " + jobExecution.jobExecId); + if (tuningInput.getRetry()) { + applyPenalty(tuningInput.getJobExecId()); + jobSuggestedParamSet = getBestParamSet(tuningInput.getJobDefId()); + } else { + logger.debug("Finding parameter suggestion for job: " + jobExecution.job.jobName); + jobSuggestedParamSet = getNewSuggestedParamSet(jobExecution.job); + markParameterSetSent(jobSuggestedParamSet); + } + + addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution); - return tuningJobExecution; + List jobSuggestedParamValues = getParamSetValues(jobSuggestedParamSet.id); + logger.debug("Number of output parameters for execution " + tuningInput.getJobExecId() + " = " + + jobSuggestedParamValues.size()); + logger.info("Finishing getCurrentRunParameters"); + return jobSuggestedParamValueListToMap(jobSuggestedParamValues); } /** - * Inserts default execution parameters in database - * @param tuningJobExecution Tuning Job Execution - * @param defaultParams Default parameters map as string + * Adds a new entry to the "tuning_job_execution_param_set" for the given param set and job execution + * @param jobSuggestedParamSet JobSuggestedParamSet: param set + * @param jobExecution JobExecution */ - @SuppressWarnings("unchecked") - private void insertDefaultParameters(TuningJobExecution tuningJobExecution, String defaultParams) { - JobExecution jobExecution = tuningJobExecution.jobExecution; - TuningAlgorithm.JobType jobType = tuningJobExecution.tuningAlgorithm.jobType; + private void addNewTuningJobExecutionParamSet(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution) { + TuningJobExecutionParamSet tuningJobExecutionParamSet = new TuningJobExecutionParamSet(); + tuningJobExecutionParamSet.jobSuggestedParamSet = jobSuggestedParamSet; + tuningJobExecutionParamSet.jobExecution = jobExecution; - ObjectMapper mapper = new ObjectMapper(); - Map paramValueMap = null; - try { - paramValueMap = (Map) mapper.readValue(defaultParams, Map.class); - } catch (Exception e) { - logger.error(e); + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobExecution.job.id) + .order() + .desc(TuningJobDefinition.TABLE.createdTs) + .setMaxRows(1) + .findUnique(); + tuningJobExecutionParamSet.tuningEnabled = tuningJobDefinition.tuningEnabled; + tuningJobExecutionParamSet.save(); + } + + /** + * Returns a parameter set in "CREATED" state corresponding to the given job definition if it exists, else returns + * the best parameter set + * @param jobDefinition jobDefinition for which param set is to be returned + * @return JobSuggestedParamSet corresponding to the given job definition + */ + private JobSuggestedParamSet getNewSuggestedParamSet(JobDefinition jobDefinition) { + JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) + .order() + .asc(JobSuggestedParamSet.TABLE.id) + .setMaxRows(1) + .findUnique(); + + if (jobSuggestedParamSet == null) { + //No new parameter set exists, returning the best parameter set + logger.info("Returning best parameter set as no parameter suggestion found for job: " + jobDefinition.jobName); + AutoTuningMetricsController.markParamSetNotFound(); + jobSuggestedParamSet = getBestParamSet(jobDefinition.jobDefId); } + return jobSuggestedParamSet; + } + + /** + * Returns the list of JobSuggestedParamValue as Map of String to Double + * @param jobSuggestedParamValues List of JobSuggestedParamValue + * @return Map of string to double containing the parameter name and corresponding value + */ + private Map jobSuggestedParamValueListToMap(List jobSuggestedParamValues) { + Map paramValues = new HashMap(); + if (jobSuggestedParamValues != null) { + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { + logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " + + jobSuggestedParamValue.paramValue); + paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); + } + } + return paramValues; + } + + /** + *Updates parameter set state to SENT if it is in CREATED state + * @param jobSuggestedParamSet JobSuggestedParamSet which is to be updated + */ + private void markParameterSetSent(JobSuggestedParamSet jobSuggestedParamSet) { + if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.CREATED)) { + logger.info("Marking paramSetID: " + jobSuggestedParamSet.id + " SENT"); + jobSuggestedParamSet.paramSetState = ParamSetStatus.SENT; + jobSuggestedParamSet.save(); + } + } + + /** + * Inserts a parameter set in database + * @param job Job + */ + private void insertParamSet(JobDefinition job, TuningAlgorithm tuningAlgorithm, Map paramValueMap) { + logger.debug("Inserting default parameter set for job: " + job.jobName); + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningAlgorithm; + jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; + jobSuggestedParamSet.isParamSetDefault = true; + jobSuggestedParamSet.areConstraintsViolated = false; + jobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.save(); + insertParameterValues(jobSuggestedParamSet, paramValueMap); + logger.debug("Default parameter set inserted for job: " + job.jobName); + } + + /** + * Inserts parameter values in database + * @param jobSuggestedParamSet Set of the parameters which is to be inserted + * @param paramValueMap Map of parameter values as string + */ + @SuppressWarnings("unchecked") + private void insertParameterValues(JobSuggestedParamSet jobSuggestedParamSet, Map paramValueMap) { + ObjectMapper mapper = new ObjectMapper(); if (paramValueMap != null) { for (Map.Entry paramValue : paramValueMap.entrySet()) { - insertExecutionParameter(jobExecution, paramValue.getKey(), paramValue.getValue()); + insertParameterValue(jobSuggestedParamSet, paramValue.getKey(), paramValue.getValue()); } } else { logger.warn("ParamValueMap is null "); @@ -479,24 +458,21 @@ private void insertDefaultParameters(TuningJobExecution tuningJobExecution, Stri } /** - * Inserts parameter of an execution in database - * @param jobExecution Job execution + * Inserts parameter value in database + * @param jobSuggestedParamSet Parameter set to which the parameter belongs * @param paramName Parameter name * @param paramValue Parameter value */ - private void insertExecutionParameter(JobExecution jobExecution, String paramName, Double paramValue) { - logger.debug("Starting insertExecutionParameter"); + private void insertParameterValue(JobSuggestedParamSet jobSuggestedParamSet, String paramName, Double paramValue) { + logger.debug("Starting insertParameterValue"); JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); - jobSuggestedParamValue.jobExecution = jobExecution; + jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; TuningParameter tuningParameter = TuningParameter.find.where().eq(TuningParameter.TABLE.paramName, paramName).findUnique(); if (tuningParameter != null) { jobSuggestedParamValue.tuningParameter = tuningParameter; jobSuggestedParamValue.paramValue = paramValue; jobSuggestedParamValue.save(); - logger.debug( - "Finishing insertDefaultJobExecution. Job Execution ID. Param ID " + jobSuggestedParamValue.tuningParameter.id - + " Param Name: " + jobSuggestedParamValue.tuningParameter.paramName); } else { logger.warn("TuningAlgorithm param null " + paramName); } diff --git a/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java b/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java index 9e527791f..4bd0113cc 100644 --- a/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java +++ b/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java @@ -24,8 +24,9 @@ import java.util.Map; import models.JobExecution; import models.JobExecution.ExecutionState; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamSet.ParamSetStatus; +import models.TuningJobExecutionParamSet; import org.apache.log4j.Logger; @@ -39,26 +40,27 @@ public class AzkabanJobCompleteDetector extends JobCompleteDetector { private AzkabanJobStatusUtil _azkabanJobStatusUtil; public enum AzkabanJobStatus { - FAILED, CANCELLED, KILLED, SUCCEEDED + FAILED, CANCELLED, KILLED, SUCCEEDED, SKIPPED } /** * Returns the list of completed executions - * @param jobExecutions Started Execution list + * @param inProgressExecutionParamSet List of executions (with corresponding param set) in progress * @return List of completed executions - * @throws MalformedURLException - * @throws URISyntaxException + * @throws MalformedURLException MalformedURLException + * @throws URISyntaxException URISyntaxException */ - protected List getCompletedExecutions(List jobExecutions) + protected List getCompletedExecutions(List inProgressExecutionParamSet) throws MalformedURLException, URISyntaxException { logger.info("Fetching the list of executions completed since last iteration"); - List completedExecutions = new ArrayList(); + List completedExecutions = new ArrayList(); try { - for (TuningJobExecution tuningJobExecution : jobExecutions) { + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : inProgressExecutionParamSet) { - JobExecution jobExecution = tuningJobExecution.jobExecution; + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; - logger.info("Checking current status of started execution: " + tuningJobExecution.jobExecution.jobExecId); + logger.info("Checking current status of started execution: " + jobExecution.jobExecId); if (_azkabanJobStatusUtil == null) { logger.info("Initializing AzkabanJobStatusUtil"); @@ -72,23 +74,32 @@ protected List getCompletedExecutions(List tuningJobDefinitions) { */ private List getJobForBaselineComputation() { logger.info("Fetching jobs for which baseline metrics need to be computed"); - List tuningJobDefinitions = new ArrayList(); - try { - tuningJobDefinitions = - TuningJobDefinition.find.where().eq(TuningJobDefinition.TABLE.averageResourceUsage, null).findList(); - } catch (NullPointerException e) { - logger.info("There are no jobs for which baseline has to be computed", e); - } + List tuningJobDefinitions = + TuningJobDefinition.find.where().eq(TuningJobDefinition.TABLE.averageResourceUsage, null).findList(); return tuningJobDefinitions; } diff --git a/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java b/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java index 658ecbbee..bf6f64d66 100644 --- a/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java +++ b/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java @@ -34,11 +34,12 @@ import models.AppResult; import models.JobDefinition; import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamSet.ParamSetStatus; import models.JobSuggestedParamValue; import models.TuningAlgorithm; import models.TuningJobDefinition; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; +import models.TuningJobExecutionParamSet; import models.TuningParameter; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -48,20 +49,53 @@ /** * This class computes the fitness of the suggested parameters after the execution is complete. This uses * Dr Elephant's DB to compute the fitness. - * Fitness is : Resource Usage/Input Size in GB + * Fitness is : Resource Usage/(Input Size in GB) * In case there is failure or resource usage/execution time goes beyond configured limit, fitness is computed by * adding a penalty. */ public class FitnessComputeUtil { private static final Logger logger = Logger.getLogger(FitnessComputeUtil.class); private static final String FITNESS_COMPUTE_WAIT_INTERVAL = "fitness.compute.wait_interval.ms"; - private static final int MAX_TUNING_EXECUTIONS = 39; - private static final int MIN_TUNING_EXECUTIONS = 18; - private Long waitInterval; + private static final String IGNORE_EXECUTION_WAIT_INTERVAL = "ignore.execution.wait.interval.ms"; + private static final String MAX_TUNING_EXECUTIONS = "max.tuning.executions"; + private static final String MIN_TUNING_EXECUTIONS = "min.tuning.executions"; + private int maxTuningExecutions; + private int minTuningExecutions; + private Long fitnessComputeWaitInterval; + private Long ignoreExecutionWaitInterval; public FitnessComputeUtil() { Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - waitInterval = Utils.getNonNegativeLong(configuration, FITNESS_COMPUTE_WAIT_INTERVAL, 5 * AutoTuner.ONE_MIN); + + // Time duration to wait for computing the fitness of a param set once the corresponding execution is completed + fitnessComputeWaitInterval = + Utils.getNonNegativeLong(configuration, FITNESS_COMPUTE_WAIT_INTERVAL, 5 * AutoTuner.ONE_MIN); + + // Time duration to wait for metrics (resource usage, execution time) of an execution to be computed before + // discarding it for fitness computation + ignoreExecutionWaitInterval = + Utils.getNonNegativeLong(configuration, IGNORE_EXECUTION_WAIT_INTERVAL, 2 * 60 * AutoTuner.ONE_MIN); + + // #executions after which tuning will stop even if parameters don't converge + maxTuningExecutions = + Utils.getNonNegativeInt(configuration, MAX_TUNING_EXECUTIONS, 39); + + // #executions before which tuning cannot stop even if parameters converge + minTuningExecutions = + Utils.getNonNegativeInt(configuration, MIN_TUNING_EXECUTIONS, 18); + } + + private boolean isTuningEnabled(Integer jobDefinitionId) { + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinitionId) + .order() + // There can be multiple entries in tuningJobDefinition if the job is switch on/off multiple times. + // The latest entry gives the information regarding whether tuning is enabled or not + .desc(TuningJobDefinition.TABLE.createdTs) + .setMaxRows(1) + .findUnique(); + + return tuningJobDefinition != null && tuningJobDefinition.tuningEnabled; } /** @@ -70,59 +104,69 @@ public FitnessComputeUtil() { */ public void updateFitness() { logger.info("Computing and updating fitness for completed executions"); - List completedExecutions = getCompletedExecutions(); - updateExecutionMetrics(completedExecutions); - updateMetrics(completedExecutions); + List completedJobExecutionParamSets = getCompletedJobExecutionParamSets(); + updateExecutionMetrics(completedJobExecutionParamSets); + updateMetrics(completedJobExecutionParamSets); Set jobDefinitionSet = new HashSet(); - for (TuningJobExecution tuningJobExecution : completedExecutions) { - jobDefinitionSet.add(tuningJobExecution.jobExecution.job); + for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { + JobDefinition jobDefinition = completedJobExecutionParamSet.jobSuggestedParamSet.jobDefinition; + if (isTuningEnabled(jobDefinition.id)) { + jobDefinitionSet.add(jobDefinition); + } } checkToDisableTuning(jobDefinitionSet); } /** * Checks if the tuning parameters converge - * @param jobExecutions List of previous executions on which parameter convergence is to be checked + * @param tuningJobExecutionParamSets List of previous executions and corresponding param sets * @return true if the parameters converge, else false */ - private boolean doesParameterSetConverge(List jobExecutions) { + private boolean didParameterSetConverge(List tuningJobExecutionParamSets) { boolean result = false; - int num_param_set_for_convergence = 3; + int numParamSetForConvergence = 3; - TuningJobExecution tuningJobExecution = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.jobExecution + '.' + JobExecution.TABLE.id, jobExecutions.get(0).id) - .findUnique(); - TuningAlgorithm.JobType jobType = tuningJobExecution.tuningAlgorithm.jobType; + if (tuningJobExecutionParamSets.size() < numParamSetForConvergence) { + return false; + } + + TuningAlgorithm.JobType jobType = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.tuningAlgorithm.jobType; if (jobType == TuningAlgorithm.JobType.PIG) { + Map> paramValueSet = new HashMap>(); - for (JobExecution jobExecution : jobExecutions) { - List jobSuggestedParamValueList = new ArrayList(); - try { - jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobExecution + '.' + JobExecution.TABLE.id, jobExecution.id) - .or(Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.id, 2), - Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.id, 5)) - .findList(); - } catch (NullPointerException e) { - logger.info("Checking param convergence: Map memory and reduce memory parameter not found"); - } - if (jobSuggestedParamValueList.size() > 0) { - num_param_set_for_convergence -= 1; + + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + + List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, + jobSuggestedParamSet.id) + .or(Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, + "mapreduce.map.memory.mb"), + Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, + "mapreduce.reduce.memory.mb")) + .findList(); + + // if jobSuggestedParamValueList contains both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb + // ie, if the size of jobSuggestedParamValueList is 2 + if (jobSuggestedParamValueList != null && jobSuggestedParamValueList.size() == 2) { + numParamSetForConvergence -= 1; for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - Set tmp; + Set tmp; if (paramValueSet.containsKey(jobSuggestedParamValue.id)) { tmp = paramValueSet.get(jobSuggestedParamValue.id); } else { - tmp = new HashSet(); + tmp = new HashSet(); } tmp.add(jobSuggestedParamValue.paramValue); paramValueSet.put(jobSuggestedParamValue.id, tmp); } } - if (num_param_set_for_convergence == 0) { + if (numParamSetForConvergence == 0) { break; } } @@ -136,8 +180,8 @@ private boolean doesParameterSetConverge(List jobExecutions) { } if (result) { - logger.info( - "Switching off tuning for job: " + jobExecutions.get(0).job.jobName + " Reason: parameter set converged"); + logger.info("Switching off tuning for job: " + tuningJobExecutionParamSets.get( + 0).jobSuggestedParamSet.jobDefinition.jobName + " Reason: parameter set converged"); } return result; } @@ -147,22 +191,25 @@ private boolean doesParameterSetConverge(List jobExecutions) { * Last 6 executions constitutes 2 iterations of PSO (given the swarm size is three). Negative average gains in * latest 2 algorithm iterations (after a fixed number of minimum iterations) imply that either the algorithm hasn't * converged or there isn't enough scope for tuning. In both the cases, switching tuning off is desired - * @param jobExecutions List of previous executions + * @param tuningJobExecutionParamSets List of previous executions * @return true if the median gain is negative, else false */ - private boolean isMedianGainNegative(List jobExecutions) { - int num_fitness_for_median = 6; - Double[] fitnessArray = new Double[num_fitness_for_median]; + private boolean isMedianGainNegative(List tuningJobExecutionParamSets) { + int numFitnessForMedian = 6; + Double[] fitnessArray = new Double[numFitnessForMedian]; int entries = 0; - for (JobExecution jobExecution : jobExecutions) { - TuningJobExecution tuningJobExecution = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.jobExecution + '.' + JobExecution.TABLE.id, jobExecution.id) - .findUnique(); + + if (tuningJobExecutionParamSets.size() < numFitnessForMedian) { + return false; + } + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; if (jobExecution.executionState == JobExecution.ExecutionState.SUCCEEDED - && tuningJobExecution.paramSetState == ParamSetStatus.FITNESS_COMPUTED) { - fitnessArray[entries] = tuningJobExecution.fitness; + && jobSuggestedParamSet.paramSetState == ParamSetStatus.FITNESS_COMPUTED) { + fitnessArray[entries] = jobSuggestedParamSet.fitness; entries += 1; - if (entries == num_fitness_for_median) { + if (entries == numFitnessForMedian) { break; } } @@ -175,15 +222,14 @@ private boolean isMedianGainNegative(List jobExecutions) { medianFitness = fitnessArray[fitnessArray.length / 2]; } - JobDefinition jobDefinition = jobExecutions.get(0).job; + JobDefinition jobDefinition = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.jobDefinition; TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where(). eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id).findUnique(); double baselineFitness = tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; if (medianFitness > baselineFitness) { - logger.info( - "Switching off tuning for job: " + jobExecutions.get(0).job.jobName + " Reason: unable to tune enough"); + logger.info("Switching off tuning for job: " + jobDefinition.jobName + " Reason: unable to tune enough"); return true; } else { return false; @@ -198,9 +244,8 @@ private void disableTuning(JobDefinition jobDefinition, String reason) { TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) .findUnique(); - if (tuningJobDefinition.tuningEnabled == 1) { - logger.info("Disabling tuning for job: " + tuningJobDefinition.job.jobDefId); - tuningJobDefinition.tuningEnabled = 0; + if (tuningJobDefinition.tuningEnabled) { + tuningJobDefinition.tuningEnabled = false; tuningJobDefinition.tuningDisabledReason = reason; tuningJobDefinition.save(); } @@ -209,45 +254,46 @@ private void disableTuning(JobDefinition jobDefinition, String reason) { /** * Checks and disables tuning for the given job definitions. * Tuning can be disabled if: - * - Number of tuning executions >= MAX_TUNING_EXECUTIONS - * - or number of tuning executions >= MIN_TUNING_EXECUTIONS and parameters converge - * - or number of tuning executions >= MIN_TUNING_EXECUTIONS and median gain (in cost function) in last 6 executions is negative + * - Number of tuning executions >= maxTuningExecutions + * - or number of tuning executions >= minTuningExecutions and parameters converge + * - or number of tuning executions >= minTuningExecutions and median gain (in cost function) in last 6 executions is negative * @param jobDefinitionSet Set of jobs to check if tuning can be switched off for them */ private void checkToDisableTuning(Set jobDefinitionSet) { for (JobDefinition jobDefinition : jobDefinitionSet) { - try { - List jobExecutions = JobExecution.find.where() - .eq(JobExecution.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) - .isNotNull(JobExecution.TABLE.jobExecId) - .orderBy("id desc") - .findList(); - if (jobExecutions.size() >= MIN_TUNING_EXECUTIONS) { - if (doesParameterSetConverge(jobExecutions)) { + List tuningJobExecutionParamSets = + TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + '.' + + JobSuggestedParamSet.TABLE.jobDefinition + '.' + JobDefinition.TABLE.id, jobDefinition.id) + .order() + .desc("job_execution_id") + .findList(); + + if (tuningJobExecutionParamSets.size() >= minTuningExecutions) { + if (didParameterSetConverge(tuningJobExecutionParamSets)) { logger.info("Parameters converged. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Parameters converged"); - } else if (isMedianGainNegative(jobExecutions)) { + } else if (isMedianGainNegative(tuningJobExecutionParamSets)) { logger.info("Unable to get gain while tuning. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Unable to get gain"); - } else if (jobExecutions.size() >= MAX_TUNING_EXECUTIONS) { + } else if (tuningJobExecutionParamSets.size() >= maxTuningExecutions) { logger.info("Maximum tuning executions limit reached. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Maximum executions reached"); } } - } catch (NullPointerException e) { - logger.info("No execution found for job: " + jobDefinition.jobName); - } } } /** * This method update metrics for auto tuning monitoring for fitness compute daemon - * @param completedExecutions List of completed tuning job executions + * @param completedJobExecutionParamSets List of completed tuning job executions */ - private void updateMetrics(List completedExecutions) { + private void updateMetrics(List completedJobExecutionParamSets) { int fitnessNotUpdated = 0; - for (TuningJobExecution tuningJobExecution : completedExecutions) { - if (!tuningJobExecution.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { + for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { + if (!completedJobExecutionParamSet.jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { fitnessNotUpdated++; } else { AutoTuningMetricsController.markFitnessComputedJobs(); @@ -260,57 +306,61 @@ private void updateMetrics(List completedExecutions) { * Returns the list of completed executions whose metrics are not computed * @return List of job execution */ - private List getCompletedExecutions() { + private List getCompletedJobExecutionParamSets() { logger.info("Fetching completed executions whose fitness are yet to be computed"); - List jobExecutions = new ArrayList(); - List outputJobExecutions = new ArrayList(); + List completedJobExecutionParamSet = new ArrayList(); - try { - jobExecutions = TuningJobExecution.find.select("*") + List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") .where() - .eq(TuningJobExecution.TABLE.paramSetState, ParamSetStatus.EXECUTED) + .or(Expr.or(Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.SUCCEEDED), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.FAILED)), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.CANCELLED)) + .isNull(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.resourceUsage) .findList(); - for (TuningJobExecution tuningJobExecution : jobExecutions) { - long diff = System.currentTimeMillis() - tuningJobExecution.jobExecution.updatedTs.getTime(); - logger.debug("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " - + tuningJobExecution.jobExecution.updatedTs.getTime()); - if (diff < waitInterval) { - logger.debug("Delaying fitness compute for execution: " + tuningJobExecution.jobExecution.jobExecId); + logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); + + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); + logger.info("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " + + jobExecution.updatedTs.getTime()); + if (diff < fitnessComputeWaitInterval) { + logger.info("Delaying fitness compute for execution: " + jobExecution.jobExecId); } else { - logger.debug("Adding execution " + tuningJobExecution.jobExecution.jobExecId + " for fitness computation"); - outputJobExecutions.add(tuningJobExecution); + logger.info("Adding execution " + jobExecution.jobExecId + " to fitness computation queue"); + completedJobExecutionParamSet.add(tuningJobExecutionParamSet); } } - } catch (NullPointerException e) { - logger.error("No completed execution found for which fitness is to be computed", e); - } - logger.info("Number of completed execution fetched for fitness computation: " + outputJobExecutions.size()); - logger.debug("Finished fetching completed executions for fitness computation"); - return outputJobExecutions; + logger.info( + "Number of completed execution fetched for fitness computation: " + completedJobExecutionParamSet.size()); + return completedJobExecutionParamSet; } /** * Updates the execution metrics - * @param completedExecutions List of completed executions + * @param completedJobExecutionParamSets List of completed executions */ - private void updateExecutionMetrics(List completedExecutions) { + private void updateExecutionMetrics(List completedJobExecutionParamSets) { + for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { - //To artificially increase the cost function value 3 times (as a penalty) in case of metric value violation - Integer penaltyConstant = 3; + JobExecution jobExecution = completedJobExecutionParamSet.jobExecution; + JobSuggestedParamSet jobSuggestedParamSet = completedJobExecutionParamSet.jobSuggestedParamSet; + JobDefinition job = jobExecution.job; - for (TuningJobExecution tuningJobExecution : completedExecutions) { - logger.info("Updating execution metrics and fitness for execution: " + tuningJobExecution.jobExecution.jobExecId); + logger.info("Updating execution metrics and fitness for execution: " + jobExecution.jobExecId); try { - JobExecution jobExecution = tuningJobExecution.jobExecution; - JobDefinition job = jobExecution.job; - - // job id match and tuning enabled TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") .fetch(TuningJobDefinition.TABLE.job, "*") .where() .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id) - .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) + .order() + .desc(TuningJobDefinition.TABLE.createdTs) .findUnique(); List results = AppResult.find.select("*") @@ -323,12 +373,9 @@ private void updateExecutionMetrics(List completedExecutions .findList(); if (results != null && results.size() > 0) { - Long totalExecutionTime = 0L; Double totalResourceUsed = 0D; Double totalInputBytesInBytes = 0D; - Map counterValuesMap = new HashMap(); - for (AppResult appResult : results) { totalResourceUsed += appResult.resourceUsed; totalInputBytesInBytes += getTotalInputBytes(appResult); @@ -336,13 +383,13 @@ private void updateExecutionMetrics(List completedExecutions Long totalRunTime = Utils.getTotalRuntime(results); Long totalDelay = Utils.getTotalWaittime(results); - totalExecutionTime = totalRunTime - totalDelay; + Long totalExecutionTime = totalRunTime - totalDelay; if (totalExecutionTime != 0) { jobExecution.executionTime = totalExecutionTime * 1.0 / (1000 * 60); jobExecution.resourceUsage = totalResourceUsed * 1.0 / (1024 * 3600); jobExecution.inputSizeInBytes = totalInputBytesInBytes; - + jobExecution.update(); logger.info( "Metric Values for execution " + jobExecution.jobExecId + ": Execution time = " + totalExecutionTime + ", Resource usage = " + totalResourceUsed + " and total input size = " + totalInputBytesInBytes); @@ -356,54 +403,112 @@ private void updateExecutionMetrics(List completedExecutions } //Compute fitness - Double averageResourceUsagePerGBInput = - tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredResourceUsagePerGBInput = - averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - Double averageExecutionTimePerGBInput = - tuningJobDefinition.averageExecutionTime * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredExecutionTimePerGBInput = - averageExecutionTimePerGBInput * tuningJobDefinition.allowedMaxExecutionTimePercent / 100.0; - Double resourceUsagePerGBInput = - jobExecution.resourceUsage * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - Double executionTimePerGBInput = - jobExecution.executionTime * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - - if (resourceUsagePerGBInput > maxDesiredResourceUsagePerGBInput - || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { - logger.info("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); - tuningJobExecution.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - } else { - tuningJobExecution.fitness = resourceUsagePerGBInput; - } - tuningJobExecution.paramSetState = ParamSetStatus.FITNESS_COMPUTED; - jobExecution.update(); - tuningJobExecution.update(); - } - TuningJobExecution currentBestTuningJobExecution; - try { - currentBestTuningJobExecution = - TuningJobExecution.find.where().eq("jobExecution.job.id", tuningJobExecution.jobExecution.job.id). - eq(TuningJobExecution.TABLE.isParamSetBest, 1).findUnique(); - if (currentBestTuningJobExecution.fitness > tuningJobExecution.fitness) { - currentBestTuningJobExecution.isParamSetBest = false; - tuningJobExecution.isParamSetBest = true; - currentBestTuningJobExecution.save(); - tuningJobExecution.save(); + if (!jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { + if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { + logger.info("Execution id: " + jobExecution.id + " succeeded"); + updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); + } else { + // Resetting param set to created state because this case captures the scenarios when + // either the job failed for reasons other than auto tuning or was killed/cancelled/skipped etc. + // In all the above scenarios, fitness cannot be computed for the param set correctly. + // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried + // after failure. + logger.info("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); + resetParamSetToCreated(jobSuggestedParamSet); + } + } + } else { + long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); + logger.debug("Current Time in millis: " + System.currentTimeMillis() + ", job execution last updated time " + + jobExecution.updatedTs.getTime()); + if (diff > ignoreExecutionWaitInterval) { + logger.info("Fitness of param set " + jobSuggestedParamSet.id + " corresponding to execution id: " + + jobExecution.id + " not computed for more than the maximum duration specified to compute fitness. " + + "Resetting the param set to CREATED state"); + resetParamSetToCreated(jobSuggestedParamSet); } - } catch (NullPointerException e) { - tuningJobExecution.isParamSetBest = true; - tuningJobExecution.save(); } } catch (Exception e) { - logger.error("Error updating fitness of execution: " + tuningJobExecution.jobExecution.id + "\n Stacktrace: ", - e); + logger.error("Error updating fitness of execution: " + jobExecution.id + "\n Stacktrace: ", e); } } logger.info("Execution metrics updated"); } + /** + * Resets the param set to CREATED state if its fitness is not already computed + * @param jobSuggestedParamSet Param set which is to be reset + */ + private void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) { + if (!jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { + logger.info("Resetting parameter set to created: " + jobSuggestedParamSet.id); + jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; + jobSuggestedParamSet.save(); + } + } + + /** + * Updates the job suggested param set when the corresponding execution was succeeded + * @param jobExecution JobExecution: succeeded job execution corresponding to the param set which is to be updated + * @param jobSuggestedParamSet param set which is to be updated + * @param tuningJobDefinition TuningJobDefinition of the job to which param set corresponds + */ + private void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExecution, + JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { + int penaltyConstant = 3; + Double averageResourceUsagePerGBInput = + tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + Double maxDesiredResourceUsagePerGBInput = + averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; + Double averageExecutionTimePerGBInput = + tuningJobDefinition.averageExecutionTime * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + Double maxDesiredExecutionTimePerGBInput = + averageExecutionTimePerGBInput * tuningJobDefinition.allowedMaxExecutionTimePercent / 100.0; + Double resourceUsagePerGBInput = jobExecution.resourceUsage * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; + Double executionTimePerGBInput = jobExecution.executionTime * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; + + if (resourceUsagePerGBInput > maxDesiredResourceUsagePerGBInput + || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { + logger.info("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + } else { + jobSuggestedParamSet.fitness = resourceUsagePerGBInput; + } + jobSuggestedParamSet.paramSetState = ParamSetStatus.FITNESS_COMPUTED; + jobSuggestedParamSet.fitnessJobExecution = jobExecution; + jobSuggestedParamSet = updateBestJobSuggestedParamSet(jobSuggestedParamSet); + jobSuggestedParamSet.update(); + } + + /** + * Updates the given job suggested param set to be the best param set if its fitness is less than the current best param set + * (since the objective is to minimize the fitness, the param set with the lowest fitness is the best) + * @param jobSuggestedParamSet JobSuggestedParamSet + */ + private JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { + logger.info("Checking if a new best param set is found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); + JobSuggestedParamSet currentBestJobSuggestedParamSet = JobSuggestedParamSet.find.where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, + jobSuggestedParamSet.jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 1) + .findUnique(); + if (currentBestJobSuggestedParamSet != null) { + if (currentBestJobSuggestedParamSet.fitness > jobSuggestedParamSet.fitness) { + logger.info("Param set: " + jobSuggestedParamSet.id + " is the new best param set for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); + currentBestJobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.isParamSetBest = true; + currentBestJobSuggestedParamSet.save(); + } + } else { + logger.info("No best param set found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId + + ". Marking current param set " + jobSuggestedParamSet.id + " as best"); + jobSuggestedParamSet.isParamSetBest = true; + } + return jobSuggestedParamSet; + } + /** * Returns the total input size * @param appResult appResult diff --git a/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java b/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java index 694cfb329..21b8a909b 100644 --- a/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java +++ b/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java @@ -16,21 +16,15 @@ package com.linkedin.drelephant.tuning; +import controllers.AutoTuningMetricsController; import java.net.MalformedURLException; import java.net.URISyntaxException; -import java.util.ArrayList; import java.util.List; - import models.JobExecution; import models.JobExecution.ExecutionState; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; - +import models.TuningJobExecutionParamSet; import org.apache.log4j.Logger; -import controllers.AutoTuningMetricsController; -import play.libs.Json; - /** * This class pools the scheduler for completion status of execution and updates the database with current status @@ -41,77 +35,54 @@ public abstract class JobCompleteDetector { /** * Updates the status of completed executions - * @return List of completed executions * @throws MalformedURLException MalformedURLException * @throws URISyntaxException URISyntaxException */ - public List updateCompletedExecutions() throws MalformedURLException, URISyntaxException { - logger.info("Checking execution status"); - List runningExecutions = getStartedExecutions(); - List completedExecutions = getCompletedExecutions(runningExecutions); - updateExecutionStatus(completedExecutions); + public void updateCompletedExecutions() throws MalformedURLException, URISyntaxException { + logger.info("Updating execution status"); + List inProgressExecutionParamSet = getExecutionsInProgress(); + List completedExecutions = getCompletedExecutions(inProgressExecutionParamSet); updateMetrics(completedExecutions); logger.info("Finished updating execution status"); - return completedExecutions; } /** - * This method is for updating metrics for auto tuning monitoring for job completion daemon + * Updates metrics for auto tuning monitoring for job completion daemon * @param completedExecutions List completed job executions */ - private void updateMetrics(List completedExecutions) { - for (TuningJobExecution tuningJobExecution : completedExecutions) { - if (tuningJobExecution.paramSetState.equals(ParamSetStatus.EXECUTED)) { - if (tuningJobExecution.jobExecution.executionState.equals(ExecutionState.SUCCEEDED)) { - AutoTuningMetricsController.markSuccessfulJobs(); - } else if (tuningJobExecution.jobExecution.executionState.equals(ExecutionState.FAILED)) { - AutoTuningMetricsController.markFailedJobs(); - } + private void updateMetrics(List completedExecutions) { + for (JobExecution jobExecution : completedExecutions) { + if (jobExecution.executionState.equals(ExecutionState.SUCCEEDED)) { + AutoTuningMetricsController.markSuccessfulJobs(); + } else if (jobExecution.executionState.equals(ExecutionState.FAILED)) { + AutoTuningMetricsController.markFailedJobs(); } } } /** - * Returns the list of executions which have already received param suggestion + * Returns the executions in progress * @return JobExecution list */ - private List getStartedExecutions() { - logger.info("Fetching the executions which were running"); - List tuningJobExecutionList = new ArrayList(); - try { - tuningJobExecutionList = TuningJobExecution.find.select("*") - .where() - .eq(TuningJobExecution.TABLE.paramSetState, ParamSetStatus.SENT) - .findList(); - } catch (NullPointerException e) { - logger.info("None of the executions were running ", e); - } - logger.info("Number of executions which were in running state: " + tuningJobExecutionList.size()); - return tuningJobExecutionList; + private List getExecutionsInProgress() { + logger.info("Fetching the executions which are in progress"); + List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobExecution) + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet) + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + ExecutionState.IN_PROGRESS) + .findList(); + logger.info("Number of executions which are in progress: " + tuningJobExecutionParamSets.size()); + return tuningJobExecutionParamSets; } /** * Returns the list of completed executions. - * @param jobExecutions Started Execution list + * @param inProgressExecutionParamSet List of executions (with corresponding param set) in progress * @return List of completed executions - * @throws MalformedURLException - * @throws URISyntaxException - */ - protected abstract List getCompletedExecutions(List jobExecutions) - throws MalformedURLException, URISyntaxException; - - /** - * Updates the job execution status - * @param jobExecutions JobExecution list - * @return Update status + * @throws MalformedURLException MalformedURLException + * @throws URISyntaxException URISyntaxException */ - private void updateExecutionStatus(List jobExecutions) { - logger.info("Updating status of executions completed since last iteration"); - for (TuningJobExecution tuningJobExecution : jobExecutions) { - JobExecution jobExecution = tuningJobExecution.jobExecution; - logger.info("Updating execution status to EXECUTED for the execution: " + jobExecution.jobExecId); - jobExecution.update(); - tuningJobExecution.update(); - } - } + protected abstract List getCompletedExecutions( + List inProgressExecutionParamSet) throws MalformedURLException, URISyntaxException; } diff --git a/app/com/linkedin/drelephant/tuning/JobTuningInfo.java b/app/com/linkedin/drelephant/tuning/JobTuningInfo.java index 3da56b6ab..36ce00f6a 100644 --- a/app/com/linkedin/drelephant/tuning/JobTuningInfo.java +++ b/app/com/linkedin/drelephant/tuning/JobTuningInfo.java @@ -18,9 +18,7 @@ import models.TuningParameter; import models.JobDefinition; - import java.util.List; - import models.TuningAlgorithm.JobType; /** diff --git a/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java b/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java index b553f8772..1d1651cad 100644 --- a/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java +++ b/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java @@ -18,17 +18,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.linkedin.drelephant.ElephantContext; - -import org.apache.hadoop.conf.Configuration; -import org.apache.log4j.Logger; - -import play.libs.Json; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; +import play.libs.Json; /** @@ -36,12 +33,11 @@ */ public class PSOParamGenerator extends ParamGenerator { - private final Logger logger = Logger.getLogger(PSOParamGenerator.class); private static final String PARAMS_TO_TUNE_FIELD_NAME = "parametersToTune"; private static final String PYTHON_PATH_CONF = "python.path"; private static final String PSO_DIR_PATH_ENV_VARIABLE = "PSO_DIR_PATH"; private static final String PYTHON_PATH_ENV_VARIABLE = "PYTHONPATH"; - + private final Logger logger = Logger.getLogger(PSOParamGenerator.class); private String PYTHON_PATH = null; private String TUNING_SCRIPT_PATH = null; @@ -78,9 +74,7 @@ public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { newJobTuningInfo.setJobType(jobTuningInfo.getJobType()); JsonNode jsonJobTuningInfo = Json.toJson(jobTuningInfo); - logger.info("Job Tuning Info for " + jobTuningInfo.getTuningJob().jobName + ": " + jsonJobTuningInfo); String parametersToTune = jsonJobTuningInfo.get(PARAMS_TO_TUNE_FIELD_NAME).toString(); - logger.info("Parameters to tune for job: " + parametersToTune); String stringTunerState = jobTuningInfo.getTunerState(); stringTunerState = stringTunerState.replaceAll("\\s+", ""); String jobType = jobTuningInfo.getJobType().toString(); @@ -88,14 +82,12 @@ public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { List error = new ArrayList(); try { - logger.info( - "Calling PSO with Job type = " + jobType + " StringTunerState= " + stringTunerState + "\nand Parameters to tune: " + parametersToTune); Process p = Runtime.getRuntime() - .exec(PYTHON_PATH + " " + TUNING_SCRIPT_PATH + " " + stringTunerState + " " + parametersToTune + " " + jobType); + .exec( + PYTHON_PATH + " " + TUNING_SCRIPT_PATH + " " + stringTunerState + " " + parametersToTune + " " + jobType); BufferedReader inputStream = new BufferedReader(new InputStreamReader(p.getInputStream())); BufferedReader errorStream = new BufferedReader(new InputStreamReader(p.getErrorStream())); String updatedStringTunerState = inputStream.readLine(); - logger.info("Output from PSO script: " + updatedStringTunerState); newJobTuningInfo.setTunerState(updatedStringTunerState); String errorLine; while ((errorLine = errorStream.readLine()) != null) { diff --git a/app/com/linkedin/drelephant/tuning/ParamGenerator.java b/app/com/linkedin/drelephant/tuning/ParamGenerator.java index 952a9e22b..4b3ea3508 100644 --- a/app/com/linkedin/drelephant/tuning/ParamGenerator.java +++ b/app/com/linkedin/drelephant/tuning/ParamGenerator.java @@ -17,37 +17,34 @@ package com.linkedin.drelephant.tuning; import com.avaje.ebean.Expr; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; - +import controllers.AutoTuningMetricsController; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; - -import models.*; - -import com.fasterxml.jackson.databind.JsonNode; - -import controllers.AutoTuningMetricsController; - +import models.JobDefinition; +import models.JobExecution; +import models.JobSavedState; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; - import play.libs.Json; -import java.util.ArrayList; -import java.util.List; - -import static java.lang.Math.*; - /** * This is an abstract class for generating parameter suggestions for jobs */ public abstract class ParamGenerator { - private final Logger logger = Logger.getLogger(getClass()); - private static final String JSON_CURRENT_POPULATION_KEY = "current_population"; + private final Logger logger = Logger.getLogger(getClass()); /** * Generates the parameters using tuningJobInfo and returns it in updated JobTuningInfo @@ -82,58 +79,45 @@ private List jsonToParticleList(JsonNode jsonParticleList) { * Fetches the list to job which need new parameter suggestion * @return Job list */ - private List fetchJobsForParamSuggestion() { - + private List getJobsForParamSuggestion() { // Todo: [Important] Change the logic. This is very rigid. Ideally you should look at the param set ids in the saved state, // todo: [continuation] if their fitness is computed, pso can generate new params for the job logger.info("Checking which jobs need new parameter suggestion"); List jobsForParamSuggestion = new ArrayList(); - List pendingParamExecutionList = new ArrayList(); - try { - pendingParamExecutionList = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .where() - .or(Expr.or(Expr.eq(TuningJobExecution.TABLE.paramSetState, TuningJobExecution.ParamSetStatus.CREATED), - Expr.eq(TuningJobExecution.TABLE.paramSetState, TuningJobExecution.ParamSetStatus.SENT)), - Expr.eq(TuningJobExecution.TABLE.paramSetState, TuningJobExecution.ParamSetStatus.EXECUTED)) - .eq(TuningJobExecution.TABLE.isDefaultExecution, 0) - .findList(); - } catch (NullPointerException e) { - logger.info("None of the non-default executions are in CREATED, SENT OR EXECUTED state"); - } + List pendingParamSetList = JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .or(Expr.or(Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED), + Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.SENT)), + Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.EXECUTED)) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0).findList(); List pendingParamJobList = new ArrayList(); - for (TuningJobExecution pendingParamExecution : pendingParamExecutionList) { - if (!pendingParamJobList.contains(pendingParamExecution.jobExecution.job)) { - pendingParamJobList.add(pendingParamExecution.jobExecution.job); + for (JobSuggestedParamSet pendingParamSet : pendingParamSetList) { + if (!pendingParamJobList.contains(pendingParamSet.jobDefinition)) { + pendingParamJobList.add(pendingParamSet.jobDefinition); } } - List tuningJobDefinitionList = new ArrayList(); + List tuningJobDefinitionList = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) + .findList(); - try { - tuningJobDefinitionList = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) - .findList(); - } catch (NullPointerException e) { + if (tuningJobDefinitionList.size() == 0) { logger.error("No auto-tuning enabled jobs found"); } for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitionList) { if (!pendingParamJobList.contains(tuningJobDefinition.job)) { - jobsForParamSuggestion.add(tuningJobDefinition); - } - } - if (jobsForParamSuggestion.size() > 0) { - for (TuningJobDefinition tuningJobDefinition : jobsForParamSuggestion) { logger.info("New parameter suggestion needed for job: " + tuningJobDefinition.job.jobName); + jobsForParamSuggestion.add(tuningJobDefinition); } - } else { - logger.info("None of the jobs need new parameter suggestion"); } + logger.info("Number of job(s) which need new parameter suggestion: " + jobsForParamSuggestion.size()); return jobsForParamSuggestion; } @@ -171,19 +155,19 @@ private List getJobsTuningInfo(List tuningJo .eq(TuningParameter.TABLE.isDerived, 0) .findList(); - try { logger.info("Fetching default parameter values for job " + tuningJobDefinition.job.jobDefId); - TuningJobExecution defaultJobExecution = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job + "." + JobDefinition.TABLE.id, - tuningJobDefinition.job.id) - .eq(TuningJobExecution.TABLE.isDefaultExecution, 1) - .orderBy(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.id + " desc") + JobSuggestedParamSet defaultJobParamSet = JobSuggestedParamSet.find.where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, tuningJobDefinition.job.id) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 1) + .order() + .desc(JobSuggestedParamSet.TABLE.id) .setMaxRows(1) .findUnique(); - if (defaultJobExecution != null && defaultJobExecution.jobExecution != null) { + + if (defaultJobParamSet != null) { List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobExecution + "." + JobExecution.TABLE.id, - defaultJobExecution.jobExecution.id) + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + "." + JobExecution.TABLE.id, + defaultJobParamSet.id) .findList(); if (jobSuggestedParamValueList.size() > 0) { @@ -205,9 +189,6 @@ private List getJobsTuningInfo(List tuningJo } } } - } catch (NullPointerException e) { - logger.error("Error extracting default value of params for job " + tuningJobDefinition.job.jobDefId, e); - } JobTuningInfo jobTuningInfo = new JobTuningInfo(); jobTuningInfo.setTuningJob(job); @@ -225,16 +206,12 @@ private List getJobsTuningInfo(List tuningJo Long paramSetId = particle.getParamSetId(); logger.info("Param set id: " + paramSetId.toString()); - TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.id, paramSetId) - .findUnique(); - - JobExecution jobExecution = tuningJobExecution.jobExecution; + JobSuggestedParamSet jobSuggestedParamSet = + JobSuggestedParamSet.find.select("*").where().eq(JobSuggestedParamSet.TABLE.id, paramSetId).findUnique(); - if (tuningJobExecution.fitness != null) { - particle.setFitness(tuningJobExecution.fitness); + if (jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) + && jobSuggestedParamSet.fitness != null) { + particle.setFitness(jobSuggestedParamSet.fitness); } else { validSavedState = false; logger.error("Invalid saved state: Fitness of previous execution not computed."); @@ -283,10 +260,8 @@ private List getParamValueList(Particle particle, List getParamValueList(Particle particle, List jobTuningInfoList) { JobDefinition job = jobTuningInfo.getTuningJob(); List paramList = jobTuningInfo.getParametersToTune(); String stringTunerState = jobTuningInfo.getTunerState(); - if (stringTunerState == null) { logger.error("Suggested parameter suggestion is empty for job id: " + job.jobDefId); continue; @@ -341,15 +315,12 @@ private void updateDatabase(List jobTuningInfoList) { .findUnique(); List derivedParameterList = new ArrayList(); - try { derivedParameterList = TuningParameter.find.where() .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, tuningJobDefinition.tuningAlgorithm.id) .eq(TuningParameter.TABLE.isDerived, 1) .findList(); - } catch (NullPointerException e) { - logger.info("No derived parameters for job: " + job.jobName); - } + logger.info("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " + derivedParameterList.size()); @@ -402,28 +373,30 @@ private void updateDatabase(List jobTuningInfoList) { } } - TuningJobExecution tuningJobExecution = new TuningJobExecution(); - JobExecution jobExecution = new JobExecution(); - jobExecution.job = job; - tuningJobExecution.jobExecution = jobExecution; - tuningJobExecution.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; - tuningJobExecution.isDefaultExecution = false; - if (isParamConstraintViolated(jobSuggestedParamValueList, tuningJobExecution.tuningAlgorithm.jobType, job.id)) { + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; + jobSuggestedParamSet.isParamSetDefault = false; + jobSuggestedParamSet.isParamSetBest = false; + if (isParamConstraintViolated(jobSuggestedParamValueList, jobSuggestedParamSet.tuningAlgorithm.jobType)) { logger.info("Parameter constraint violated. Applying penalty."); int penaltyConstant = 3; Double averageResourceUsagePerGBInput = - tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; Double maxDesiredResourceUsagePerGBInput = - averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - tuningJobExecution.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - tuningJobExecution.paramSetState = TuningJobExecution.ParamSetStatus.FITNESS_COMPUTED; + averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; + + jobSuggestedParamSet.areConstraintsViolated = true; + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED; } else { - tuningJobExecution.paramSetState = TuningJobExecution.ParamSetStatus.CREATED; + jobSuggestedParamSet.areConstraintsViolated = false; + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; } - Long paramSetId = saveSuggestedParamMetadata(tuningJobExecution); + Long paramSetId = saveSuggestedParamSet(jobSuggestedParamSet); for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - jobSuggestedParamValue.jobExecution = jobExecution; + jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; } suggestedParticle.setPramSetId(paramSetId); saveSuggestedParams(jobSuggestedParamValueList); @@ -445,11 +418,12 @@ private void updateDatabase(List jobTuningInfoList) { * Constraint 1: sort.mb > 60% of map.memory: To avoid heap memory failure * Constraint 2: map.memory - sort.mb < 768: To avoid heap memory failure * Constraint 3: pig.maxCombinedSplitSize > 1.8*mapreduce.map.memory.mb - * @param jobSuggestedParamValueList + * @param jobSuggestedParamValueList List of suggested param values + * @param jobType Job type * @return true if the constraint is violated, false otherwise */ private boolean isParamConstraintViolated(List jobSuggestedParamValueList, - TuningAlgorithm.JobType jobType, Integer jobDefinitionId) { + TuningAlgorithm.JobType jobType) { logger.info("Checking whether parameter values are within constraints"); Integer violations = 0; @@ -513,7 +487,7 @@ private void saveTunerState(List jobTuningInfoList) { } /** - * Saved the list of suggested parameter values to database + * Saves the list of suggested parameter values to database * @param jobSuggestedParamValueList Suggested Parameter Values List */ private void saveSuggestedParams(List jobSuggestedParamValueList) { @@ -523,21 +497,20 @@ private void saveSuggestedParams(List jobSuggestedParamV } /** - * Save the job execution in the database and returns the param set id - * @param tuningJobExecution JobExecution + * Saves the suggested param set in the database and returns the param set id + * @param jobSuggestedParamSet JobExecution * @return Param Set Id */ - - private Long saveSuggestedParamMetadata(TuningJobExecution tuningJobExecution) { - tuningJobExecution.save(); - return tuningJobExecution.jobExecution.id; + private Long saveSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { + jobSuggestedParamSet.save(); + return jobSuggestedParamSet.id; } /** * Fetches job which need parameters, generates parameters and stores it in the database */ public void getParams() { - List jobsForSwarmSuggestion = fetchJobsForParamSuggestion(); + List jobsForSwarmSuggestion = getJobsForParamSuggestion(); List jobTuningInfoList = getJobsTuningInfo(jobsForSwarmSuggestion); List updatedJobTuningInfoList = new ArrayList(); for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { diff --git a/app/com/linkedin/drelephant/tuning/Particle.java b/app/com/linkedin/drelephant/tuning/Particle.java index 978829285..961d839f5 100644 --- a/app/com/linkedin/drelephant/tuning/Particle.java +++ b/app/com/linkedin/drelephant/tuning/Particle.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.List; diff --git a/app/com/linkedin/drelephant/tuning/TuningInput.java b/app/com/linkedin/drelephant/tuning/TuningInput.java index b37efae92..7ccf80a23 100644 --- a/app/com/linkedin/drelephant/tuning/TuningInput.java +++ b/app/com/linkedin/drelephant/tuning/TuningInput.java @@ -16,6 +16,12 @@ package com.linkedin.drelephant.tuning; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import models.TuningAlgorithm; @@ -46,6 +52,15 @@ public class TuningInput { private Double _allowedMaxResourceUsagePercent; private Double _allowedMaxExecutionTimePercent; private TuningAlgorithm _tuningAlgorithm; + private Integer version; + + public Integer getVersion() { + return version; + } + + public void setVersion(Integer version) { + this.version = version; + } public TuningAlgorithm getTuningAlgorithm() { return _tuningAlgorithm; @@ -274,9 +289,36 @@ public void setScheduler(String scheduler) { /** * Returns the default parameters * @return default parameters - */ - public String getDefaultParams() { - return _defaultParams; + * @throws IOException + * @throws JsonMappingException + * @throws JsonParseException + */ + @SuppressWarnings("unchecked") + public Map getDefaultParams() throws JsonParseException, JsonMappingException, IOException { + ObjectMapper mapper = new ObjectMapper(); + Map paramValueMap; + if (version == 1) { + paramValueMap = (Map) mapper.readValue(this._defaultParams, Map.class); + } else { + paramValueMap = new HashMap(); + Map paramsStringMap = (Map) mapper.readValue(this._defaultParams, Map.class); + for (Map.Entry entry : paramsStringMap.entrySet()) { + String confKey = entry.getKey(); + String confVal = entry.getValue(); + Double confValDouble = null; + try { + if (confVal != null) { + confValDouble = Double.parseDouble(confVal); + } + } catch (NumberFormatException nfe) { + //Do Nothing + } + if (confValDouble != null) { + paramValueMap.put(confKey, confValDouble); + } + } + } + return paramValueMap; } /** diff --git a/app/controllers/Application.java b/app/controllers/Application.java index 1f3bff60f..1c1a60900 100644 --- a/app/controllers/Application.java +++ b/app/controllers/Application.java @@ -71,6 +71,7 @@ import views.html.page.searchPage; import views.html.results.*; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -914,6 +915,10 @@ public static Result getCurrentRunParameters() { String client = paramValueMap.get("client"); String scheduler = paramValueMap.get("scheduler"); String defaultParams = paramValueMap.get("defaultParams"); + Integer version = 1; + if (paramValueMap.containsKey("version")) { + version = Integer.parseInt(paramValueMap.get("version")); + } Boolean isRetry = false; if (paramValueMap.containsKey("isRetry")) { isRetry = Boolean.parseBoolean(paramValueMap.get("isRetry")); @@ -948,6 +953,7 @@ public static Result getCurrentRunParameters() { tuningInput.setClient(client); tuningInput.setScheduler(scheduler); tuningInput.setDefaultParams(defaultParams); + tuningInput.setVersion(version); tuningInput.setRetry(isRetry); tuningInput.setSkipExecutionForOptimization(skipExecutionForOptimization); tuningInput.setJobType(jobType); @@ -960,21 +966,44 @@ public static Result getCurrentRunParameters() { } catch (Exception e) { AutoTuningMetricsController.markGetCurrentRunParametersFailures(); logger.error("Exception parsing input: ", e); - return notFound("Error parsing input "); - }finally{ - if(context!=null) - { + return notFound("Error parsing input " + e.getMessage()); + } finally { + if (context != null) { context.stop(); } } } - private static Result getCurrentRunParameters(TuningInput tuningInput) { + private static JsonNode formatGetCurrentRunParametersOutput(Map outputParams, Integer version) { + if (version == 1) { + return Json.toJson(outputParams); + } else { + Map outputParamFormatted = new HashMap(); + + //Temporarily removing input split parameters + outputParams.remove("pig.maxCombinedSplitSize"); + outputParams.remove("mapreduce.input.fileinputformat.split.maxsize"); + + for (Map.Entry param : outputParams.entrySet()) { + if (param.getKey().equals("mapreduce.map.sort.spill.percent")) { + outputParamFormatted.put(param.getKey(), String.valueOf(param.getValue())); + } else if (param.getKey().equals("mapreduce.map.java.opts") + || param.getKey().equals("mapreduce.reduce.java.opts")) { + outputParamFormatted.put(param.getKey(), "-Xmx" + Math.round(param.getValue()) + "m"); + } else { + outputParamFormatted.put(param.getKey(), String.valueOf(Math.round(param.getValue()))); + } + } + return Json.toJson(outputParamFormatted); + } + } + + private static Result getCurrentRunParameters(TuningInput tuningInput) throws Exception { AutoTuningAPIHelper autoTuningAPIHelper = new AutoTuningAPIHelper(); Map outputParams = autoTuningAPIHelper.getCurrentRunParameters(tuningInput); if (outputParams != null) { logger.info("Output params " + outputParams); - return ok(Json.toJson(outputParams)); + return ok(formatGetCurrentRunParametersOutput(outputParams, tuningInput.getVersion())); } else { AutoTuningMetricsController.markGetCurrentRunParametersFailures(); return notFound("Unable to find parameters. Job id: " + tuningInput.getJobDefId() + " Flow id: " diff --git a/app/models/FlowDefinition.java b/app/models/FlowDefinition.java index e6023ad00..d01a347b1 100644 --- a/app/models/FlowDefinition.java +++ b/app/models/FlowDefinition.java @@ -16,6 +16,8 @@ package models; +import com.avaje.ebean.annotation.UpdatedTimestamp; +import java.sql.Timestamp; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; @@ -37,6 +39,8 @@ public static class TABLE { public static final String id = "id"; public static final String flowDefId = "flowDefId"; public static final String flowDefUrl = "flowDefUrl"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } @Id @@ -49,8 +53,28 @@ public static class TABLE { @Column(nullable = false) public String flowDefUrl; + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + public static Model.Finder find = new Model.Finder(Integer.class, FlowDefinition.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } + } diff --git a/app/models/FlowExecution.java b/app/models/FlowExecution.java index 3fb66ef72..0bcee835f 100644 --- a/app/models/FlowExecution.java +++ b/app/models/FlowExecution.java @@ -16,6 +16,8 @@ package models; +import com.avaje.ebean.annotation.UpdatedTimestamp; +import java.sql.Timestamp; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -42,6 +44,8 @@ public static class TABLE { public static final String flowExecId = "flowExecId"; public static final String flowExecUrl = "flowExecUrl"; public static final String flowDefinition = "flowDefinition"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } @Id @@ -54,10 +58,30 @@ public static class TABLE { @Column(nullable = false) public String flowExecUrl; + @Column(nullable = false) @ManyToOne(cascade = CascadeType.ALL) @JoinTable(name = "flow_definition", joinColumns = {@JoinColumn(name = "flow_definition_id", referencedColumnName = "id")}) public FlowDefinition flowDefinition; + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + public static Model.Finder find = new Model.Finder(Integer.class, FlowExecution.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/JobDefinition.java b/app/models/JobDefinition.java index 2b4d5732e..387b7ff12 100644 --- a/app/models/JobDefinition.java +++ b/app/models/JobDefinition.java @@ -74,17 +74,30 @@ public static class TABLE { @Column(length = JOB_NAME_LIMIT, nullable = false) public String jobDefUrl; + @Column(nullable = false) @ManyToOne(cascade = CascadeType.ALL) @JoinTable(name = "flow_definition", joinColumns = {@JoinColumn(name = "flow_definition_id", referencedColumnName = "id")}) public FlowDefinition flowDefinition; - @Column(nullable = true) + @Column(nullable = false) public Timestamp createdTs; - @Column(nullable = true) + @Column(nullable = false) @UpdatedTimestamp public Timestamp updatedTs; + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } + public static Finder find = new Finder(Integer.class, JobDefinition.class); } diff --git a/app/models/JobExecution.java b/app/models/JobExecution.java index 8c0c6a2d6..2f67f0c88 100644 --- a/app/models/JobExecution.java +++ b/app/models/JobExecution.java @@ -56,36 +56,29 @@ public static class TABLE { public static final String inputSizeInBytes = "inputSizeInBytes"; public static final String jobExecUrl = "jobExecUrl"; public static final String jobDefinition = "jobDefinition"; - public static final String createdTs = "createdTs"; - public static final String updatedTs = "updatedTs"; public static final String flowExecution = "flowExecution"; public static final String job = "job"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } @Id @GeneratedValue(strategy = GenerationType.IDENTITY) public Long id; - @Column(nullable = true) public String jobExecId; - @Column(nullable = true) + public String jobExecUrl; + @Enumerated(EnumType.STRING) public ExecutionState executionState; - @Column(nullable = true) public Double resourceUsage; - @Column(nullable = true) public Double executionTime; - @Column(nullable = true) public Double inputSizeInBytes; - @Column(nullable = true) - public String jobExecUrl; - - @Column(nullable = true) @ManyToOne(cascade = CascadeType.ALL) @JoinTable(name = "flow_execution", joinColumns = {@JoinColumn(name = "flow_execution_id", referencedColumnName = "id")}) public FlowExecution flowExecution; @@ -95,12 +88,24 @@ public static class TABLE { @JoinTable(name = "job_definition", joinColumns = {@JoinColumn(name = "job_definition_id", referencedColumnName = "id")}) public JobDefinition job; - @Column(nullable = true) + @Column(nullable = false) public Timestamp createdTs; - @Column(nullable = true) + @Column(nullable = false) @UpdatedTimestamp public Timestamp updatedTs; + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } + public static Finder find = new Finder(Long.class, JobExecution.class); } diff --git a/app/models/JobSavedState.java b/app/models/JobSavedState.java index 02f8688de..f7ca624ae 100644 --- a/app/models/JobSavedState.java +++ b/app/models/JobSavedState.java @@ -49,8 +49,10 @@ public static class TABLE { @Column(nullable = false) public byte[] savedState; + @Column(nullable = false) public Timestamp createdTs; + @Column(nullable = false) @UpdatedTimestamp public Timestamp updatedTs; @@ -60,4 +62,16 @@ public boolean isValid() { public static Finder find = new Finder(Integer.class, JobSavedState.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/JobSuggestedParamSet.java b/app/models/JobSuggestedParamSet.java new file mode 100644 index 000000000..bad279932 --- /dev/null +++ b/app/models/JobSuggestedParamSet.java @@ -0,0 +1,118 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package models; + +import com.avaje.ebean.annotation.UpdatedTimestamp; +import java.sql.Timestamp; +import javax.persistence.CascadeType; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.ManyToOne; +import javax.persistence.OneToOne; +import javax.persistence.Table; + +import play.db.ebean.Model; + + +@Entity +@Table(name = "job_suggested_param_set") +public class JobSuggestedParamSet extends Model { + + private static final long serialVersionUID = -294471313051608818L; + + public enum ParamSetStatus { + CREATED, SENT, EXECUTED, FITNESS_COMPUTED, DISCARDED + } + + public static class TABLE { + public static final String TABLE_NAME = "job_suggested_param_set"; + public static final String id = "id"; + public static final String jobDefinition = "jobDefinition"; + public static final String tuningAlgorithm = "tuningAlgorithm"; + public static final String paramSetState = "paramSetState"; + public static final String isParamSetDefault = "isParamSetDefault"; + public static final String fitness = "fitness"; + public static final String fitnessJobExecution = "fitnessJobExecution"; + public static final String isParamSetBest = "isParamSetBest"; + public static final String areConstraintsViolated = "areConstraintsViolated"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; + + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + public Long id; + + @Column(nullable = false) + @OneToOne(cascade = CascadeType.ALL) + @JoinTable(name = "job_definition", joinColumns = {@JoinColumn(name = "job_definition_id", referencedColumnName = "id")}) + public JobDefinition jobDefinition; + + @OneToOne(cascade = CascadeType.ALL) + @JoinTable(name = "job_execution", joinColumns = {@JoinColumn(name = "fitness_job_execution_id", referencedColumnName = "id")}) + public JobExecution fitnessJobExecution; + + @Column(nullable = false) + @ManyToOne(cascade = CascadeType.ALL) + @JoinTable(name = "tuning_algorithm", joinColumns = {@JoinColumn(name = "tuning_algorithm_id", referencedColumnName = "id")}) + public TuningAlgorithm tuningAlgorithm; + + + @Enumerated(EnumType.STRING) + public ParamSetStatus paramSetState; + + @Column(nullable = false) + public Boolean isParamSetDefault; + + public Double fitness; + + @Column(nullable = false) + public Boolean isParamSetBest; + + @Column(nullable = false) + public Boolean areConstraintsViolated; + + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + + public static Model.Finder find = + new Model.Finder(Long.class, JobSuggestedParamSet.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } +} diff --git a/app/models/JobSuggestedParamValue.java b/app/models/JobSuggestedParamValue.java index 84dcef981..d78ba21ba 100644 --- a/app/models/JobSuggestedParamValue.java +++ b/app/models/JobSuggestedParamValue.java @@ -20,6 +20,7 @@ import java.sql.Timestamp; import javax.persistence.CascadeType; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; @@ -41,7 +42,7 @@ public class JobSuggestedParamValue extends Model { public static class TABLE { public static final String TABLE_NAME = "job_suggested_param_value"; public static final String id = "id"; - public static final String jobExecution = "jobExecution"; + public static final String jobSuggestedParamSet = "jobSuggestedParamSet"; public static final String tuningParameter = "tuningParameter"; public static final String paramValue = "paramValue"; public static final String createdTs = "createdTs"; @@ -52,19 +53,34 @@ public static class TABLE { @GeneratedValue(strategy = GenerationType.IDENTITY) public Integer id; public Double paramValue; - public Timestamp createdTs; - - @UpdatedTimestamp - public Timestamp updatedTs; @ManyToOne(cascade = CascadeType.ALL) - @JoinTable(name = "job_execution", joinColumns = {@JoinColumn(name = "job_execution_id", referencedColumnName = "id")}) - public JobExecution jobExecution; + @JoinTable(name = "job_suggested_param_set", joinColumns = {@JoinColumn(name = "job_suggested_param_set_id", referencedColumnName = "id")}) + public JobSuggestedParamSet jobSuggestedParamSet; @ManyToOne(cascade = CascadeType.ALL) @JoinTable(name = "tuning_parameter", joinColumns = {@JoinColumn(name = "tuning_parameter_id", referencedColumnName = "id")}) public TuningParameter tuningParameter; + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + public static Finder find = new Finder(Long.class, JobSuggestedParamValue.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/TuningAlgorithm.java b/app/models/TuningAlgorithm.java index 4f2fdfb06..69b94c9b7 100644 --- a/app/models/TuningAlgorithm.java +++ b/app/models/TuningAlgorithm.java @@ -18,6 +18,7 @@ import java.sql.Timestamp; +import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.EnumType; import javax.persistence.Enumerated; @@ -75,11 +76,25 @@ public static class TABLE { @Enumerated(EnumType.STRING) public OptimizationMetric optimizationMetric; + @Column(nullable = false) public Timestamp createdTs; + @Column(nullable = false) @UpdatedTimestamp public Timestamp updatedTs; public static Finder find = new Finder(Integer.class, TuningAlgorithm.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/TuningJobDefinition.java b/app/models/TuningJobDefinition.java index 1c0891f75..27c7c61d3 100644 --- a/app/models/TuningJobDefinition.java +++ b/app/models/TuningJobDefinition.java @@ -50,9 +50,9 @@ public static class TABLE { public static final String allowedMaxResourceUsagePercent = "allowedMaxResourceUsagePercent"; public static final String allowedMaxExecutionTimePercent = "allowedMaxExecutionTimePercent"; public static final String job = "job"; + public static final String tuningDisabledReason = "tuningDisabledReason"; public static final String createdTs = "createdTs"; public static final String updatedTs = "updatedTs"; - public static final String tuningDisabledReason = "tuningDisabledReason"; } @ManyToOne(cascade = CascadeType.ALL) @@ -67,7 +67,7 @@ public static class TABLE { public TuningAlgorithm tuningAlgorithm; @Column(nullable = false) - public int tuningEnabled; + public boolean tuningEnabled; @Column(nullable = true) public Double averageResourceUsage; @@ -105,4 +105,16 @@ public Double getAverageInputSizeInGB() { @Column(nullable = true) public String tuningDisabledReason; + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/TuningJobExecution.java b/app/models/TuningJobExecutionParamSet.java similarity index 50% rename from app/models/TuningJobExecution.java rename to app/models/TuningJobExecutionParamSet.java index 6f57acaa7..1be048e17 100644 --- a/app/models/TuningJobExecution.java +++ b/app/models/TuningJobExecutionParamSet.java @@ -16,58 +16,60 @@ package models; +import com.avaje.ebean.annotation.UpdatedTimestamp; +import java.sql.Timestamp; import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; import javax.persistence.JoinColumn; import javax.persistence.JoinTable; -import javax.persistence.ManyToOne; import javax.persistence.OneToOne; import javax.persistence.Table; - import play.db.ebean.Model; - @Entity -@Table(name = "tuning_job_execution") -public class TuningJobExecution extends Model { - - private static final long serialVersionUID = -294471313051608818L; - - public enum ParamSetStatus { - CREATED, SENT, EXECUTED, FITNESS_COMPUTED, DISCARDED - } +@Table(name = "tuning_job_execution_param_set") +public class TuningJobExecutionParamSet extends Model { + private static final long serialVersionUID = 1L; public static class TABLE { - public static final String TABLE_NAME = "tuning_job_execution"; - public static final String paramSetState = "paramSetState"; - public static final String isDefaultExecution = "isDefaultExecution"; - public static final String fitness = "fitness"; - public static final String isParamSetBest = "isParamSetBest"; + public static final String TABLE_NAME = "tuning_job_execution_param_set"; + public static final String jobSuggestedParamSet = "jobSuggestedParamSet"; public static final String jobExecution = "jobExecution"; - public static final String tuningAlgorithm = "tuningAlgorithm"; + public static final String tuningEnabled = "tuningEnabled"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } + @OneToOne(cascade = CascadeType.ALL) + @JoinTable(name = "job_suggested_param_set", joinColumns = {@JoinColumn(name = "job_suggested_param_set_id", referencedColumnName = "id")}) + public JobSuggestedParamSet jobSuggestedParamSet; + @OneToOne(cascade = CascadeType.ALL) @JoinTable(name = "job_execution", joinColumns = {@JoinColumn(name = "job_execution_id", referencedColumnName = "id")}) public JobExecution jobExecution; - @ManyToOne(cascade = CascadeType.ALL) - @JoinTable(name = "tuning_algorithm", joinColumns = {@JoinColumn(name = "tuning_algorithm_id", referencedColumnName = "id")}) - public TuningAlgorithm tuningAlgorithm; + public Boolean tuningEnabled; - @Enumerated(EnumType.STRING) @Column(nullable = false) - public ParamSetStatus paramSetState; + public Timestamp createdTs; - public Boolean isDefaultExecution; + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; - public Double fitness; + public static Finder find = + new Finder(Long.class, TuningJobExecutionParamSet.class); - public Boolean isParamSetBest; + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } - public static Model.Finder find = - new Model.Finder(Long.class, TuningJobExecution.class); + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/app/models/TuningParameter.java b/app/models/TuningParameter.java index d19e70b3b..7ecce1acc 100644 --- a/app/models/TuningParameter.java +++ b/app/models/TuningParameter.java @@ -52,10 +52,10 @@ public static class TABLE { public static final String minValue = "minValue"; public static final String maxValue = "maxValue"; public static final String stepSize = "stepSize"; - public static final String createdTs = "createdTs"; - public static final String updatedTs = "updatedTs"; public static final String tuningAlgorithm = "tuningAlgorithm"; public static final String isDerived = "isDerived"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; } @Id @@ -93,4 +93,16 @@ public static class TABLE { public static Finder find = new Finder(Integer.class, TuningParameter.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } } diff --git a/conf/evolutions/default/2.sql b/conf/evolutions/default/2.sql index 9f9e39e51..4f3d899b7 100644 --- a/conf/evolutions/default/2.sql +++ b/conf/evolutions/default/2.sql @@ -1,4 +1,20 @@ -# --- Indexing on queue for seach by queue feature +# +# Copyright 2016 LinkedIn Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# --- Indexing on queue for search by queue feature # --- !Ups create index yarn_app_result_i8 on yarn_app_result (queue_name); diff --git a/conf/evolutions/default/3.sql b/conf/evolutions/default/3.sql index a8cd34093..e01a3a8c8 100644 --- a/conf/evolutions/default/3.sql +++ b/conf/evolutions/default/3.sql @@ -1,3 +1,19 @@ +# +# Copyright 2016 LinkedIn Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + # --- Indexing on queue for seach by queue feature # --- !Ups diff --git a/conf/evolutions/default/4.sql b/conf/evolutions/default/4.sql index c3d9b844b..c9852e7e8 100644 --- a/conf/evolutions/default/4.sql +++ b/conf/evolutions/default/4.sql @@ -1,3 +1,19 @@ +# +# Copyright 2016 LinkedIn Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + # --- Indexing on severity,finish_time for count on welcome page # --- !Ups diff --git a/conf/evolutions/default/5.sql b/conf/evolutions/default/5.sql index 174d42ee0..640c292be 100644 --- a/conf/evolutions/default/5.sql +++ b/conf/evolutions/default/5.sql @@ -28,10 +28,12 @@ CREATE TABLE IF NOT EXISTS tuning_algorithm ( optimization_metric enum('RESOURCE','EXECUTION_TIME') DEFAULT NULL COMMENT 'metric to be optimized', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , - PRIMARY KEY (id) + PRIMARY KEY (id), + UNIQUE KEY tuning_algorithm_u1 (optimization_algo, optimization_algo_version) ) ENGINE=InnoDB; -INSERT INTO tuning_algorithm VALUES (1, 'PIG', 'PSO', '1', 'RESOURCE', current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_algorithm(id, job_type, optimization_algo, optimization_algo_version, optimization_metric, created_ts, updated_ts) +VALUES (1, 'PIG', 'PSO', '1', 'RESOURCE', current_timestamp(0), current_timestamp(0)); /** * This table represents hadoop parameters to be optimized for each algo in tuning_algorithm. @@ -52,15 +54,16 @@ CREATE TABLE IF NOT EXISTS tuning_parameter ( CONSTRAINT tuning_parameter_ibfk_1 FOREIGN KEY (tuning_algorithm_id) REFERENCES tuning_algorithm (id) ) ENGINE=InnoDB; -INSERT INTO tuning_parameter VALUES (1,'mapreduce.task.io.sort.mb',1,100,50,1920,50, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (2,'mapreduce.map.memory.mb',1,2048,1536,8192,128, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (3,'mapreduce.task.io.sort.factor',1,10,10,150,10 ,0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (4,'mapreduce.map.sort.spill.percent',1,0.8,0.6,0.9,0.1, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (5,'mapreduce.reduce.memory.mb',1,2048,1536,8192,128, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (6,'pig.maxCombinedSplitSize',1,536870912,536870912,536870912,128, 0, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (7,'mapreduce.reduce.java.opts',1,1536,1152,6144,128, 1, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (8,'mapreduce.map.java.opts',1,1536,1152,6144,128, 1, current_timestamp(0), current_timestamp(0)); -INSERT INTO tuning_parameter VALUES (9,'mapreduce.input.fileinputformat.split.maxsize',1,536870912,536870912,536870912,128, 1, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter (id, param_name, tuning_algorithm_id, default_value, min_value, max_value, step_size, is_derived, created_ts, updated_ts) VALUES +(1,'mapreduce.task.io.sort.mb',1,100,50,1920,50, 0, current_timestamp(0), current_timestamp(0)), +(2,'mapreduce.map.memory.mb',1,2048,1536,8192,128, 0, current_timestamp(0), current_timestamp(0)), +(3,'mapreduce.task.io.sort.factor',1,10,10,150,10 ,0, current_timestamp(0), current_timestamp(0)), +(4,'mapreduce.map.sort.spill.percent',1,0.8,0.6,0.9,0.1, 0, current_timestamp(0), current_timestamp(0)), +(5,'mapreduce.reduce.memory.mb',1,2048,1536,8192,128, 0, current_timestamp(0), current_timestamp(0)), +(6,'pig.maxCombinedSplitSize',1,536870912,536870912,536870912,128, 0, current_timestamp(0), current_timestamp(0)), +(7,'mapreduce.reduce.java.opts',1,1536,1152,6144,128, 1, current_timestamp(0), current_timestamp(0)), +(8,'mapreduce.map.java.opts',1,1536,1152,6144,128, 1, current_timestamp(0), current_timestamp(0)), +(9,'mapreduce.input.fileinputformat.split.maxsize',1,536870912,536870912,536870912,128, 1, current_timestamp(0), current_timestamp(0)); create index index_tp_algo_id on tuning_parameter (tuning_algorithm_id); @@ -71,8 +74,10 @@ CREATE TABLE IF NOT EXISTS flow_definition ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', flow_def_id varchar(700) NOT NULL COMMENT 'unique flow definition id from scheduler like azkaban, oozie, appworx etc', flow_def_url varchar(700) NOT NULL COMMENT 'flow definition URL from scheduler like azkaban, oozie, appworx etc', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY flow_def_id (flow_def_id) + UNIQUE KEY flow_definition_u1 (flow_def_id) ) ENGINE=InnoDB AUTO_INCREMENT=10000; /** @@ -83,15 +88,15 @@ CREATE TABLE IF NOT EXISTS flow_definition ( CREATE TABLE IF NOT EXISTS job_definition ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', job_def_id varchar(700) NOT NULL COMMENT 'unique job definition id from scheduler like azkaban, oozie etc', + job_def_url varchar(700) NOT NULL COMMENT 'job definition URL from scheduler like azkaban, oozie, appworx etc', flow_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_definition table', job_name varchar(700) DEFAULT NULL COMMENT 'name of the job', - job_def_url varchar(700) NOT NULL COMMENT 'job definition URL from scheduler like azkaban, oozie, appworx etc', scheduler varchar(100) NOT NULL COMMENT 'name of the scheduler like azkaban. oozie ', username varchar(100) NOT NULL COMMENT 'name of the user', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY job_def_id (job_def_id) , + UNIQUE KEY job_definition_u1 (job_def_id) , CONSTRAINT job_definition_ibfk_1 FOREIGN KEY (flow_definition_id) REFERENCES flow_definition (id) ) ENGINE=InnoDB AUTO_INCREMENT=100000; @@ -110,6 +115,7 @@ CREATE TABLE IF NOT EXISTS tuning_job_definition ( average_input_size_in_bytes bigint(20) DEFAULT NULL COMMENT 'Average input size in bytes when optimization started on this job', allowed_max_resource_usage_percent double DEFAULT NULL COMMENT 'Limit on resource usage, For ex 150 means it should not go beyond 150% ', allowed_max_execution_time_percent double DEFAULT NULL COMMENT 'Limit on execution time, For ex 150 means it should not go beyond 150% ', + tuning_disabled_reason text NULL COMMENT 'reason for disabling tuning, if any', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT tuning_job_definition_ibfk_1 FOREIGN KEY (job_definition_id) REFERENCES job_definition (id), @@ -127,6 +133,8 @@ CREATE TABLE IF NOT EXISTS flow_execution ( flow_exec_id varchar(700) NOT NULL COMMENT 'unique flow execution id from scheduler like azkaban, oozie etc ', flow_exec_url varchar(700) NOT NULL COMMENT 'execution url from scheduler like azkaban, oozie etc', flow_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_definition table', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), CONSTRAINT flow_execution_ibfk_1 FOREIGN KEY (flow_definition_id) REFERENCES flow_definition (id) ) ENGINE=InnoDB AUTO_INCREMENT=1000; @@ -140,11 +148,11 @@ create index index_fe_flow_definition_id on flow_execution (flow_definition_id); */ CREATE TABLE IF NOT EXISTS job_execution ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', - job_exec_id varchar(700) DEFAULT NULL COMMENT 'unique job execution id from scheduler like azkaban, oozie etc', - job_exec_url varchar(700) DEFAULT NULL COMMENT 'job execution url from scheduler like azkaban, oozie etc', + job_exec_id varchar(700) NOT NULL COMMENT 'unique job execution id from scheduler like azkaban, oozie etc', + job_exec_url varchar(700) NOT NULL COMMENT 'job execution url from scheduler like azkaban, oozie etc', job_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_definition table', - flow_execution_id int(10) unsigned DEFAULT NULL COMMENT 'foreign key from flow_execution table', - execution_state enum('SUCCEEDED','FAILED','NOT_STARTED','IN_PROGRESS','CANCELLED') DEFAULT NULL COMMENT 'current state of execution of the job ', + flow_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_execution table', + execution_state enum('SUCCEEDED','FAILED','NOT_STARTED','IN_PROGRESS','CANCELLED') NOT NULL COMMENT 'current state of execution of the job ', resource_usage double DEFAULT NULL COMMENT 'resource usage in GB Hours for this execution of the job', execution_time double DEFAULT NULL COMMENT 'execution time excluding delay for this execution of the job', input_size_in_bytes bigint(20) DEFAULT NULL COMMENT 'input size in bytes for this execution of the job', @@ -164,19 +172,28 @@ create index index_je_flow_execution_id on job_execution (flow_execution_id); * This table represent jobs from one execution of a flow and contains auto tuning related information. * This one execution is corresponding to one set of parameters. */ -CREATE TABLE IF NOT EXISTS tuning_job_execution ( - job_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_execution table', +CREATE TABLE IF NOT EXISTS job_suggested_param_set ( + id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', + job_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_definition table', tuning_algorithm_id int(10) unsigned NOT NULL COMMENT 'foreign key from tuning_algorithm table', - param_set_state enum('CREATED','SENT','EXECUTED','FITNESS_COMPUTED','DISCARDED') DEFAULT NULL COMMENT 'state of this execution parameter set', - is_default_execution tinyint(4) NOT NULL COMMENT 'Is this default execution', + param_set_state enum('CREATED','SENT','EXECUTED','FITNESS_COMPUTED','DISCARDED') NOT NULL COMMENT 'state of this execution parameter set', + are_constraints_violated tinyint(4) default 0 NOT NULL COMMENT 'are constraints violated for the parameter set', + is_param_set_default tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set default', + is_param_set_best tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set best', fitness double DEFAULT NULL COMMENT 'fitness of this parameter set', - UNIQUE KEY job_execution_id_2 (job_execution_id), - CONSTRAINT tuning_job_execution_ibfk_1 FOREIGN KEY (tuning_algorithm_id) REFERENCES tuning_algorithm (id), - CONSTRAINT tuning_job_execution_ibfk_2 FOREIGN KEY (job_execution_id) REFERENCES job_execution (id) -) ENGINE=InnoDB ; + fitness_job_execution_id int(10) unsigned NULL COMMENT 'foreign key from job_execution table', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + CONSTRAINT job_suggested_param_set_f1 FOREIGN KEY (tuning_algorithm_id) REFERENCES tuning_algorithm (id), + -- The following statement is commented as it leads to unit test failures though it works fine when deployed. + -- This is happening because unlike mysql, h2 database doesn't support nullable foreign keys. + -- CONSTRAINT job_suggested_param_set_f2 FOREIGN KEY (fitness_job_execution_id) REFERENCES job_execution (id), + CONSTRAINT job_suggested_param_set_f3 FOREIGN KEY (job_definition_id) REFERENCES job_definition (id) +) ENGINE=InnoDB AUTO_INCREMENT=1000; -create index index_tje_job_execution_id on tuning_job_execution (job_execution_id); -create index index_tje_tuning_algorithm_id on tuning_job_execution (tuning_algorithm_id); +create index index_tje_job_definition_id on job_suggested_param_set (job_definition_id); +create index index_tje_tuning_algorithm_id on job_suggested_param_set (tuning_algorithm_id); /** * Internal table for optimization algorithm. Stores the current state of job to be optimized/ @@ -192,31 +209,47 @@ CREATE TABLE IF NOT EXISTS job_saved_state ( /** - * Suggested parameter value corresponding to one execution of the job. + * Stores the suggested parameter value corresponding to one execution of the job. */ CREATE TABLE IF NOT EXISTS job_suggested_param_value ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', - job_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_execution table', + job_suggested_param_set_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_suggested_param_set table', tuning_parameter_id int(10) unsigned NOT NULL COMMENT 'foreign key from tuning_parameter table', - param_value double NOT NULL COMMENT 'value of the parameter suggested by algo', + param_value double NOT NULL COMMENT 'suggested value of the parameter', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY job_execution_id (job_execution_id,tuning_parameter_id), - CONSTRAINT job_suggested_param_values_f1 FOREIGN KEY (job_execution_id) REFERENCES job_execution (id), - CONSTRAINT job_suggested_param_values_f2 FOREIGN KEY (tuning_parameter_id) REFERENCES tuning_parameter (id) -) ENGINE=InnoDB AUTO_INCREMENT=1000 ; + UNIQUE KEY job_suggested_param_value_u1 (job_suggested_param_set_id, tuning_parameter_id), + CONSTRAINT job_suggested_param_value_f1 FOREIGN KEY (job_suggested_param_set_id) REFERENCES job_suggested_param_set (id), + CONSTRAINT job_suggested_param_value_f2 FOREIGN KEY (tuning_parameter_id) REFERENCES tuning_parameter (id) +) ENGINE=InnoDB AUTO_INCREMENT=1000; create index index_jspv_tuning_parameter_id on job_suggested_param_value (tuning_parameter_id); +/** + * Stores the mapping of job execution and corresponding parameter set + */ + +CREATE TABLE IF NOT EXISTS tuning_job_execution_param_set ( + job_suggested_param_set_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_suggested_param_set table', + job_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_execution table', + tuning_enabled tinyint(4) NOT NULL COMMENT 'Is tuning enabled for the execution', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE KEY tuning_job_execution_param_set_u1 (job_suggested_param_set_id, job_execution_id), + CONSTRAINT tuning_job_execution_param_set_f1 FOREIGN KEY (job_suggested_param_set_id) REFERENCES job_suggested_param_set (id), + CONSTRAINT tuning_job_execution_param_set_f2 FOREIGN KEY (job_execution_id) REFERENCES job_execution (id) +) ENGINE=InnoDB; + # --- !Downs -drop table job_suggested_param_value ; +drop table tuning_job_execution_param_set; +drop table job_suggested_param_value; drop table job_saved_state; -drop table tuning_job_execution; +drop table job_suggested_param_set; drop table tuning_job_definition; drop table job_execution; drop table flow_execution; drop table job_definition; drop table flow_definition; drop table tuning_parameter; -drop table tuning_algorithm; \ No newline at end of file +drop table tuning_algorithm; diff --git a/conf/evolutions/default/6.sql b/conf/evolutions/default/6.sql deleted file mode 100644 index f16030727..000000000 --- a/conf/evolutions/default/6.sql +++ /dev/null @@ -1,11 +0,0 @@ -# --- Support for auto tuning spark -# --- !Ups - -ALTER TABLE tuning_algorithm ADD UNIQUE KEY tuning_algorithm_uk1(optimization_algo, optimization_algo_version); -ALTER TABLE tuning_job_execution ADD COLUMN is_param_set_best tinyint(4) default 0 NOT NULL; -ALTER TABLE tuning_job_definition ADD COLUMN tuning_disabled_reason text; - -# --- !Downs -ALTER TABLE tuning_job_definition DROP COLUMN tuning_disabled_reason; -ALTER TABLE tuning_job_execution DROP COLUMN is_param_set_best; -ALTER TABLE tuning_algorithm DROP INDEX tuning_algorithm_uk1; \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java index 99aa1f0dd..255268255 100644 --- a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java +++ b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java @@ -25,10 +25,10 @@ import java.util.Map; import models.JobDefinition; import models.JobExecution; +import models.JobSuggestedParamSet; import models.JobSuggestedParamValue; import models.TuningAlgorithm; import models.TuningJobDefinition; -import models.TuningJobExecution; import models.TuningParameter; import org.junit.Before; import org.junit.Test; @@ -203,19 +203,19 @@ public void run() { PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); psoParamGenerator.getParams(); - List tuningJobExecutionList = TuningJobExecution.find.where() - .eq(TuningJobExecution.TABLE.paramSetState, TuningJobExecution.ParamSetStatus.CREATED) + List jobSuggestedParamSetList = JobSuggestedParamSet.find.where() + .eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED) .findList(); - assertEquals("Swarm size did not match", SWARM_SIZE, tuningJobExecutionList.size()); + assertEquals("Swarm size did not match", SWARM_SIZE, jobSuggestedParamSetList.size()); - TuningJobExecution tuningJobExecution = tuningJobExecutionList.get(0); + JobSuggestedParamSet jobSuggestedParamSet = jobSuggestedParamSetList.get(0); List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobExecution + '.' + JobExecution.TABLE.id, - tuningJobExecution.jobExecution.id) + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, + jobSuggestedParamSet.id) .findList(); - TuningAlgorithm tuningAlgorithm = tuningJobExecution.tuningAlgorithm; + TuningAlgorithm tuningAlgorithm = jobSuggestedParamSet.tuningAlgorithm; List tuningParameterList = TuningParameter.find.where() .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, tuningAlgorithm.id) .findList(); diff --git a/test/resources/AutoTuningConf.xml b/test/resources/AutoTuningConf.xml index b505e52ed..e2e6cacea 100644 --- a/test/resources/AutoTuningConf.xml +++ b/test/resources/AutoTuningConf.xml @@ -44,8 +44,8 @@ fitness.compute.wait_interval.ms - 180 - Wait time after the job is completed for fitness computation + 0 + Wait time after the job is completed till fitness is computer dr.elephant.api.url diff --git a/test/resources/test-init.sql b/test/resources/test-init.sql index 024cd9155..522247546 100644 --- a/test/resources/test-init.sql +++ b/test/resources/test-init.sql @@ -11,27 +11,28 @@ INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES (10003,'https: INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES (100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','mkumar1','2018-02-12 08:40:42','2018-02-12 08:40:43'); ---INSERT INTO tuning_algorithm VALUES (1,'PIG','PSO',1,'RESOURCE','2018-02-06 17:03:23','2018-02-06 17:03:23'); - --- INSERT INTO tuning_parameter VALUES (1,'mapreduce.task.io.sort.mb',1,100,50,1920,50,0,'2018-02-06 17:03:23','2018-02-06 17:03:23'),(2,'mapreduce.map.memory.mb',1,2048,1536,8192,128,0,'2018-02-06 17:03:23','2018-02-06 17:03:23'),(3,'mapreduce.task.io.sort.factor',1,10,10,150,10,0,'2018-02-06 17:03:23','2018-02-06 17:03:23'),(4,'mapreduce.map.sort.spill.percent',1,0.8,0.6,0.9,0.1,0,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(5,'mapreduce.reduce.memory.mb',1,2048,1536,8192,128,0,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(6,'pig.maxCombinedSplitSize',1,536870912,536870912,536870912,128,0,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(7,'mapreduce.reduce.java.opts',1,1536,1152,6144,128,1,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(8,'mapreduce.map.java.opts',1,1536,1152,6144,128,1,'2018-02-06 17:03:24','2018-02-06 17:03:24'),(9,'mapreduce.input.fileinputformat.split.maxsize',1,536870912,536870912,536870912,128,1,'2018-02-06 17:03:24','2018-02-06 17:03:24'); - INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) VALUES (100003,'azkaban',1,1,40.29456456163195,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL); INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES (1496,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003),(1497,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389',10003),(1498,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495',10003),(1499,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589',10003),(1500,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680',10003),(1501,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818',10003),(1502,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057',10003); -INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES (1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1496,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(1542,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0',100003,1497,'SUCCEEDED',23.334004991319443,3.6118166666666665,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(1543,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0',100003,1498,'SUCCEEDED',21.28552951388889,3.2940833333333335,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(1544,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0',100003,1499,'SUCCEEDED',21.630970052083335,3.9560833333333334,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(1545,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0',100003,1500,'SUCCEEDED',22.328486328125,3.7285166666666667,324713861757,'2018-02-14 07:29:47','2018-02-14 07:29:48'),(1546,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0',100003,1501,'SUCCEEDED',32.16945149739583,5.203783333333333,324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(1547,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0',100003,1502,'SUCCEEDED', 27.2955078125, 4.047583333333334, 324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'); - -INSERT INTO tuning_job_execution(job_execution_id, tuning_algorithm_id, param_set_state, is_default_execution, fitness, is_param_set_best) VALUES (1541,1,'FITNESS_COMPUTED',0,0.06987967161749142,0), -(1542,1,'FITNESS_COMPUTED',0,0.07715930864495756,0), -(1543,1,'FITNESS_COMPUTED',0,0.07038554856075895,0), -(1544,1,'FITNESS_COMPUTED',0,0.07152782795578526,0), -(1545,1,'FITNESS_COMPUTED',0,0.07383432757503201,0), -(1546,1,'FITNESS_COMPUTED',0,0.10637576523832741,0), -(1547,1,'FITNESS_COMPUTED',0,0.09025893809095505,0); - --- INSERT INTO job_saved_state (job_definition_id, saved_state) VALUES (100003,'{\"current_population\":[{\"birthdate\":1.518593387909695E9,\"maximize\":false,\"candidate\":[159.88928672056016,1536.0,10.0,0.770164839202443,2863.372720073011,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":1545,\"_candidate\":[159.88928672056016,1536.0,10.0,0.770164839202443,2863.372720073011,5.36870912E8]},{\"birthdate\":1.518593387909696E9,\"maximize\":false,\"candidate\":[201.64225529876035,1536.0,10.29839988592941,0.7635183100860585,2789.189282499988,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":1546,\"_candidate\":[201.64225529876035,1536.0,10.29839988592941,0.7635183100860585,2789.189282499988,5.36870912E8]},{\"birthdate\":1.518593387909698E9,\"maximize\":false,\"candidate\":[149.52419594024295,1536.0,10.0,0.7630834894363029,2844.1716734703073,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":1547,\"_candidate\":[149.52419594024295,1536.0,10.0,0.7630834894363029,2844.1716734703073,5.36870912E8]}],\"prev_population\":[{\"_candidate\":[162.7907384046847,1536.0,10.0,0.7632081784681852,2809.5806453243313,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387909403E9,\"fitness\":0.07715930864495756},{\"_candidate\":[124.05878355054111,1536.0,12.521341191290857,0.7622909149004323,2041.562366831904,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387909406E9,\"fitness\":0.07038554856075895},{\"_candidate\":[149.51252503919468,1536.0,10.0,0.7619961998308155,2844.326081249364,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387909407E9,\"fitness\":0.07152782795578526}],\"archive\":[{\"birthday\":1.518589785163133E9,\"_candidate\":[214.91248065459718,1536.0,10.0,0.7684337983774014,2810.072296417105,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387904912E9,\"fitness\":0.06798720858096566},{\"_candidate\":[124.05878355054111,1536.0,12.521341191290857,0.7622909149004323,2041.562366831904,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387909406E9,\"fitness\":0.07038554856075895},{\"birthday\":1.518589785163142E9,\"_candidate\":[149.5180195836781,1536.0,10.0,0.7624156183109899,2844.240241069899,5.36870912E8],\"maximize\":false,\"birthdate\":1.518593387904921E9,\"fitness\":0.06753575007445276}],\"rnd_state\":\"[3, [436764630, 3845696172, 1536093292, 3091223851, 1875971029, 1714861982, 2792470647, 3065135847, 2614095987, 3861985834, 3271806303, 1346964832, 3502746452, 2479553754, 2365815367, 4052784148, 98894269, 865838453, 378106826, 223993858, 4003299431, 4066675665, 203485080, 591015950, 2009306943, 209194928, 1127412241, 780814913, 2127898994, 1397125766, 2388957036, 2132624000, 2298888066, 2872527307, 2469348483, 1291602844, 3055370485, 2875413348, 1561776770, 44416253, 2399434358, 3500131466, 921216978, 4241064332, 436769369, 3842353021, 307088723, 1108438886, 3147936013, 2878320949, 306583468, 2017374956, 3295572824, 3273465645, 1897933257, 319521868, 2207782720, 1273667531, 372255185, 4062657208, 411612960, 282273738, 1946543237, 3282301954, 441294767, 4284945843, 1049714129, 2018344119, 1501407651, 3952558232, 3193503080, 3824343591, 3881423919, 77231396, 3043228678, 4105014263, 1763641998, 2079208800, 4077989659, 3535532484, 266896722, 3238452269, 3389015947, 3300573445, 400677334, 86727356, 492103313, 2873593589, 3349834695, 2913301047, 1420741021, 50847724, 1599203833, 3743114912, 2826729030, 3192901431, 2123130625, 392187268, 3073037125, 929590156, 4037814058, 1884481338, 2743891587, 4008113834, 258439194, 1605236424, 65715142, 1777541766, 2364169967, 3153165247, 3110291887, 1630800170, 1838776728, 3106288739, 430636498, 2880404702, 1852472028, 1552499468, 2539957287, 3785077327, 3823438269, 3242387062, 2983664566, 3249832040, 865349220, 1794595840, 2821840008, 3923234509, 1623391669, 2809627269, 2617933073, 3106890857, 3058957671, 2708135756, 1562894481, 2523137886, 527517498, 2114931269, 4254571978, 568227586, 93804775, 2796328419, 558871677, 2758944023, 1124005519, 1183509574, 335486627, 504724538, 1826353090, 1127161878, 1410745185, 4037978787, 3102147541, 398636727, 4216182187, 4099107742, 3961538036, 3264622209, 2654911086, 90501146, 3002202423, 190590765, 2420050097, 2767758974, 4057723606, 252185447, 2305162223, 3613701152, 1854930647, 2683774155, 1530637790, 2320258070, 1086704260, 691720499, 3286101825, 4280735926, 568632376, 1274202416, 4194385357, 2917109897, 3616771536, 1911799859, 3508013413, 4107695527, 2298948322, 2918077160, 2166469136, 1566810333, 666358996, 3019106791, 2355339603, 127848796, 309049116, 4172598364, 451598967, 3396456615, 3397855157, 4160530966, 3792287385, 2933569340, 3158078446, 1799420732, 4233446360, 1195562564, 3812502285, 3340167346, 832972998, 4236732687, 2813517707, 660207563, 1870462423, 3783309115, 2416745730, 1342644731, 3950482077, 754564373, 2068468799, 1031651649, 2486955671, 1715492367, 2327446476, 572839923, 1207676383, 2061193945, 196943407, 1925383922, 3870569689, 3894282386, 1212693631, 1404071290, 3046879375, 2924493982, 3410043685, 3809721374, 2324452872, 834464932, 2330176976, 2860749140, 2576919325, 1731581763, 2734233631, 1307721060, 1364026888, 3650978588, 1517407670, 2245310205, 1530006937, 2955120239, 3188064498, 3417491514, 2397531047, 459089077, 4053955510, 443399345, 1796085719, 1924236593, 2043248158, 186441553, 2326275902, 2472457891, 3132406322, 939610904, 2208858507, 945878056, 109589586, 548812211, 1240173202, 3204593993, 721035164, 733233892, 4129866603, 2629432835, 103841909, 4233482652, 850252801, 662253897, 1653616235, 3491341022, 475692128, 2503334188, 355990982, 2296073816, 838210242, 3287833079, 827619964, 3900262990, 1395434945, 3626514356, 3890029739, 3246993302, 2756484700, 1020069034, 3097281098, 448129499, 3328003156, 350913378, 2016280706, 2504330849, 1069039051, 2606681503, 4105161434, 3689449592, 2290999300, 890022371, 1248474502, 2805549586, 3381461720, 3786043854, 2720553646, 994750777, 915794947, 2191214736, 207734182, 2034668448, 3462612507, 2931162527, 3404334608, 3955422702, 4101042191, 4245026154, 3080687510, 241496869, 3336337400, 3961557738, 1762565244, 982272855, 201112283, 839026428, 1053533360, 3454224402, 1310868263, 2693848699, 2099992077, 3451484296, 769531652, 2865700439, 1878592864, 2721136361, 2214123932, 3828307988, 209596659, 741099773, 3199029910, 1334965681, 1544576669, 3504078266, 25459822, 3656324319, 2163345613, 3538725464, 4097275680, 3727367002, 2639924031, 3133419113, 1947965051, 1635935439, 1049126814, 3324554030, 3132507893, 3038879776, 329990888, 3582827510, 2917953579, 3027357988, 1091197630, 1490696964, 3381934548, 1603833838, 3482203517, 2183868252, 3349292431, 2412825052, 1716350840, 1475395653, 431844396, 2931362300, 1504434771, 2130908322, 4187762748, 2537958471, 4258559043, 679552214, 4150134889, 2946590212, 50221051, 3358795689, 2017939006, 850243093, 3672955852, 249808095, 2780041635, 942783646, 574550556, 3534933632, 547576575, 2354608175, 1511213271, 4913332, 1354252006, 3751258543, 137446248, 3343068228, 3233484676, 1974227414, 3362209801, 2838722690, 2222596348, 3987178007, 3099562962, 1447343198, 2266421070, 2171410780, 724137152, 3590971377, 642460272, 358356132, 2516335475, 2281385363, 1673631698, 567570379, 2820831217, 3459305266, 593075908, 685000684, 1637393981, 3084948420, 3808715397, 786812600, 1404480468, 3736739260, 4046356764, 2277487134, 273026609, 121302633, 4029458939, 3880997922, 824988722, 1623515710, 1297465894, 3789533293, 207730144, 2664490926, 331830354, 1133560012, 2069816875, 927389900, 1259060843, 941566906, 835436573, 3953926187, 3930136621, 389315519, 4183568153, 2881497915, 1324589511, 2048359325, 1043570944, 2083395465, 1478226476, 3087455126, 23006780, 606435476, 494531655, 2130588953, 3466795601, 3747817133, 3211608416, 1749127250, 1473512503, 4141802367, 4009020451, 2169062501, 486602714, 3005612659, 260005846, 1336645235, 3457721846, 4035714886, 3551248844, 543614424, 2620675740, 972732959, 2146370990, 2444228783, 3757226413, 1787341042, 1622705784, 1002005711, 2926567845, 2450964264, 275604472, 462449927, 372609898, 2731933797, 3288882335, 28113805, 2715428189, 3304559815, 2784867699, 3629574622, 3881588857, 772925804, 3315006110, 2788002026, 1552017355, 1158208914, 3011680491, 465445743, 1709915594, 120028768, 2125877175, 3730264378, 3321247512, 3213599728, 2381392366, 136022703, 55061352, 1075511919, 1650581160, 1366656740, 949909697, 1448637601, 361121780, 3717238255, 3525154122, 734851236, 76931684, 3646466304, 1361374960, 71539262, 2847860259, 103976093, 2690312143, 2203268177, 2351464896, 3376181365, 1606789530, 2476053465, 912604440, 3183693432, 3672535607, 2110979845, 636817180, 549564144, 412837280, 40658714, 1197547156, 4071713833, 3278498849, 2457986482, 2570925070, 2773406614, 2173220569, 1144522688, 3620770026, 1622735997, 2368411802, 2011454842, 1827133664, 2585489840, 3315467888, 1711780406, 288791384, 3181545850, 703884497, 3482409752, 3690605116, 1256264409, 3474559514, 2575819952, 347868803, 1414887195, 3049697508, 2286946758, 1924076795, 3990073716, 343931456, 1347596990, 1443539675, 2711173653, 1516854237, 2203898925, 3570346377, 785743864, 78265891, 1764085862, 1871947520, 3185452642, 2924684299, 2788837988, 2502034127, 2492118039, 12747637, 2645726181, 1779979579, 2417304151, 2013047543, 2938718119, 2151179730, 1964454002, 1323558464, 182394583, 25293277, 846729988, 3138476563, 1376710613, 3520885930, 1300980659, 2039674586, 3257949269, 2308477698, 2463152670, 1053115398, 3638265799, 96951187, 1351905935, 2209891741, 748742656, 1115277914, 3976004334, 3353523651, 2009742989, 115220113, 387413820, 12], null]\"}','2018-02-12 08:41:20','2018-02-12 08:41:20'); - --- INSERT INTO job_saved_state (job_definition_id, saved_state) VALUES (100003,'{}'); - -INSERT INTO job_suggested_param_value(id, job_execution_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES (3209,1541,1,149.5239493606563,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3210,1541,2,1536,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3211,1541,3,10,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3212,1541,4,0.761466551875019,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3213,1541,5,2844.365182469904,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3214,1541,6,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3215,1541,7,2133.273886852428,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3216,1541,8,1152,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3217,1541,9,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'), (3218,1542,1,162.7907384046847,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3219,1542,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3220,1542,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3221,1542,4,0.7632081784681852,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3222,1542,5,2809.5806453243313,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3223,1542,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3224,1542,7,2107.1854839932485,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3225,1542,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3226,1542,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3227,1543,1,124.05878355054111,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3228,1543,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3229,1543,3,12.521341191290857,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3230,1543,4,0.7622909149004323,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3231,1543,5,2041.562366831904,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3232,1543,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3233,1543,7,1531.171775123928,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3234,1543,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3235,1543,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3236,1544,1,149.51252503919468,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3237,1544,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3238,1544,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3239,1544,4,0.7619961998308155,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3240,1544,5,2844.326081249364,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3241,1544,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3242,1544,7,2133.244560937023,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3243,1544,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3244,1544,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3245,1545,1,159.88928672056016,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3246,1545,2,1536,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3247,1545,3,10,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3248,1545,4,0.770164839202443,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3249,1545,5,2863.372720073011,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3250,1545,6,536870912,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3251,1545,7,2147.5295400547584,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3252,1545,8,1152,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3253,1545,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3254,1546,1,201.64225529876035,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3255,1546,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3256,1546,3,10.29839988592941,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3257,1546,4,0.7635183100860585,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3258,1546,5,2789.189282499988,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3259,1546,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3260,1546,7,2091.891961874991,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3261,1546,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3262,1546,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3263,1547,1,149.52419594024295,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3264,1547,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3265,1547,3,10,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3266,1547,4,0.7630834894363029,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3267,1547,5,2844.1716734703073,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3268,1547,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3269,1547,7,2133.1287551027303,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3270,1547,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3271,1547,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'); +INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1496,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(1542,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0',100003,1497,'SUCCEEDED',23.334004991319443,3.6118166666666665,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), +(1543,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0',100003,1498,'SUCCEEDED',21.28552951388889,3.2940833333333335,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), +(1544,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0',100003,1499,'SUCCEEDED',21.630970052083335,3.9560833333333334,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), +(1545,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0',100003,1500,'SUCCEEDED',22.328486328125,3.7285166666666667,324713861757,'2018-02-14 07:29:47','2018-02-14 07:29:48'), +(1546,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0',100003,1501,'SUCCEEDED',32.16945149739583,5.203783333333333,324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'), +(1547,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0',100003,1502,'SUCCEEDED', 27.2955078125, 4.047583333333334, 324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'); + +INSERT INTO job_suggested_param_set(fitness_job_execution_id, tuning_algorithm_id, param_set_state, is_param_set_default, fitness, is_param_set_best, are_constraints_violated, job_definition_id,id) VALUES +(1541,1,'FITNESS_COMPUTED',0,0.06987967161749142,0,0,100003,1541), +(1542,1,'FITNESS_COMPUTED',0,0.07715930864495756,0,0,100003,1542), +(1543,1,'FITNESS_COMPUTED',0,0.07038554856075895,0,0,100003,1543), +(1544,1,'FITNESS_COMPUTED',0,0.07152782795578526,0,0,100003,1544), +(1545,1,'FITNESS_COMPUTED',0,0.07383432757503201,0,0,100003,1545), +(1546,1,'FITNESS_COMPUTED',0,0.10637576523832741,0,0,100003,1546), +(1547,1,'FITNESS_COMPUTED',0,0.09025893809095505,0,0,100003,1547); + +INSERT INTO job_suggested_param_value(id, job_suggested_param_set_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES +(3209,1541,1,149.5239493606563,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3210,1541,2,1536,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3211,1541,3,10,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3212,1541,4,0.761466551875019,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3213,1541,5,2844.365182469904,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3214,1541,6,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3215,1541,7,2133.273886852428,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3216,1541,8,1152,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3217,1541,9,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'), (3218,1542,1,162.7907384046847,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3219,1542,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3220,1542,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3221,1542,4,0.7632081784681852,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3222,1542,5,2809.5806453243313,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3223,1542,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3224,1542,7,2107.1854839932485,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3225,1542,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3226,1542,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3227,1543,1,124.05878355054111,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3228,1543,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3229,1543,3,12.521341191290857,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3230,1543,4,0.7622909149004323,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3231,1543,5,2041.562366831904,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3232,1543,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3233,1543,7,1531.171775123928,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3234,1543,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3235,1543,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3236,1544,1,149.51252503919468,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3237,1544,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3238,1544,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3239,1544,4,0.7619961998308155,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3240,1544,5,2844.326081249364,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3241,1544,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3242,1544,7,2133.244560937023,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3243,1544,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3244,1544,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3245,1545,1,159.88928672056016,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3246,1545,2,1536,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3247,1545,3,10,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3248,1545,4,0.770164839202443,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3249,1545,5,2863.372720073011,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3250,1545,6,536870912,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3251,1545,7,2147.5295400547584,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3252,1545,8,1152,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3253,1545,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3254,1546,1,201.64225529876035,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3255,1546,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3256,1546,3,10.29839988592941,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3257,1546,4,0.7635183100860585,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3258,1546,5,2789.189282499988,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3259,1546,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3260,1546,7,2091.891961874991,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3261,1546,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3262,1546,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3263,1547,1,149.52419594024295,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3264,1547,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3265,1547,3,10,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3266,1547,4,0.7630834894363029,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3267,1547,5,2844.1716734703073,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3268,1547,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3269,1547,7,2133.1287551027303,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3270,1547,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3271,1547,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'); diff --git a/test/resources/tunein-test1.sql b/test/resources/tunein-test1.sql index ebb33dd33..aee0f0d22 100644 --- a/test/resources/tunein-test1.sql +++ b/test/resources/tunein-test1.sql @@ -1,24 +1,44 @@ -INSERT INTO `flow_definition`(id, flow_def_id, flow_def_url) VALUES (57,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images'),(60,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall'); -INSERT INTO `job_definition`(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES (51,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action',57,'score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','azkaban','dev_svc',parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss')),(54,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry',60,'countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','azkaban','mkumar1',parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss')); -INSERT INTO `tuning_job_definition`(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) VALUES +INSERT INTO `flow_definition`(id, flow_def_id, flow_def_url) VALUES +(57,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images'), +(60,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall'); + + +INSERT INTO `job_definition`(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES +(51,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action',57,'score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','azkaban','dev_svc',parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss')), +(54,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry',60,'countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','azkaban','mkumar1',parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss')); + +INSERT INTO `tuning_job_definition`(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, +average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) VALUES (51,'azkaban',1,1,458.4372180906033,61.92735384666667,5845242598595,200,200,parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-13 17:28:34','dd-MM-yyyy hh:mm:ss'), NULL), (54,'azkaban',1,1,0.09060929361979166,1.8950733333333332,532614133,200,200,parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-22 15:49:33','dd-MM-yyyy hh:mm:ss'), NULL); + INSERT INTO `flow_execution`(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES (846,'https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/executor?execid=5157830',57),(847,'https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/executor?execid=5158194',57),(848,'https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/executor?execid=5158533',57),(2035,'https://elephant.linkedin.com:8443/executor?execid=5356366','https://elephant.linkedin.com:8443/executor?execid=5356366',60),(2036,'https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/executor?execid=5356377',60),(2047,'https://elephant.linkedin.com:8443/executor?execid=5356853','https://elephant.linkedin.com:8443/executor?execid=5356853',60); -INSERT INTO `job_execution`(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES (1624,'https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0',51,847,'SUCCEEDED',196.53583333333333,43.996566666666666,3990998122525,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:08:44','dd-MM-yyyy hh:mm:ss')),(1625,'https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0',51,846,'SUCCEEDED',169.09791666666666,34.86125,3990827471029,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:58:41','dd-MM-yyyy hh:mm:ss')),(1626,'https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0',51,848,'SUCCEEDED',166.11400634765624,33.7234,3990138654674,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(1627,NULL,NULL,51,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(1628,NULL,NULL,51,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(1629,NULL,NULL,51,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(1719,'https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2035,'FAILED',0,0,0,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')),(1720,'https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2036,'SUCCEEDED',0.07686197916666666,3.5777,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')),(1721,'https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2047,'SUCCEEDED',0.21555555555555556,3.5456666666666665,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(2833,NULL,NULL,54,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(2834,NULL,NULL,54,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(2835,NULL,NULL,54,NULL,NULL,NULL,NULL,NULL,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')); -INSERT INTO `tuning_job_execution`(job_execution_id, tuning_algorithm_id, param_set_state, is_default_execution, fitness, is_param_set_best) VALUES (1624,1,'FITNESS_COMPUTED',0,0.05287618226970775,0), -(1625,1,'FITNESS_COMPUTED',0,0.04549620518409709,0), -(1626,1,'FITNESS_COMPUTED',0,0.044701092268747876,0), -(1627,1,'CREATED',0,NULL,0), -(1628,1,'CREATED',0,NULL,0), -(1629,1,'CREATED',0,NULL,0), -(1719,1,'FITNESS_COMPUTED',0,1.0960015760903588,0), -(1720,1,'FITNESS_COMPUTED',0,0.15266389313494158,0), -(1721,1,'FITNESS_COMPUTED',0,1.079813530812908,0), -(2833,1,'FITNESS_COMPUTED',0,0.54365576171875,0), -(2834,1,'CREATED',0,NULL,0), -(2835,1,'CREATED',0,NULL,0); ---INSERT INTO `job_saved_state` VALUES (51,'{\"current_population\":[{\"birthdate\":1.515995961168332E9,\"maximize\":false,\"candidate\":[171.02605253760447,1536.0,54.84590775607139,0.7115250205281223,1536.0,5.36870912E8],\"fitness\":1.0E9,\"paramSetId\":1627,\"_candidate\":[171.02605253760447,1536.0,54.84590775607139,0.7115250205281223,1536.0,5.36870912E8]},{\"birthdate\":1.515995961168334E9,\"maximize\":false,\"candidate\":[181.9245696813968,1536.0,42.047012100221565,0.7037704502574763,1536.0,5.36870912E8],\"fitness\":1.0E9,\"paramSetId\":1628,\"_candidate\":[181.9245696813968,1536.0,42.047012100221565,0.7037704502574763,1536.0,5.36870912E8]},{\"birthdate\":1.515995961168335E9,\"maximize\":false,\"candidate\":[213.25485474773512,1536.0,31.525782197486357,0.7476923282427653,1581.3588284589678,5.36870912E8],\"fitness\":1.0E9,\"paramSetId\":1629,\"_candidate\":[213.25485474773512,1536.0,31.525782197486357,0.7476923282427653,1581.3588284589678,5.36870912E8]}],\"prev_population\":[{\"_candidate\":[218.6447449212757,1536.0,43.796210012462936,0.7053483939633272,1536.0,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961167665E9,\"fitness\":0.05287618226970775},{\"_candidate\":[181.9245696813968,1536.0,42.047012100221565,0.7037704502574763,1536.0,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961167667E9,\"fitness\":0.04549620518409709},{\"_candidate\":[140.63900164342985,1536.0,51.426234701754865,0.6951498772580411,1976.5719588408601,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961167669E9,\"fitness\":0.044701092268747876}],\"archive\":[{\"birthday\":1.515985385404277E9,\"_candidate\":[219.39716200011478,1536.0,66.26037015196239,0.6912182600949739,1536.0,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961166964E9,\"fitness\":0.0428721215116997},{\"birthday\":1.515985385403604E9,\"_candidate\":[181.9245696813968,1536.0,42.047012100221565,0.7037704502574763,1536.0,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961166971E9,\"fitness\":0.039280067116591405},{\"birthday\":1.515985385403608E9,\"_candidate\":[275.86885706468235,1536.0,31.422304005393446,0.7301792632677065,2015.8347602357646,5.36870912E8],\"maximize\":false,\"birthdate\":1.515995961166975E9,\"fitness\":0.04447846959473768}],\"rnd_state\":\"[3, [700660853, 980765611, 350269045, 550794061, 3508660058, 2280194506, 372390680, 2086446374, 239386402, 971362604, 4241231975, 858459806, 3033502845, 814217089, 177306281, 1390604632, 2849717269, 3701489124, 2643708056, 1542339197, 1555385004, 3634877832, 573164398, 566979896, 2206975893, 1503874227, 1236610411, 2573036830, 2711844444, 4209994160, 3990240984, 128106381, 663247204, 1653265324, 3765434946, 3330504426, 4276961481, 837144373, 3095304940, 984937884, 139122733, 4043403675, 1544045642, 2533499146, 3571481965, 962064161, 3801182180, 2403202969, 3721566863, 1565524117, 822198358, 3375360787, 3509804184, 3870723125, 1875023332, 3008398872, 3751590940, 3309048627, 1182977075, 3777424954, 2883041912, 2657794984, 3378766220, 3152549006, 2684336427, 2187987130, 3983079230, 547491665, 1289795226, 854873566, 1972709764, 1765551945, 3392981810, 3517935071, 2310497346, 3676457131, 1798697065, 3091394835, 1290859076, 144558004, 1796870482, 1477299519, 1427562024, 549877811, 1597433131, 807147957, 3905500832, 261506349, 2560082346, 3179854567, 3240841291, 413948645, 2250452566, 1819800580, 3883891184, 632396012, 1519436197, 1409698969, 3375891321, 1473582645, 829906832, 2390021020, 3047321716, 2571315456, 2450064940, 2015213036, 3701906702, 1611515348, 4040359636, 3318495044, 1604983146, 2492332685, 789975929, 3194197706, 4147041789, 1135227509, 3723966140, 1499070128, 1495663007, 4004414959, 4129672433, 1784336180, 1506720970, 3367678585, 1049352334, 952862338, 1367689768, 3421301844, 3992023287, 3787999149, 607781132, 899298268, 2429143458, 1782950959, 3532504857, 1132237626, 2088890029, 1760382924, 1130657110, 3363127838, 3043321367, 2945251619, 1568851313, 2735984651, 766093001, 3088031430, 3822048557, 772213702, 1219747991, 1167235346, 1148978281, 1418791530, 2671812486, 3026023846, 215814664, 2474836935, 3742432459, 3886554830, 1239586734, 2033249033, 2061707261, 3582021973, 1205282201, 2940872178, 1647198252, 864907932, 2580542635, 4231988675, 1726503113, 2353555357, 1827771300, 3600123622, 4005348928, 2539198000, 4037701258, 2589333679, 3456867157, 1352943897, 1822690465, 2408381181, 1868778566, 378948152, 3749762946, 502288181, 3936503490, 2971031075, 2374321618, 31726935, 860889312, 1515652253, 844878931, 1684891814, 1454985016, 2424639673, 2260089345, 2799910537, 3496783614, 3225515131, 3459630561, 2432878302, 1646494133, 2227658705, 1257136958, 3706525986, 1094286407, 850469210, 2995793692, 2211513328, 1755894528, 3571575386, 1708963361, 2362822514, 2224335253, 4293751999, 1805463544, 2453039741, 1886293255, 3289381616, 2288127674, 2172693547, 2982972306, 3215964883, 888146238, 2417861490, 7187980, 3713409018, 738436957, 579588742, 2709938763, 3959994685, 1948771268, 2379578759, 1434401739, 1013355849, 2528670801, 2524228596, 1098550737, 3832353534, 4218333751, 2179914854, 3741310823, 1659901498, 415821831, 520299428, 1153497622, 1635045824, 2373128365, 1511012134, 2109214492, 3941675131, 2968940495, 1077608592, 546487837, 560757984, 2830915546, 1277936098, 2984058125, 3107790490, 1928094975, 883897300, 2378898857, 2535418754, 1991596485, 307019245, 4253943697, 2102844823, 2387327534, 1295679535, 257012774, 2644984219, 496082329, 3302964332, 799662144, 4064163474, 1223259384, 3774897138, 1573702522, 2325115513, 729338819, 353910412, 716335921, 1209662376, 4070293169, 1270690977, 1695305659, 697493579, 2651477605, 730652454, 3792581705, 1327538356, 2327699294, 562929078, 3595526182, 4113762841, 1829261651, 1794412928, 418993809, 4047909193, 3312977071, 1977410559, 3787797897, 684781661, 1694762123, 659979161, 1871116812, 915617124, 1116356383, 1136153717, 4222588389, 3252249914, 4119031047, 3508584846, 2789171200, 61632834, 1952607520, 3315629861, 856596078, 2412263936, 2305926159, 776419518, 271200364, 2257276232, 603198351, 3194379378, 1151535969, 2880976824, 1701962330, 930448142, 1207562140, 1406230557, 1188380320, 3125895500, 175152937, 4249855047, 216407144, 1520784966, 4072420197, 886879248, 4007320206, 506365179, 1851689952, 3197299601, 1213562517, 954668668, 3567328526, 3626984841, 255329757, 302263481, 1043515423, 2971370561, 1282039976, 1697427886, 3639571623, 3235043793, 1889879468, 1716754125, 2664459343, 383321614, 3172541016, 938820727, 1437985042, 1864633700, 1346599334, 3493651062, 813423282, 3136060154, 1298327746, 960933564, 2589411966, 407108904, 1328419689, 630919073, 1136835355, 2886715215, 3562424719, 495242583, 558900013, 4158088180, 2904510921, 2567796600, 1598024941, 2892566110, 3703661632, 1541123992, 4115353702, 1209327198, 4005599061, 3947903442, 2786172353, 1230913432, 1724749753, 2587833659, 965981368, 344510153, 3951342280, 3750898439, 2523499260, 3731616870, 293764110, 321250565, 3970020252, 2406840029, 2748315886, 1254048724, 1753412283, 3984132930, 3264149240, 3078734508, 1107941005, 3567886053, 2404486768, 1977906686, 4221601803, 610582347, 3157608813, 3522391435, 3369971915, 1634098420, 1035231476, 2305434161, 1097389492, 4129366118, 62724611, 436096798, 2879207403, 2045009166, 4184819944, 3434611449, 1783700513, 3421889769, 721589734, 3824308930, 73869467, 3677536367, 330561138, 167572255, 4289033113, 4228147727, 4102390954, 2285859672, 3394854823, 319437854, 1302681615, 2461968483, 225425264, 2398301733, 327178348, 3374586849, 2263215758, 1148329531, 1658710282, 3122582876, 4218881881, 1375881267, 3661335421, 2866320394, 2610257014, 1001734510, 326380055, 2920059200, 2935750748, 1949485840, 455098597, 863729740, 1151764749, 2716044236, 3942654125, 3622670978, 2544737265, 2421681654, 3782970175, 2918035276, 1850235112, 662428057, 4015171728, 2273865476, 3153121123, 670489394, 3680101668, 3765364135, 3414953034, 1729149762, 1092291501, 3631975206, 1968379780, 4033704983, 1038531297, 1379024270, 350261516, 662580983, 2666693522, 109449532, 3945334796, 520141517, 68422090, 428256539, 3160977087, 4150490870, 1519097291, 399058475, 4044038884, 429407636, 2681023844, 3959975732, 17861206, 1324335466, 3020138261, 4094586935, 1756933566, 3958575872, 405493436, 3255720634, 4187397878, 3412452110, 3660056351, 3684801802, 913893747, 3654082722, 3323334105, 1819591112, 1880678757, 889448818, 3748811275, 309205782, 2312707061, 1171686701, 2674633737, 1550440591, 2436816799, 2702844283, 2695031040, 201888479, 3452874601, 970377544, 1460081872, 2744496799, 748920411, 829469534, 1426574992, 1628101393, 3245963172, 2652123456, 3154243638, 566872445, 2861698301, 3936811112, 1790296253, 3506496756, 396730205, 1076765041, 128885341, 2260558166, 30213662, 3891947035, 2110845573, 2636313809, 1421844627, 1055180454, 3897169890, 2369585737, 2948350514, 1622381331, 2499854322, 3008886194, 1370901601, 2285886995, 2659664530, 452806941, 734310705, 38974153, 1266839691, 4040765272, 451966518, 1034593261, 3996792012, 3262702261, 2095968672, 2064347438, 1597581201, 2195067931, 3277290478, 181898705, 2859461848, 194913269, 3518323108, 1911542790, 1323553652, 1552994486, 2365121217, 3305517045, 1197074508, 1904733312, 2752495057, 3837469761, 2421121539, 1729750982, 2322937327, 638292048, 130682510, 4110785961, 709631645, 435142599, 2912303377, 3032706254, 3625438081, 2521030328, 481154033, 2357799157, 1791268394, 849004735, 481956827, 448804740, 135372808, 2781507049, 2021713577, 818843241, 3803606416, 1787431803, 3045742311, 2023574429, 1185539163, 1216654837, 789835050, 1535157273, 276], null]\"}',parsedatetime('2018-01-13 18:14:06','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(54,'{\"current_population\":[{\"birthdate\":1.518544975273291E9,\"maximize\":false,\"candidate\":[1459.6899166265118,2421.8450860158327,45.39309760195506,0.7338175361303875,2690.0010009930634,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":2833,\"_candidate\":[1459.6899166265118,2421.8450860158327,45.39309760195506,0.7338175361303875,2690.0010009930634,5.36870912E8]},{\"birthdate\":1.518544975273293E9,\"maximize\":false,\"candidate\":[678.4611819565604,2233.783342819729,67.46065454683335,0.7337482196069645,3068.4288351321584,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":2834,\"_candidate\":[678.4611819565604,2233.783342819729,67.46065454683335,0.7337482196069645,3068.4288351321584,5.36870912E8]},{\"birthdate\":1.518544975273295E9,\"maximize\":false,\"candidate\":[888.1718411828987,2391.4206410271354,63.83042592878694,0.7326134736198681,2987.2156090895137,5.36870912E8],\"fitness\":10000.0,\"paramSetId\":2835,\"_candidate\":[888.1718411828987,2391.4206410271354,63.83042592878694,0.7326134736198681,2987.2156090895137,5.36870912E8]}],\"prev_population\":[{\"_candidate\":[792.634660843181,2492.7849019072983,60.45853018964071,0.7336072933432902,2962.9708043614564,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975272647E9,\"fitness\":1.0960015760903588},{\"_candidate\":[872.5176554279782,2535.6175654819463,44.33267220160804,0.7337286195207247,3083.5815515791987,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975272649E9,\"fitness\":0.15266389313494158},{\"_candidate\":[885.9516064178075,2085.536516326114,42.55444706800388,0.7348330520625498,3354.63139252968,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975272651E9,\"fitness\":1.079813530812908}],\"archive\":[{\"birthday\":1.516680401836294E9,\"_candidate\":[1295.8008807732717,2637.186857988159,42.28551979944946,0.7335562104905394,3356.143076920849,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975271854E9,\"fitness\":0.10811803652893556},{\"birthday\":1.516680401837006E9,\"_candidate\":[814.7560112996814,2337.07365521413,54.984365215088005,0.7337628011079379,2957.576817126595,5.36870912E8],\"maximize\":false,\"birthdate\":1.51854497527186E9,\"fitness\":0.0912464245175376},{\"birthday\":1.516680401837007E9,\"_candidate\":[712.2723914467045,2063.6121485420485,62.79467361078451,0.732881328518512,3039.7339669001685,5.36870912E8],\"maximize\":false,\"birthdate\":1.518544975271864E9,\"fitness\":0.09649709137100561}],\"rnd_state\":\"[3, [3065724126, 3544512728, 2711128429, 3097451895, 967083626, 2260479002, 2011495360, 3348237249, 2378344902, 2134917540, 2742669276, 3328880153, 2822272025, 2880910783, 1441442737, 3231087398, 1146288325, 1625737643, 1228322673, 3721661216, 575561661, 2074324378, 3288388299, 1544300809, 2639678219, 2232788457, 1814056570, 2999882935, 2251501756, 2780554798, 2514692131, 687916693, 243141057, 610462178, 692982146, 3824954777, 365727273, 3022710399, 515788952, 4291459547, 667212329, 3358240868, 1492129036, 956252795, 2157732118, 1110061891, 2718486345, 4130618438, 2288404453, 1191040542, 507334165, 2697022142, 121071918, 1290502834, 1145014349, 3916513172, 3188936928, 3598964253, 1811700608, 797604560, 1459985284, 396308629, 536293065, 1861432351, 530738018, 2930646177, 3035022980, 3998314976, 1898847351, 1507100324, 1607137972, 1193970507, 1325837842, 959726587, 4042090604, 3224750321, 1424027500, 1443061177, 1237044770, 928822690, 1302176037, 3568612499, 3723148185, 1430469816, 692515008, 1093644720, 1683640516, 4174602693, 2331072580, 2762007312, 3686093550, 25874128, 229571105, 1580926638, 3078362483, 3211385514, 2382600957, 451342856, 1566049870, 877376711, 1242306867, 3657755180, 1303113933, 1237917316, 833353971, 2461580676, 148331876, 1558943638, 3170368962, 214736109, 3872311144, 2456901485, 1795447425, 1184953807, 3605341307, 580593700, 595817309, 3108476809, 3002792197, 1433369078, 2866216661, 3655047686, 3554797575, 1374808059, 2612494986, 1054441843, 3372403829, 897630532, 1434894456, 340091712, 1412255824, 153502320, 1082509383, 2192258062, 1212656009, 4097614680, 4211472344, 792137819, 326652612, 287540853, 649689138, 3991116082, 3709897114, 807708655, 645794625, 2900452471, 2514181414, 4279235972, 1223752520, 2150305618, 1702086023, 872116565, 3864479340, 1675395747, 2268218023, 1917912323, 2037116366, 2058231706, 2644080529, 2722765597, 4021962428, 3021783242, 3681936083, 425397342, 1090869761, 3299571049, 2177660117, 3662979482, 108545502, 3886174031, 2590960463, 695447006, 1487047828, 2178393656, 4025597868, 255761026, 1767121742, 3200850413, 2301247846, 214790591, 2641517169, 3048332331, 2844465691, 2738599240, 186184734, 2048643725, 1496536706, 3201056846, 2594104286, 1684879809, 1470534656, 1563583168, 1492709862, 3921781135, 1231021635, 1143335356, 868590525, 3688084842, 3363015415, 1081697634, 2064351382, 406335233, 1478861566, 2162286375, 2881004503, 1944164859, 3242596185, 2726419641, 816907820, 2930849747, 3927693308, 253252771, 2885236710, 584068619, 2055555515, 1618917124, 3511474096, 2610543554, 1371679093, 3262865804, 3139346620, 3540194239, 4002012437, 3625334564, 1812113079, 166562410, 106177145, 1899457561, 2200707317, 1039681884, 4172463313, 2407766685, 4146126707, 3004768120, 730301797, 3443113670, 3670981937, 586475702, 2947490368, 2948713681, 3450965965, 1293897586, 952983259, 889011649, 3267239110, 4161489389, 1360814786, 501755720, 2816906094, 2842137958, 2660216416, 419200669, 1408294612, 2373223348, 2328800876, 1584237670, 436351506, 3395527473, 1548715965, 3690533652, 2945734992, 674160573, 1989518247, 3100889221, 2763719839, 2357271150, 212780741, 2985115917, 1457888565, 3607299527, 980443153, 1542209158, 770291775, 746678077, 1412193049, 413891745, 1682193555, 765198090, 4230400852, 2045987415, 2340942203, 615884366, 2478755388, 833488046, 50735976, 2993136664, 889095813, 821147425, 2622280532, 706652556, 1844492101, 3799084036, 3330515368, 542924309, 3530527023, 1845988562, 2375814810, 3642831172, 3051224805, 4172514400, 1886513068, 3672472993, 4151775118, 2762329627, 2221399727, 1473049558, 3816789642, 2398938446, 2845606889, 4284897326, 2904177938, 1598351959, 1855445204, 2594015278, 1528024137, 1980953461, 3474774638, 3255002272, 2105370151, 3250245822, 2344078572, 2735834183, 318492123, 2905820022, 4143524390, 1758152212, 26419169, 3504744669, 922897638, 2284226546, 123574126, 2237741988, 171212654, 1776315349, 2849978615, 3228771518, 1184260950, 1032684138, 3181470798, 2673690187, 4173768130, 2623194448, 3133756108, 851870142, 28138823, 1206311932, 741037646, 3670265831, 404270308, 619902761, 3287601440, 3208299557, 2761870961, 2124804014, 796011685, 2971745401, 784966770, 784332828, 578351065, 725624721, 3683445932, 3771024580, 2422147209, 3762031872, 2347660671, 1933243950, 3711755263, 176475674, 1591596727, 3651471810, 4270080122, 353326368, 3734423915, 3894466421, 2605405796, 4108660146, 2956628083, 3316574506, 2589541445, 3572104663, 3153232913, 1053856203, 2895389043, 1365063224, 1671772292, 1925028221, 902873584, 3595807393, 3097817112, 4108789053, 1759616424, 3746272037, 3717258263, 2672653056, 4147971464, 949492224, 942543068, 1119921660, 4008015534, 3435957522, 3184057267, 4167360692, 1249859248, 7558097, 1133845986, 2943139418, 2073521243, 3617451110, 2229897027, 2555647933, 3726176874, 889452764, 820644837, 1327154049, 2265672064, 2129248553, 412606852, 63204922, 3234344952, 320707343, 3092740087, 44853598, 1945226434, 3159550172, 3086394144, 2012592804, 3990228399, 2867389493, 2781617757, 2882552002, 2806351707, 909035806, 1294586605, 3840428648, 602515456, 2997467429, 2755363986, 147729940, 870496959, 3924354611, 1525531182, 1923355511, 1562510856, 3915804036, 2592881318, 648423540, 3725293333, 2269428648, 467073447, 2595389432, 495610460, 2386665387, 116759568, 750721290, 1275499854, 1875377515, 2736580174, 3938718880, 290001066, 3522290030, 29953608, 3064174778, 4090117350, 2624194794, 1376276064, 2665796666, 634445370, 70168509, 32198189, 3001234425, 1631602258, 277495047, 3121947616, 453054115, 2838793569, 4146704714, 2831492717, 596378763, 2178259917, 1832432377, 716277901, 3659429932, 1323761615, 66127783, 384298520, 774313993, 1449039381, 1428144251, 2553931515, 281546735, 2989769939, 3020480002, 3096161781, 2155328679, 3728427667, 4115395479, 4229426936, 1338120410, 3780413236, 3122514188, 3388096654, 2062867830, 3244651183, 832086074, 3991330458, 1136227927, 3604120290, 3712104605, 3520462657, 495374736, 2067270593, 2387664894, 1602148084, 1240393197, 2419712601, 3040375177, 1081946879, 3023252227, 3890705166, 482850694, 4013987459, 1571040181, 1521329843, 1705401874, 1682094409, 1128205548, 1899120173, 241188982, 36185350, 78024647, 509896880, 3877417777, 3269955810, 2336805412, 1743390219, 2792517786, 3058532670, 3731231273, 3104120029, 1006319366, 748566713, 1149848531, 1009523072, 3209403427, 3084624506, 554189074, 4207439260, 2186236223, 2662732177, 335450920, 662450073, 1116746604, 3056019694, 702760901, 3778868255, 2871867096, 1719987053, 3734422564, 39611381, 2168517922, 148059197, 794689514, 3195118058, 2819533870, 3577103586, 1534358750, 2054529561, 1340212594, 1707171826, 980558901, 3541812777, 2304960896, 2998240578, 32794602, 1503372561, 2976532095, 2080503067, 1190284614, 448190919, 1424477249, 3703469626, 278436083, 2339142365, 849039042, 1068240596, 2250237233, 1480326781, 1116357061, 3778674107, 3209970900, 3562388741, 1667085518, 1988254279, 3554198962, 1065233563, 1150015936, 3712709535, 3084752011, 1219620914, 4018979140, 3511230272, 1047524801, 314747652, 4224631036, 570175320, 304193502, 2970362892, 3636317124, 3535211386, 1561392126, 4107979711, 687771063, 1576513160, 3777146659, 3706627510, 2997104755, 1060609260, 3065422355, 1894842799, 3754972445, 3533484985, 2371212970, 2398165984, 1668517555, 588], null]\"}',parsedatetime('2018-01-22 15:58:58','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')); -INSERT INTO `job_suggested_param_value`(id, job_execution_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES (12569,1624,1,218.6447449212757,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12570,1624,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12571,1624,3,43.796210012462936,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12572,1624,4,0.7053483939633272,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12573,1624,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12574,1624,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12575,1624,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12576,1624,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12577,1624,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12578,1625,1,181.9245696813968,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12579,1625,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12580,1625,3,42.047012100221565,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12581,1625,4,0.7037704502574763,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12582,1625,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12583,1625,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12584,1625,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12585,1625,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12586,1625,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12587,1626,1,140.63900164342985,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12588,1626,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12589,1626,3,51.426234701754865,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12590,1626,4,0.6951498772580411,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12591,1626,5,1976.5719588408601,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12592,1626,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12593,1626,7,1482.428969130645,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12594,1626,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12595,1626,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12596,1627,1,171.02605253760447,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12597,1627,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12598,1627,3,54.84590775607139,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12599,1627,4,0.7115250205281223,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12600,1627,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12601,1627,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12602,1627,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12603,1627,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12604,1627,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12605,1628,1,181.9245696813968,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12606,1628,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12607,1628,3,42.047012100221565,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12608,1628,4,0.7037704502574763,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12609,1628,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12610,1628,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12611,1628,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12612,1628,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12613,1628,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12614,1629,1,213.25485474773512,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12615,1629,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12616,1629,3,31.525782197486357,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12617,1629,4,0.7476923282427653,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12618,1629,5,1581.3588284589678,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12619,1629,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12620,1629,7,1186.0191213442258,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12621,1629,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12622,1629,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(13330,1719,1,792.634660843181,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13331,1719,2,2492.7849019072983,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13332,1719,3,60.45853018964071,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13333,1719,4,0.7336072933432902,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13334,1719,5,2962.9708043614564,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13335,1719,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13336,1719,7,2222.2281032710926,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13337,1719,8,1869.5886764304737,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13338,1719,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13339,1720,1,872.5176554279782,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13340,1720,2,2535.6175654819463,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13341,1720,3,44.33267220160804,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13342,1720,4,0.7337286195207247,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13343,1720,5,3083.5815515791987,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13344,1720,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13345,1720,7,2312.686163684399,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13346,1720,8,1901.7131741114597,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13347,1720,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13348,1721,1,885.9516064178075,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13349,1721,2,2085.536516326114,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13350,1721,3,42.55444706800388,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13351,1721,4,0.7348330520625498,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13352,1721,5,3354.63139252968,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13353,1721,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13354,1721,7,2515.97354439726,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13355,1721,8,1564.1523872445855,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(13356,1721,9,536870912,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(20462,2833,1,1459.6899166265118,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20463,2833,2,2421.8450860158327,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20464,2833,3,45.39309760195506,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20465,2833,4,0.7338175361303875,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20466,2833,5,2690.0010009930634,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20467,2833,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20468,2833,7,2017.5007507447976,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20469,2833,8,1816.3838145118746,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20470,2833,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20471,2834,1,678.4611819565604,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20472,2834,2,2233.783342819729,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20473,2834,3,67.46065454683335,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20474,2834,4,0.7337482196069645,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20475,2834,5,3068.4288351321584,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20476,2834,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20477,2834,7,2301.3216263491186,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20478,2834,8,1675.3375071147968,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20479,2834,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20480,2835,1,888.1718411828987,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20481,2835,2,2391.4206410271354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20482,2835,3,63.83042592878694,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20483,2835,4,0.7326134736198681,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20484,2835,5,2987.2156090895137,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20485,2835,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20486,2835,7,2240.4117068171354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20487,2835,8,1793.5654807703515,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20488,2835,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')); + +INSERT INTO `job_execution`(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1624,'https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0',51,847,'SUCCEEDED',196.53583333333333,43.996566666666666,3990998122525,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:08:44','dd-MM-yyyy hh:mm:ss')), +(1625,'https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0',51,846,'SUCCEEDED',169.09791666666666,34.86125,3990827471029,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:58:41','dd-MM-yyyy hh:mm:ss')), +(1626,'https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0',51,848,'SUCCEEDED',166.11400634765624,33.7234,3990138654674,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')), +(1719,'https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2035,'FAILED',0,0,0,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')), +(1720,'https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2036,'SUCCEEDED',0.07686197916666666,3.5777,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')), +(1721,'https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2047,'SUCCEEDED',0.21555555555555556,3.5456666666666665,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')); + +INSERT INTO `job_suggested_param_set`(fitness_job_execution_id, tuning_algorithm_id, param_set_state, is_param_set_default, fitness, is_param_set_best, are_constraints_violated, job_definition_id,id) VALUES +(1624,1,'FITNESS_COMPUTED',0,0.05287618226970775,0,0,51,1624), +(1625,1,'FITNESS_COMPUTED',0,0.04549620518409709,0,0,51,1625), +(1626,1,'FITNESS_COMPUTED',0,0.044701092268747876,0,0,51,1626), +(NULL,1,'CREATED',0,NULL,0,0,51,1627), +(NULL,1,'CREATED',0,NULL,0,0,51,1628), +(NULL,1,'CREATED',0,NULL,0,0,51,1629), +(1719,1,'FITNESS_COMPUTED',0,1.0960015760903588,0,0,54,1719), +(1720,1,'FITNESS_COMPUTED',0,0.15266389313494158,0,0,54,1720), +(1721,1,'FITNESS_COMPUTED',0,1.079813530812908,0,0,54,1721), +(2833,1,'FITNESS_COMPUTED',0,0.54365576171875,0,0,54,2833), +(NULL,1,'CREATED',0,NULL,0,0,54,2834), +(NULL,1,'CREATED',0,NULL,0,0,54,2835); + +INSERT INTO `job_suggested_param_value`(id, job_suggested_param_set_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES +(12569,1624,1,218.6447449212757,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')), +(12570,1624,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12571,1624,3,43.796210012462936,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12572,1624,4,0.7053483939633272,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12573,1624,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12574,1624,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12575,1624,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12576,1624,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12577,1624,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12578,1625,1,181.9245696813968,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12579,1625,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12580,1625,3,42.047012100221565,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12581,1625,4,0.7037704502574763,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12582,1625,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12583,1625,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12584,1625,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12585,1625,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12586,1625,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12587,1626,1,140.63900164342985,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12588,1626,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12589,1626,3,51.426234701754865,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12590,1626,4,0.6951498772580411,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12591,1626,5,1976.5719588408601,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12592,1626,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12593,1626,7,1482.428969130645,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12594,1626,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12595,1626,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12596,1627,1,171.02605253760447,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12597,1627,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12598,1627,3,54.84590775607139,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12599,1627,4,0.7115250205281223,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12600,1627,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12601,1627,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12602,1627,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12603,1627,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12604,1627,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12605,1628,1,181.9245696813968,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12606,1628,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12607,1628,3,42.047012100221565,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12608,1628,4,0.7037704502574763,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12609,1628,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12610,1628,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12611,1628,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12612,1628,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12613,1628,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12614,1629,1,213.25485474773512,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12615,1629,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12616,1629,3,31.525782197486357,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12617,1629,4,0.7476923282427653,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12618,1629,5,1581.3588284589678,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12619,1629,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12620,1629,7,1186.0191213442258,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12621,1629,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12622,1629,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(13330,1719,1,792.634660843181,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13331,1719,2,2492.7849019072983,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13332,1719,3,60.45853018964071,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13333,1719,4,0.7336072933432902,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13334,1719,5,2962.9708043614564,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13335,1719,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13336,1719,7,2222.2281032710926,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13337,1719,8,1869.5886764304737,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13338,1719,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13339,1720,1,872.5176554279782,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13340,1720,2,2535.6175654819463,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13341,1720,3,44.33267220160804,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13342,1720,4,0.7337286195207247,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13343,1720,5,3083.5815515791987,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13344,1720,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13345,1720,7,2312.686163684399,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13346,1720,8,1901.7131741114597,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13347,1720,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13348,1721,1,885.9516064178075,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13349,1721,2,2085.536516326114,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13350,1721,3,42.55444706800388,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13351,1721,4,0.7348330520625498,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13352,1721,5,3354.63139252968,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13353,1721,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13354,1721,7,2515.97354439726,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13355,1721,8,1564.1523872445855,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(13356,1721,9,536870912,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(20462,2833,1,1459.6899166265118,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20463,2833,2,2421.8450860158327,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20464,2833,3,45.39309760195506,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20465,2833,4,0.7338175361303875,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20466,2833,5,2690.0010009930634,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20467,2833,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20468,2833,7,2017.5007507447976,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20469,2833,8,1816.3838145118746,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20470,2833,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20471,2834,1,678.4611819565604,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20472,2834,2,2233.783342819729,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20473,2834,3,67.46065454683335,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20474,2834,4,0.7337482196069645,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20475,2834,5,3068.4288351321584,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20476,2834,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20477,2834,7,2301.3216263491186,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20478,2834,8,1675.3375071147968,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20479,2834,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20480,2835,1,888.1718411828987,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20481,2835,2,2391.4206410271354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20482,2835,3,63.83042592878694,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20483,2835,4,0.7326134736198681,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20484,2835,5,2987.2156090895137,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20485,2835,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20486,2835,7,2240.4117068171354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20487,2835,8,1793.5654807703515,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20488,2835,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')); INSERT INTO `yarn_app_result`(id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) VALUES ('application_1506645932520_14618965','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864742550,1515864947012,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618965','Pig',3,1809,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',104318976,16860147,12093),('application_1506645932520_14618978','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864747221,1515865052849,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618978','Pig',3,5835,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',349622272,53433482,24885),('application_1506645932520_14618986','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864749064,1515864871027,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618986','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',423936,75348,3168),('application_1506645932520_14618990','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864750944,1515864963817,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618990','Pig',2,408,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',33875968,5673563,42593),('application_1506645932520_14618994','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864752042,1515864968073,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618994','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',985088,149591,3109),('application_1506645932520_14819554','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987211575,1515987345216,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819554','Pig',2,994,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41111040,0,5629),('application_1506645932520_14819557','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987213364,1515987353613,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819557','Pig',4,6520,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',148128768,0,4803),('application_1506645932520_14819559','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987214957,1515987289457,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819559','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',159744,0,2335),('application_1506645932520_14819561','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987216374,1515987328839,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819561','Pig',2,298,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',12905472,0,2710),('application_1506645932520_14819564','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987217458,1515987328877,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819564','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',242688,0,3036),('application_1506645932520_14819566','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987218578,1515987337373,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819566','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',276480,0,2428),('application_1506645932520_14819567','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987219755,1515987345926,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819567','Pig',2,996,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',39914496,0,4385),('application_1506645932520_14819799','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987384869,1515987654239,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819799','Pig',3,3171,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',84836352,0,8879),('application_1506645932520_14819804','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987386490,1515987624024,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819804','Pig',3,3168,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',86373888,0,7496),('application_1506645932520_14819806','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987387715,1515987628906,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819806','Pig',4,10113,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',204218880,0,12516),('application_1506645932520_14820502','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987672309,1515987731074,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820502','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',95232,0,4795),('application_1506645932520_14820601','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987743030,1515987809262,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820601','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',79872,0,4812),('application_1506645932520_14820710','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987821752,1515987866959,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820710','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',46080,0,5151),('application_1506645932520_14820781','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987880705,1515988060797,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820781','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',2429952,0,30350),('application_1506645932520_14820903','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988082745,1515988166282,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820903','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1453056,0,4934),('application_1506645932520_14820904','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988083623,1515988172147,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820904','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',615936,0,5217),('application_1506645932520_14820960','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988182390,1515988255602,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820960','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',328704,0,5271),('application_1506645932520_14820995','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988270096,1515988320392,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820995','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',145920,0,4778),('application_1506645932520_14824675','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991076714,1515991231731,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824675','Pig',3,1518,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',51489792,0,3885),('application_1506645932520_14824676','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991079060,1515991237503,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824676','Pig',3,4896,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',158307840,0,5956),('application_1506645932520_14824677','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991080425,1515991187851,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824677','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',233472,0,2877),('application_1506645932520_14824680','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991082049,1515991240032,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824680','Pig',2,302,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',13275648,0,43306),('application_1506645932520_14824683','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991083758,1515991247488,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824683','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',281088,0,40714),('application_1506645932520_14824685','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991085047,1515991265419,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824685','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',287232,0,41607),('application_1506645932520_14824687','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991086759,1515991294536,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824687','Pig',3,1500,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41547264,0,70495),('application_1506645932520_14824996','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991326143,1515991641190,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824996','Pig',3,3213,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',92054016,0,6387),('application_1506645932520_14824999','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991327452,1515991670369,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824999','Pig',3,3165,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',117229056,0,11406),('application_1506645932520_14825000','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991328750,1515991660916,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825000','Pig',4,10129,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',243571200,0,19257),('application_1506645932520_14825298','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991684212,1515991767524,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825298','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',127488,0,4634),('application_1506645932520_14825364','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991778730,1515991873235,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825364','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',124416,0,4678),('application_1506645932520_14825463','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991888162,1515991971005,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825463','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',56832,0,37362),('application_1506645932520_14825523','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991982696,1515992198290,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825523','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',3177984,0,6575),('application_1506645932520_14825763','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992221397,1515992318476,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825763','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1463808,0,18682),('application_1506645932520_14825769','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992222425,1515992350395,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825769','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',737280,0,4593),('application_1506645932520_14826095','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992373009,1515992451751,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14826095','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',347136,0,5224),('application_1506645932520_14826163','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992466247,1515992538144,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14826163','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',198144,0,5388),('application_1506645932520_14828901','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994413751,1515994540123,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828901','Pig',3,1527,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41195520,0,3799),('application_1506645932520_14828909','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994415531,1515994564943,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828909','Pig',4,6496,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',135762432,0,21059),('application_1506645932520_14828915','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994417787,1515994516440,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828915','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',192000,0,2372),('application_1506645932520_14828918','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994418955,1515994535247,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828918','Pig',2,300,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',11008512,0,3043),('application_1506645932520_14828920','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994420115,1515994548888,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828920','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',271872,0,2402),('application_1506645932520_14828927','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994421136,1515994524042,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828927','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',215040,0,3006),('application_1506645932520_14828931','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994422273,1515994554177,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828931','Pig',2,990,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',40410624,0,17972),('application_1506645932520_14829374','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994598599,1515994826719,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829374','Pig',3,3234,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',76623663,0,13428),('application_1506645932520_14829375','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994599701,1515994811605,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829375','Pig',3,2643,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',73923525,0,12955),('application_1506645932520_14829377','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994601160,1515994871831,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829377','Pig',4,10093,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',227204472,0,12798),('application_1506645932520_14829662','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994884732,1515994942466,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829662','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',100554,0,5346),('application_1506645932520_14829738','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994954814,1515995030508,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829738','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',102957,0,4962),('application_1506645932520_14829819','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995045481,1515995093836,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829819','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',58839,0,5399),('application_1506645932520_14829871','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995113082,1515995270705,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829871','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',2724270,0,4927),('application_1506645932520_14830034','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995291854,1515995356948,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830034','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1442952,0,4843),('application_1506645932520_14830035','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995293330,1515995363322,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830035','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',609561,0,4607),('application_1506645932520_14830183','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995379978,1515995475455,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830183','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',346434,0,35580),('application_1506645932520_14830345','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995489495,1515995541281,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830345','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',169446,0,4860),('application_1506645932520_18304591','PigLatin:count_by_country_small.pig','mkumar1','sna_default',1517997086752,1517997308615,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_18304591','Pig',2,2,0,'azkaban','countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmallNew_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmallNew_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew',28334400,1248,7201),('application_1506645932520_18311651','PigLatin:count_by_country_small.pig','mkumar1','sna_default',1518001319781,1518001775270,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_18311651','Pig',3,6,0,'azkaban','countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5221700','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall','https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5221700','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall',794624,469908,242749); INSERT INTO `yarn_app_heuristic_result` VALUES (1511830,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511831,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511832,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1511833,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511834,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511835,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511836,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511837,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511838,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511839,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511840,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511841,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511842,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511843,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511844,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1511845,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511846,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511847,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511848,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511849,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511850,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511851,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511852,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511853,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511866,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511867,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511868,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,408),(1511869,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511870,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511871,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511872,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511873,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511874,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511875,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511876,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511877,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511878,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511879,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511880,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1809),(1511881,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511882,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511883,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511884,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511885,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511886,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511887,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511888,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511889,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511902,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511903,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511904,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,5835),(1511905,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511906,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511907,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511908,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511909,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511910,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511911,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511912,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511913,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519428,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519429,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519430,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519431,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519432,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519433,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519434,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519435,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519436,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519437,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519438,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519439,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519440,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519441,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519442,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519443,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519444,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519445,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519446,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519447,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519448,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519449,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519450,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519451,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519452,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519453,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519454,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519455,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519456,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519457,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519458,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519459,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519460,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519461,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519462,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519463,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519464,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519465,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519466,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,298),(1519467,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519468,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519469,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519470,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519471,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519472,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519473,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519474,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519475,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519476,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519477,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519478,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,996),(1519479,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519480,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519481,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519482,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519483,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519484,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519485,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519486,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519487,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519488,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519489,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519490,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,994),(1519491,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519492,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519493,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519494,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519495,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519496,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519497,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519498,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519499,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519512,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519513,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519514,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,6520),(1519515,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519516,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519517,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519518,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519519,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519520,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519521,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519522,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519523,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519524,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519525,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519526,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1503),(1519527,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519528,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519529,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519530,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519531,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519532,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1665),(1519533,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519534,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519535,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519536,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519537,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519538,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519539,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519540,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519541,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519542,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519543,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519544,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519545,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519546,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519547,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519548,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519549,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519550,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519551,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519552,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519553,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519554,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519555,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519556,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519557,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519558,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519559,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519560,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519561,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519562,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519563,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519564,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519565,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519566,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519567,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519568,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519569,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519570,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519571,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519572,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519573,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519574,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519575,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519576,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519577,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519578,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1519579,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519580,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519581,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519582,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519583,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519584,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519585,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519586,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519587,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519588,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519589,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519590,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519591,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519592,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519593,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519594,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519595,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519596,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519597,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519598,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519599,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519600,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519601,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519602,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519603,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519604,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519605,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519606,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519607,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519608,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519609,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519610,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519611,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519612,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519613,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519614,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519615,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519616,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519617,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519618,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519619,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519620,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519621,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519622,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519623,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519624,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519625,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519626,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519627,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519628,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519629,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519630,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519631,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519632,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519633,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519634,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1500),(1519635,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519636,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519637,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519638,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519639,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519640,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1671),(1519641,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519642,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519643,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519644,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519645,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519646,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519647,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519648,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519649,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519650,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519651,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519652,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519653,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519654,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519655,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519656,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519657,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519658,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,302),(1519659,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519660,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519661,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519662,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519663,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519664,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519665,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519666,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519667,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519668,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519669,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519670,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1518),(1519671,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519672,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519673,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519674,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519675,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519676,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519677,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519678,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519679,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519680,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519681,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519682,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7116),(1519683,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519684,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519685,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519686,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519687,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519688,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1519689,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519690,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519691,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519692,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519693,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519694,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519695,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519696,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519697,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519698,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519699,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519700,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519701,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519702,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519703,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519704,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519705,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519706,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519707,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519708,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519709,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519710,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519711,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519712,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519713,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519714,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519715,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519716,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519717,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519718,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1500),(1519719,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519720,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519721,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519722,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519723,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519724,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519725,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519726,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519727,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519728,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519729,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519730,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,4896),(1519731,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519732,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519733,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519734,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519735,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519736,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519737,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519738,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519739,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519740,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519741,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519742,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1527),(1519743,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519744,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519745,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519746,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519747,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519748,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1686),(1519749,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519750,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519751,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519752,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519753,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519754,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519755,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519756,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519757,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519758,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519759,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519760,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519761,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519762,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519763,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519764,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519765,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519766,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519767,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519768,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519769,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519770,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519771,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519772,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519773,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519774,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519775,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519776,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519777,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519778,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519779,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519780,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519781,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519782,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519783,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519784,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519785,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519786,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519787,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519788,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519789,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519790,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519791,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519792,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519793,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519794,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1519795,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519796,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519797,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519798,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519799,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519800,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519801,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519802,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519803,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519804,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519805,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519806,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519807,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519808,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519809,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519810,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519811,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519812,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519813,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519814,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519815,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519816,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519817,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519818,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519819,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519820,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519821,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519822,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519823,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519824,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519825,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519826,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519827,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519828,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519829,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519830,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519831,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519832,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519833,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519834,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519835,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519836,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519837,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519838,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519839,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519840,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519841,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519842,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519843,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519844,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519845,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519846,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519847,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519848,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519849,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519850,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1509),(1519851,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519852,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519853,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519854,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519855,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519856,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1656),(1519857,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519858,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519859,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519860,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519861,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519862,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519863,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519864,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519865,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519866,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519867,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519868,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519869,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519870,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519871,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519872,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519873,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519874,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,300),(1519875,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519876,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519877,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519878,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519879,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519880,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519881,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519882,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519883,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519884,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519885,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519886,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519887,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519888,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519889,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519890,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519891,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519892,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519893,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519894,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519895,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519896,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519897,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519898,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1527),(1519899,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519900,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519901,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519902,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519903,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519904,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519905,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519906,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519907,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519908,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519909,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519910,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519911,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519912,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519913,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519914,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519915,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519916,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519917,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519918,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519919,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519920,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519921,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519922,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,990),(1519923,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519924,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519925,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519926,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519927,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519928,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519929,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519930,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519931,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519932,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519933,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519934,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7132),(1519935,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519936,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519937,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519938,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519939,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519940,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1519941,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519942,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519943,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519944,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519945,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519946,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,6496),(1519947,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519948,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519949,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519950,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519951,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519952,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519953,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519954,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519955,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519956,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519957,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519958,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,996),(1519959,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519960,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519961,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519962,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519963,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519964,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1647),(1519965,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519966,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519967,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519968,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519969,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519970,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519971,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519972,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519973,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519974,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519975,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519976,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519977,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519978,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519979,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519980,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519981,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519982,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519983,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519984,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519985,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519986,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519987,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519988,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519989,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519990,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519991,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519992,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519993,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519994,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519995,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519996,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519997,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519998,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519999,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520000,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520001,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520002,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520003,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520004,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520005,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520006,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520007,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520008,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520009,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520010,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1520011,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520012,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520013,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520014,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520015,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520016,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520017,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520018,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520019,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520020,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520021,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520022,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520023,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520024,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520025,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520026,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520027,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520028,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520029,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520030,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520031,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520032,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520033,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520034,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520035,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520036,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520037,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520038,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520039,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520040,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520041,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520042,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520043,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520044,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520045,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520046,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520047,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520048,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520049,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520050,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520051,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520052,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520053,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520054,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520055,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520056,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520057,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520058,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520059,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520060,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520061,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520062,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520063,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520064,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520065,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520066,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1536),(1520067,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520068,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520069,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520070,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520071,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520072,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1698),(1520073,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520074,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520075,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520076,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520077,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520078,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7096),(1520079,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520080,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520081,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520082,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520083,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520084,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1520085,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520086,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520087,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1536115,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1536116,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1536117,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1536118,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1536119,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1536120,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1536121,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1536122,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1536123,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1536124,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',2,2),(1536125,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1536126,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1536163,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1536164,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1536165,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1536166,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1536167,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1536168,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',3,3),(1536169,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1536170,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1536171,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1536172,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',3,3),(1536173,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1536174,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0); diff --git a/test/rest/RestAPITest.java b/test/rest/RestAPITest.java index 0a4b44330..5ac9b93f6 100644 --- a/test/rest/RestAPITest.java +++ b/test/rest/RestAPITest.java @@ -19,19 +19,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.linkedin.drelephant.AutoTuner; import com.linkedin.drelephant.DrElephant; import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.tuning.BaselineComputeUtil; import com.linkedin.drelephant.tuning.FitnessComputeUtil; -import com.linkedin.drelephant.tuning.JobCompleteDetector; import com.linkedin.drelephant.util.Utils; -import common.DBTestUtil; -import controllers.AutoTuningMetricsController; - -import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -42,10 +35,11 @@ import models.JobDefinition; import models.JobExecution; import models.JobExecution.ExecutionState; +import models.JobSuggestedParamSet; import models.TuningJobDefinition; -import models.TuningJobExecution; -import models.TuningJobExecution.ParamSetStatus; +import models.JobSuggestedParamSet.ParamSetStatus; +import models.TuningJobExecutionParamSet; import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Before; @@ -55,7 +49,6 @@ import play.Application; import play.GlobalSettings; -import play.libs.Json; import play.libs.WS; import play.test.FakeApplication; @@ -154,31 +147,29 @@ public void run() { jsonResponse.path("mapreduce.reduce.memory.mb").asDouble() > 0); assertTrue("Get current run param output size did not match", jsonResponse.size() == 9); - TuningJobExecution tuningJobExecution = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*") + TuningJobExecutionParamSet tuningJobExecutionParamSet = TuningJobExecutionParamSet.find.select("*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0") .findUnique(); - tuningJobExecution.paramSetState = ParamSetStatus.EXECUTED; - tuningJobExecution.jobExecution.executionState = ExecutionState.SUCCEEDED; - tuningJobExecution.update(); + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + + jobExecution.executionState = ExecutionState.SUCCEEDED; + jobExecution.update(); + jobSuggestedParamSet.paramSetState = ParamSetStatus.EXECUTED; + jobSuggestedParamSet.update(); FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil(); fitnessComputeUtil.updateFitness(); - tuningJobExecution = TuningJobExecution.find.select("*") - .fetch(TuningJobExecution.TABLE.jobExecution, "*") - .fetch(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.job, "*") - .where() - .eq(TuningJobExecution.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, - "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0") - .findUnique(); + jobSuggestedParamSet = JobSuggestedParamSet.find.byId(jobSuggestedParamSet.id); - assertTrue("Fitness not computed", tuningJobExecution.paramSetState == ParamSetStatus.FITNESS_COMPUTED); - assertTrue("Fitness not computed and have zero value", tuningJobExecution.fitness > 0); + assertTrue("Fitness not computed", jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)); + assertTrue("Fitness is non-positive", jobSuggestedParamSet.fitness > 0); } }); } From 17fcb601ec2391054b19dee6a24a749b6fed7d6c Mon Sep 17 00:00:00 2001 From: ShubhamGupta29 Date: Thu, 9 Aug 2018 12:28:45 +0530 Subject: [PATCH 12/15] Spark heuristic modification (#407) * Revert "Dr. Elephant Tez Support working patch (#313)" This reverts commit a0470a3e6874527291bcc546a39cce28fc9cdd3e. * Rerevert "Dr. Elephant Tez Support working patch (#313)" including attribution. This reverts commit e3fd598d048838006f2da26fe64538bd39d746ed. Co-authored-by: Abhishek Das * Auto tuning: Support for parameter set multi-try (#386) * Changes in some of the Spark Heuristics * Adding test for changes executor gc heuristic and unified memory heuristic * Update ExecutorGcHeuristic.scala * Update UnifiedMemoryHeuristic.scala * Changed some hard coded values to variables * Due to strict inequality changing the other thereshold levels for executor and driver --- .../linkedin/drelephant/ElephantContext.java | 1 - .../spark/fetchers/SparkRestClient.scala | 1 + .../heuristics/ConfigurationHeuristic.scala | 4 +- .../spark/heuristics/DriverHeuristic.scala | 13 ++++--- .../heuristics/ExecutorGcHeuristic.scala | 11 ++++-- .../heuristics/UnifiedMemoryHeuristic.scala | 13 +++++-- .../ConfigurationHeuristicTest.scala | 2 +- .../heuristics/ExecutorGcHeuristicTest.scala | 39 +++++++++++++++++++ .../UnifiedMemoryHeuristicTest.scala | 23 +++++++++++ 9 files changed, 92 insertions(+), 15 deletions(-) diff --git a/app/com/linkedin/drelephant/ElephantContext.java b/app/com/linkedin/drelephant/ElephantContext.java index af09d3dc4..51c016ad9 100644 --- a/app/com/linkedin/drelephant/ElephantContext.java +++ b/app/com/linkedin/drelephant/ElephantContext.java @@ -110,7 +110,6 @@ private void loadConfiguration() { loadFetchers(); loadHeuristics(); loadJobTypes(); - loadGeneralConf(); loadAutoTuningConf(); diff --git a/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala b/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala index 21c4fb1a3..20ec8cbb9 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala @@ -101,6 +101,7 @@ class SparkRestClient(sparkConf: SparkConf) { Await.result(futureJobDatas, DEFAULT_TIMEOUT), Await.result(futureStageDatas, DEFAULT_TIMEOUT), Await.result(futureExecutorSummaries, Duration(5, SECONDS)), + Seq.empty, Await.result(futureLogData, Duration(5, SECONDS)) ) diff --git a/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala index 5af1ed94f..7b5bd3939 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala @@ -118,7 +118,7 @@ class ConfigurationHeuristic(private val heuristicConfigurationData: HeuristicCo result.addResultDetail("Executor Overhead Memory", "Please do not specify excessive amount of overhead memory for Executors. Change it in the field " + SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD) } if(evaluator.severityExecutorCores != Severity.NONE) { - result.addResultDetail("Executor cores", "The number of executor cores should be <=" + evaluator.DEFAULT_SPARK_CORES_THRESHOLDS.low + ". Please change it in the field " + SPARK_EXECUTOR_CORES_KEY) + result.addResultDetail("Executor cores", "The number of executor cores should be <" + evaluator.DEFAULT_SPARK_CORES_THRESHOLDS.low + ". Please change it in the field " + SPARK_EXECUTOR_CORES_KEY) } if(evaluator.severityExecutorMemory != Severity.NONE) { result.addResultDetail("Executor memory", "Please do not specify excessive amount of executor memory. Change it in the field " + SPARK_EXECUTOR_MEMORY_KEY) @@ -204,7 +204,7 @@ object ConfigurationHeuristic { SeverityThresholds(low = MemoryFormatUtils.stringToBytes("10G"), MemoryFormatUtils.stringToBytes("15G"), severe = MemoryFormatUtils.stringToBytes("20G"), critical = MemoryFormatUtils.stringToBytes("25G"), ascending = true) val DEFAULT_SPARK_CORES_THRESHOLDS = - SeverityThresholds(low = 4, moderate = 6, severe = 8, critical = 10, ascending = true) + SeverityThresholds(low = 5, moderate = 7, severe = 9, critical = 11, ascending = true) val severityExecutorMemory = DEFAULT_SPARK_MEMORY_THRESHOLDS.severityOf(executorMemoryBytes.getOrElse(0).asInstanceOf[Number].longValue) val severityExecutorCores = DEFAULT_SPARK_CORES_THRESHOLDS.severityOf(executorCores.getOrElse(0).asInstanceOf[Number].intValue) diff --git a/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala index b52356b07..a1bd12a15 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala @@ -61,9 +61,10 @@ class DriverHeuristic(private val heuristicConfigurationData: HeuristicConfigura SPARK_DRIVER_MEMORY_KEY, formatProperty(evaluator.driverMemoryBytes.map(MemoryFormatUtils.bytesToString)) ), - new HeuristicResultDetails( - "Ratio of time spent in GC to total time", evaluator.ratio.toString - ), + //Removing driver GC heuristics for now +// new HeuristicResultDetails( +// "Ratio of time spent in GC to total time", evaluator.ratio.toString +// ), new HeuristicResultDetails( SPARK_DRIVER_CORES_KEY, formatProperty(evaluator.driverCores.map(_.toString)) @@ -76,6 +77,7 @@ class DriverHeuristic(private val heuristicConfigurationData: HeuristicConfigura ) if(evaluator.severityJvmUsedMemory != Severity.NONE) { resultDetails = resultDetails :+ new HeuristicResultDetails("Driver Peak JVM used Memory", "The allocated memory for the driver (in " + SPARK_DRIVER_MEMORY_KEY + ") is much more than the peak JVM used memory by the driver.") + resultDetails = resultDetails :+ new HeuristicResultDetails("Suggested spark.driver.memory", MemoryFormatUtils.roundOffMemoryStringToNextInteger(MemoryFormatUtils.bytesToString(((1 + BUFFER_FRACTION) * (evaluator.maxDriverPeakJvmUsedMemory + reservedMemory)).toLong))) } if (evaluator.severityGc != Severity.NONE) { resultDetails = resultDetails :+ new HeuristicResultDetails("Gc ratio high", "The driver is spending too much time on GC. We recommend increasing the driver memory.") @@ -113,6 +115,7 @@ object DriverHeuristic { val EXECUTION_MEMORY = "executionMemory" val STORAGE_MEMORY = "storageMemory" val JVM_USED_MEMORY = "jvmUsedMemory" + val BUFFER_FRACTION = 0.2 // 300 * FileUtils.ONE_MB (300 * 1024 * 1024) val reservedMemory : Long = 314572800 @@ -172,10 +175,10 @@ object DriverHeuristic { //The following thresholds are for checking if the memory and cores values (driver) are above normal. These thresholds are experimental, and may change in the future. val DEFAULT_SPARK_MEMORY_THRESHOLDS = - SeverityThresholds(low = MemoryFormatUtils.stringToBytes("10G"), MemoryFormatUtils.stringToBytes("15G"), + SeverityThresholds(low = MemoryFormatUtils.stringToBytes("10G"), moderate = MemoryFormatUtils.stringToBytes("15G"), severe = MemoryFormatUtils.stringToBytes("20G"), critical = MemoryFormatUtils.stringToBytes("25G"), ascending = true) val DEFAULT_SPARK_CORES_THRESHOLDS = - SeverityThresholds(low = 4, moderate = 6, severe = 8, critical = 10, ascending = true) + SeverityThresholds(low = 5, moderate = 7, severe = 9, critical = 11, ascending = true) val severityDriverMemory = DEFAULT_SPARK_MEMORY_THRESHOLDS.severityOf(driverMemoryBytes.getOrElse(0).asInstanceOf[Number].longValue) val severityDriverCores = DEFAULT_SPARK_CORES_THRESHOLDS.severityOf(driverCores.getOrElse(0).asInstanceOf[Number].intValue) diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala index 7b8929ed7..32ca515ce 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala @@ -59,7 +59,7 @@ class ExecutorGcHeuristic(private val heuristicConfigurationData: HeuristicConfi } //severityTimeD corresponds to the descending severity calculation if (evaluator.severityTimeD.getValue > Severity.LOW.getValue) { - resultDetails = resultDetails :+ new HeuristicResultDetails("Gc ratio low", "The job is spending too less time in GC. Please check if you have asked for more executor memory than required.") + resultDetails = resultDetails :+ new HeuristicResultDetails("Gc ratio low", "The job is spending too little time in GC. Please check if you have asked for more executor memory than required.") } val result = new HeuristicResult( @@ -76,6 +76,7 @@ class ExecutorGcHeuristic(private val heuristicConfigurationData: HeuristicConfi object ExecutorGcHeuristic { val SPARK_EXECUTOR_MEMORY = "spark.executor.memory" val SPARK_EXECUTOR_CORES = "spark.executor.cores" + val EXECUTOR_RUNTIME_THRESHOLD_IN_MINUTES = 5 /** The ascending severity thresholds for the ratio of JVM GC Time and executor Run Time (checking whether ratio is above normal) * These thresholds are experimental and are likely to change */ @@ -107,7 +108,12 @@ object ExecutorGcHeuristic { var ratio: Double = jvmTime.toDouble / executorRunTimeTotal.toDouble - lazy val severityTimeA: Severity = executorGcHeuristic.gcSeverityAThresholds.severityOf(ratio) + //If the total Executor Runtime is less then 5 minutes then we won't consider for the severity due to GC + lazy val severityTimeA: Severity = if (executorRunTimeTotal >= (EXECUTOR_RUNTIME_THRESHOLD_IN_MINUTES * Statistics.MINUTE_IN_MS)) + executorGcHeuristic.gcSeverityAThresholds.severityOf(ratio) + else + Severity.NONE + lazy val severityTimeD: Severity = executorGcHeuristic.gcSeverityDThresholds.severityOf(ratio) /** @@ -145,4 +151,3 @@ object ExecutorGcHeuristic { } } } - diff --git a/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala index 53d261d9b..9c4c3397a 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala @@ -72,6 +72,8 @@ object UnifiedMemoryHeuristic { val PEAK_UNIFIED_MEMORY_THRESHOLD_KEY = "peak_unified_memory_threshold" val SPARK_EXECUTOR_MEMORY_THRESHOLD_KEY = "spark_executor_memory_threshold" val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD = "2G" + val UNIFIED_MEMORY_ALLOCATED_THRESHOLD = "256M" + val SPARK_MEMORY_FRACTION_THRESHOLD : Double = 0.05 class Evaluator(unifiedMemoryHeuristic: UnifiedMemoryHeuristic, data: SparkApplicationData) { lazy val appConfigurationProperties: Map[String, String] = @@ -116,10 +118,15 @@ object UnifiedMemoryHeuristic { } }.max - lazy val severity: Severity = if (sparkExecutorMemory <= MemoryFormatUtils.stringToBytes(unifiedMemoryHeuristic.sparkExecutorMemoryThreshold)) { - Severity.NONE + //If sparkMemoryFraction or total Unified Memory allocated is less than their respective thresholds then won't consider for severity + lazy val severity: Severity = if (sparkMemoryFraction > SPARK_MEMORY_FRACTION_THRESHOLD && maxMemory > MemoryFormatUtils.stringToBytes(UNIFIED_MEMORY_ALLOCATED_THRESHOLD)) { + if (sparkExecutorMemory <= MemoryFormatUtils.stringToBytes(unifiedMemoryHeuristic.sparkExecutorMemoryThreshold)) { + Severity.NONE + } else { + PEAK_UNIFIED_MEMORY_THRESHOLDS.severityOf(maxUnifiedMemory) + } } else { - PEAK_UNIFIED_MEMORY_THRESHOLDS.severityOf(maxUnifiedMemory) + Severity.NONE } } } diff --git a/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala index f22b5379e..80be4541c 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala @@ -143,7 +143,7 @@ class ConfigurationHeuristicTest extends FunSpec with Matchers { it("returns executor cores") { val details = heuristicResultDetails.get(10) details.getName should include("Executor cores") - details.getValue should be("The number of executor cores should be <=4. Please change it in the field spark.executor.cores") + details.getValue should be("The number of executor cores should be <5. Please change it in the field spark.executor.cores") } } diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala index 2204922c1..f60d453b2 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala @@ -67,18 +67,57 @@ class ExecutorGcHeuristicTest extends FunSpec with Matchers { ) ) + val executorSummaries2 = Seq( + newFakeExecutorSummary( + id = "1", + totalGCTime = 12000, + totalDuration = Duration("4min").toMillis + ), + newFakeExecutorSummary( + id = "2", + totalGCTime = 13000, + totalDuration = Duration("1min").toMillis + ) + ) + + val executorSummaries3 = Seq( + newFakeExecutorSummary( + id = "1", + totalGCTime = 9000, + totalDuration = Duration("2min").toMillis + ) + ) + describe(".apply") { val data = newFakeSparkApplicationData(executorSummaries) val data1 = newFakeSparkApplicationData(executorSummaries1) + val data2 = newFakeSparkApplicationData(executorSummaries2) + val data3 = newFakeSparkApplicationData(executorSummaries3) val heuristicResult = executorGcHeuristic.apply(data) val heuristicResult1 = executorGcHeuristic.apply(data1) + val heuristicResult2 = executorGcHeuristic.apply(data2) + val heuristicResult3 = executorGcHeuristic.apply(data3) val heuristicResultDetails = heuristicResult.getHeuristicResultDetails val heuristicResultDetails1 = heuristicResult1.getHeuristicResultDetails + val heuristicResultDetails2 = heuristicResult2.getHeuristicResultDetails + val heuristicResultDetails3 = heuristicResult3.getHeuristicResultDetails it("returns the severity") { heuristicResult.getSeverity should be(Severity.CRITICAL) } + it("return the low severity") { + heuristicResult2.getSeverity should be(Severity.LOW) + } + + it("return NONE severity for runtime less than 5 min") { + heuristicResult2.getSeverity should be(Severity.LOW) + } + + it("return none severity") { + heuristicResult3.getSeverity should be(Severity.NONE) + } + it("returns the JVM GC time to Executor Run time duration") { val details = heuristicResultDetails.get(0) details.getName should include("GC time to Executor Run time ratio") diff --git a/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala index 68f668efe..5677e3ffa 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala @@ -21,6 +21,7 @@ class UnifiedMemoryHeuristicTest extends FunSpec with Matchers { val unifiedMemoryHeuristic = new UnifiedMemoryHeuristic(heuristicConfigurationData) val appConfigurationProperties = Map("spark.executor.memory"->"3147483647") val appConfigurationProperties1 = Map("spark.executor.memory"->"214567874847") + val appConfigurationProperties2 = Map("spark.executor.memory"->"214567874847", "spark.memory.fraction"->"0.06") val executorData = Seq( newDummyExecutorData("1", 999999999, Map("executionMemory" -> 300000, "storageMemory" -> 94567)), @@ -41,13 +42,27 @@ class UnifiedMemoryHeuristicTest extends FunSpec with Matchers { newDummyExecutorData("2", 999999999, Map("executionMemory" -> 999999990, "storageMemory" -> 9)) ) + val executorData3 = Seq( + newDummyExecutorData("1", 400000, Map("executionMemory" -> 300000, "storageMemory" -> 94567)), + newDummyExecutorData("2", 500000, Map("executionMemory" -> 5000, "storageMemory" -> 9)) + ) + + val executorData4 = Seq( + newDummyExecutorData("1", 268435460L, Map("executionMemory" -> 300000, "storageMemory" -> 94567)), + newDummyExecutorData("2", 268435460L, Map("executionMemory" -> 900000, "storageMemory" -> 500000)) + ) + describe(".apply") { val data = newFakeSparkApplicationData(appConfigurationProperties, executorData) val data1 = newFakeSparkApplicationData(appConfigurationProperties1, executorData1) val data2 = newFakeSparkApplicationData(appConfigurationProperties1, executorData2) + val data3 = newFakeSparkApplicationData(appConfigurationProperties1, executorData3) + val data4 = newFakeSparkApplicationData(appConfigurationProperties2, executorData4) val heuristicResult = unifiedMemoryHeuristic.apply(data) val heuristicResult1 = unifiedMemoryHeuristic.apply(data1) val heuristicResult2 = unifiedMemoryHeuristic.apply(data2) + val heuristicResult3 = unifiedMemoryHeuristic.apply(data3) + val heuristicResult4 = unifiedMemoryHeuristic.apply(data4) val evaluator = new Evaluator(unifiedMemoryHeuristic, data1) it("has severity") { @@ -85,6 +100,14 @@ class UnifiedMemoryHeuristicTest extends FunSpec with Matchers { it("has no severity when max and allocated memory are the same") { heuristicResult2.getSeverity should be(Severity.NONE) } + + it("has no severity when maxMemory is less than 256Mb") { + heuristicResult3.getSeverity should be(Severity.NONE) + } + + it("has critical severity when maxMemory is greater than 256Mb and spark memory fraction is greater than 0.05") { + heuristicResult4.getSeverity should be(Severity.CRITICAL) + } } } From 0c391771e3907a1416926decd166c21fc65135af Mon Sep 17 00:00:00 2001 From: Himani Jha Date: Mon, 24 Sep 2018 01:17:59 -0700 Subject: [PATCH 13/15] Total Duration changed to difference of executor add and remover time --- .../spark/SparkMetricsAggregator.scala | 2 + .../spark/data/SparkApplicationData.scala | 1 - .../spark/fetchers/SparkFetcher.scala | 2 +- .../fetchers/statusapiv1/statusapiv1.scala | 4 + .../heuristics/ExecutorGcHeuristic.scala | 10 +- .../spark/heuristics/ExecutorsHeuristic.scala | 7 +- .../legacydata/LegacyDataConverters.scala | 2 + .../spark/legacydata/SparkExecutorData.java | 7 +- .../spark/SparkMetricsAggregatorTest.scala | 3 + .../heuristics/DriverHeuristicTest.scala | 3 + .../heuristics/ExecutorGcHeuristicTest.scala | 95 +++++++++++++++---- .../ExecutorStorageSpillHeuristicTest.scala | 3 + .../heuristics/ExecutorsHeuristicTest.scala | 47 ++++++++- .../JvmUsedMemoryHeuristicTest.scala | 3 + .../UnifiedMemoryHeuristicTest.scala | 3 + 15 files changed, 161 insertions(+), 31 deletions(-) diff --git a/app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala b/app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala index c8216cdf4..be6f072a6 100644 --- a/app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala +++ b/app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala @@ -1,3 +1,4 @@ +/* /* * Copyright 2016 LinkedIn Corp. * @@ -13,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + * */ package com.linkedin.drelephant.spark diff --git a/app/com/linkedin/drelephant/spark/data/SparkApplicationData.scala b/app/com/linkedin/drelephant/spark/data/SparkApplicationData.scala index d3be30f7d..766fcb525 100644 --- a/app/com/linkedin/drelephant/spark/data/SparkApplicationData.scala +++ b/app/com/linkedin/drelephant/spark/data/SparkApplicationData.scala @@ -23,7 +23,6 @@ import scala.collection.JavaConverters import com.linkedin.drelephant.analysis.{ApplicationType, HadoopApplicationData} import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfo, ExecutorSummary, JobData, StageData} - case class SparkApplicationData( appId: String, appConfigurationProperties: Map[String, String], diff --git a/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala b/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala index ae425f4df..823b5a6c2 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala @@ -112,7 +112,7 @@ class SparkFetcher(fetcherConfigurationData: FetcherConfigurationData) private def doFetchDataUsingRestAndLogClients(analyticJob: AnalyticJob): Future[SparkApplicationData] = Future { val appId = analyticJob.getAppId val restDerivedData = Await.result(sparkRestClient.fetchData(appId, eventLogSource == EventLogSource.Rest), DEFAULT_TIMEOUT) - + val logDerivedData = eventLogSource match { case EventLogSource.None => None case EventLogSource.Rest => restDerivedData.logDerivedData diff --git a/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala b/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala index 808a3f4b0..cde9f053b 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/statusapiv1/statusapiv1.scala @@ -83,6 +83,8 @@ trait ExecutorSummary{ def totalTasks: Int def maxTasks: Int def totalDuration: Long + def addTime: Date + def endTime: Option[Date] def totalInputBytes: Long def totalShuffleRead: Long def totalShuffleWrite: Long @@ -294,6 +296,8 @@ class ExecutorSummaryImpl( var totalTasks: Int, var maxTasks: Int, var totalDuration: Long, + var addTime: Date, + var endTime: Option[Date], var totalInputBytes: Long, var totalShuffleRead: Long, var totalShuffleWrite: Long, diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala index 32ca515ce..2a2e13c41 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala @@ -102,9 +102,13 @@ object ExecutorGcHeuristic { throw new Exception("No executor information available.") } + lazy val applicationInfo: Seq[ApplicationAttemptInfo] = data.applicationInfo.attempts.filter(_.completed) + + var appInfo: ApplicationAttemptInfo = applicationInfo(0) + lazy val appConfigurationProperties: Map[String, String] = data.appConfigurationProperties - var (jvmTime, executorRunTimeTotal) = getTimeValues(executorSummaries) + var (jvmTime, executorRunTimeTotal) = getTimeValues(executorSummaries, appInfo) var ratio: Double = jvmTime.toDouble / executorRunTimeTotal.toDouble @@ -121,12 +125,12 @@ object ExecutorGcHeuristic { * @param executorSummaries * @return */ - private def getTimeValues(executorSummaries: Seq[ExecutorSummary]): (Long, Long) = { + private def getTimeValues(executorSummaries: Seq[ExecutorSummary], applInfo: ApplicationAttemptInfo): (Long, Long) = { var jvmGcTimeTotal: Long = 0 var executorRunTimeTotal: Long = 0 executorSummaries.foreach(executorSummary => { jvmGcTimeTotal+=executorSummary.totalGCTime - executorRunTimeTotal+=executorSummary.totalDuration + executorRunTimeTotal+=executorSummary.endTime.getOrElse(appInfo.endTime).getTime - executorSummary.addTime.getTime }) (jvmGcTimeTotal, executorRunTimeTotal) } diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala index d125bc492..9b42fe8a0 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala @@ -21,6 +21,7 @@ import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationDa import com.linkedin.drelephant.math.Statistics import com.linkedin.drelephant.spark.data.SparkApplicationData import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary +import com.linkedin.drelephant.spark.fetchers.statusapiv1.ApplicationAttemptInfo import com.linkedin.drelephant.util.MemoryFormatUtils import scala.collection.JavaConverters @@ -150,6 +151,8 @@ object ExecutorsHeuristic { class Evaluator(executorsHeuristic: ExecutorsHeuristic, data: SparkApplicationData) { lazy val executorSummaries: Seq[ExecutorSummary] = data.executorSummaries + lazy val appAttemptInfo : ApplicationAttemptInfo = data.applicationInfo.attempts.filter(_.completed)(0) + lazy val totalStorageMemoryAllocated: Long = executorSummaries.map { _.maxMemory }.sum lazy val totalStorageMemoryUsed: Long = executorSummaries.map { _.memoryUsed }.sum @@ -163,9 +166,9 @@ object ExecutorsHeuristic { severityOfDistribution(storageMemoryUsedDistribution, ignoreMaxBytesLessThanThreshold) lazy val taskTimeDistribution: Distribution = - Distribution(executorSummaries.map { _.totalDuration }) + Distribution(executorSummaries.map {executionSummary => executionSummary.endTime.getOrElse(appAttemptInfo.endTime).getTime - executionSummary.addTime.getTime }) - lazy val totalTaskTime : Long = executorSummaries.map(_.totalDuration).sum + lazy val totalTaskTime : Long = executorSummaries.map(executionSummary => executionSummary.endTime.getOrElse(appAttemptInfo.endTime).getTime - executionSummary.addTime.getTime).sum lazy val taskTimeSeverity: Severity = severityOfDistribution(taskTimeDistribution, ignoreMaxMillisLessThanThreshold) diff --git a/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala b/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala index 4c0d6f727..4649129a1 100644 --- a/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala +++ b/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala @@ -200,6 +200,8 @@ object LegacyDataConverters { executorInfo.totalTasks, executorInfo.maxTasks, executorInfo.duration, + executorInfo.addTime, + Option(executorInfo.endTime), executorInfo.inputBytes, executorInfo.shuffleRead, executorInfo.shuffleWrite, diff --git a/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java b/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java index 50f27ba00..55331daad 100644 --- a/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java +++ b/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java @@ -16,9 +16,12 @@ package com.linkedin.drelephant.spark.legacydata; +import scala.None; + import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.Date; /** @@ -41,6 +44,8 @@ public static class ExecutorInfo { public int totalTasks = 0; public int maxTasks = 0; public long duration = 0L; + public Date addTime = new Date(); + public Date endTime = new Date(); public long inputBytes = 0L; public long outputBytes = 0L; public long shuffleRead = 0L; @@ -52,7 +57,7 @@ public String toString() { return "{execId: " + execId + ", hostPort:" + hostPort + " , rddBlocks: " + rddBlocks + ", memUsed: " + memUsed + ", maxMem: " + maxMem + ", diskUsed: " + diskUsed + ", totalTasks" + totalTasks + ", maxTasks" + maxTasks + ", tasksActive: " + activeTasks + ", tasksComplete: " + completedTasks + ", tasksFailed: " + failedTasks + ", duration: " - + duration + ", inputBytes: " + inputBytes + ", outputBytes:" + outputBytes + ", shuffleRead: " + shuffleRead + + duration + ", addTime: " + addTime + ", end time:" + endTime + ", inputBytes: " + inputBytes + ", outputBytes:" + outputBytes + ", shuffleRead: " + shuffleRead + ", shuffleWrite: " + shuffleWrite + ", totalGCTime: " + totalGCTime + ", totalMemoryBytesSpilled: " + totalMemoryBytesSpilled + "}"; } } diff --git a/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala b/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala index 70064494c..8d6a79024 100644 --- a/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala +++ b/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala @@ -17,6 +17,7 @@ package com.linkedin.drelephant.spark import java.util.Date +import java.util.Calendar import scala.collection.JavaConverters @@ -187,6 +188,8 @@ object SparkMetricsAggregatorTest { totalTasks = 0, maxTasks = 0, totalDuration, + addTime = Calendar.getInstance().getTime, + endTime = None, totalInputBytes = 0, totalShuffleRead = 0, totalShuffleWrite = 0, diff --git a/test/com/linkedin/drelephant/spark/heuristics/DriverHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/DriverHeuristicTest.scala index fe0da4ec6..a61256455 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/DriverHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/DriverHeuristicTest.scala @@ -9,6 +9,7 @@ import org.scalatest.{FunSpec, Matchers} import scala.collection.JavaConverters import scala.concurrent.duration.Duration +import java.util.Calendar /** * Test class for Driver Metrics Heuristic. It checks whether all the values used in the heuristic are calculated correctly. @@ -77,6 +78,8 @@ object DriverHeuristicTest { totalTasks = 0, maxTasks = 0, totalDuration, + addTime = Calendar.getInstance().getTime, + endTime = None, totalInputBytes = 0, totalShuffleRead = 0, totalShuffleWrite = 0, diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala index f60d453b2..2fa1dc39a 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala @@ -20,10 +20,10 @@ import scala.collection.JavaConverters import com.linkedin.drelephant.analysis.{ApplicationType, Severity, SeverityThresholds} import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl, StageDataImpl} +import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationAttemptInfoImpl, ApplicationInfoImpl, ExecutorSummaryImpl, StageDataImpl} import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} - +import java.util.{Calendar,Date} import scala.concurrent.duration.Duration /** @@ -35,27 +35,65 @@ class ExecutorGcHeuristicTest extends FunSpec with Matchers { describe("ExecutorGcHeuristic") { val heuristicConfigurationData = newFakeHeuristicConfigurationData() val executorGcHeuristic = new ExecutorGcHeuristic(heuristicConfigurationData) + val dateArray : Array[Date] = new Array[Date](20) + val cal: Calendar = Calendar.getInstance() + dateArray(0) = cal.getTime + cal.add(Calendar.MILLISECOND,700) + dateArray(1) = cal.getTime + cal.add(Calendar.MILLISECOND,300) + for(i <- 2 to 11) + { + dateArray(i) = cal.getTime + cal.add(Calendar.MINUTE,1) + } + for(i <- 12 to 19) + { + dateArray(i) = cal.getTime + cal.add(Calendar.MINUTE,5) + } + val appAttemptInfo = Seq( + newFakeApplicationAttemptInfo(endTime = dateArray(14)) + ) + + val appAttemptInfo1 = Seq( + newFakeApplicationAttemptInfo(endTime = dateArray(1)) + ) + + val appAttemptInfo2 = Seq( + newFakeApplicationAttemptInfo(endTime = dateArray(6)) + ) + val appAttemptInfo3 = Seq( + newFakeApplicationAttemptInfo(endTime = dateArray(9)) + ) val executorSummaries = Seq( newFakeExecutorSummary( id = "1", totalGCTime = Duration("2min").toMillis, - totalDuration = Duration("15min").toMillis + totalDuration = Duration("15min").toMillis, + addTime = dateArray(12), + endTime = Option(dateArray(15)) ), newFakeExecutorSummary( id = "2", totalGCTime = Duration("6min").toMillis, - totalDuration = Duration("14min").toMillis + totalDuration = Duration("14min").toMillis, + addTime = dateArray(8), + endTime = None ), newFakeExecutorSummary( id = "3", totalGCTime = Duration("4min").toMillis, - totalDuration = Duration("20min").toMillis - ), + totalDuration = Duration("20min").toMillis, + addTime = dateArray(2), + endTime = None + ), newFakeExecutorSummary( id = "4", totalGCTime = Duration("8min").toMillis, - totalDuration = Duration("30min").toMillis + totalDuration = Duration("30min").toMillis, + addTime = dateArray(13), + endTime = Option(dateArray(19)) ) ) @@ -63,7 +101,9 @@ class ExecutorGcHeuristicTest extends FunSpec with Matchers { newFakeExecutorSummary( id = "1", totalGCTime = 500, - totalDuration = 700 + totalDuration = 700, + addTime = dateArray(0), + endTime = Option(dateArray(1)) ) ) @@ -71,12 +111,16 @@ class ExecutorGcHeuristicTest extends FunSpec with Matchers { newFakeExecutorSummary( id = "1", totalGCTime = 12000, - totalDuration = Duration("4min").toMillis + totalDuration = Duration("4min").toMillis, + addTime = dateArray(2), + endTime = None ), newFakeExecutorSummary( id = "2", totalGCTime = 13000, - totalDuration = Duration("1min").toMillis + totalDuration = Duration("1min").toMillis, + addTime = dateArray(5), + endTime = None ) ) @@ -84,15 +128,17 @@ class ExecutorGcHeuristicTest extends FunSpec with Matchers { newFakeExecutorSummary( id = "1", totalGCTime = 9000, - totalDuration = Duration("2min").toMillis + totalDuration = Duration("2min").toMillis, + addTime = dateArray(7), + endTime = None ) ) describe(".apply") { - val data = newFakeSparkApplicationData(executorSummaries) - val data1 = newFakeSparkApplicationData(executorSummaries1) - val data2 = newFakeSparkApplicationData(executorSummaries2) - val data3 = newFakeSparkApplicationData(executorSummaries3) + val data = newFakeSparkApplicationData(executorSummaries, appAttemptInfo) + val data1 = newFakeSparkApplicationData(executorSummaries1, appAttemptInfo1) + val data2 = newFakeSparkApplicationData(executorSummaries2, appAttemptInfo2) + val data3 = newFakeSparkApplicationData(executorSummaries3, appAttemptInfo3) val heuristicResult = executorGcHeuristic.apply(data) val heuristicResult1 = executorGcHeuristic.apply(data1) val heuristicResult2 = executorGcHeuristic.apply(data2) @@ -160,7 +206,9 @@ object ExecutorGcHeuristicTest { def newFakeExecutorSummary( id: String, totalGCTime: Long, - totalDuration: Long + totalDuration: Long, + addTime: Date, + endTime: Option[Date] ): ExecutorSummaryImpl = new ExecutorSummaryImpl( id, hostPort = "", @@ -173,6 +221,8 @@ object ExecutorGcHeuristicTest { totalTasks = 0, maxTasks = 0, totalDuration, + addTime, + endTime, totalInputBytes=0, totalShuffleRead=0, totalShuffleWrite= 0, @@ -183,14 +233,23 @@ object ExecutorGcHeuristicTest { peakJvmUsedMemory = Map.empty, peakUnifiedMemory = Map.empty ) + def newFakeApplicationAttemptInfo( + endTime: Date + ): ApplicationAttemptInfoImpl = new ApplicationAttemptInfoImpl( + attemptId = Option("attemptId_1"), + startTime= Calendar.getInstance().getTime, + endTime, + sparkUser = "", + completed = true + ) def newFakeSparkApplicationData( - executorSummaries: Seq[ExecutorSummaryImpl] + executorSummaries: Seq[ExecutorSummaryImpl] , appAttemptInfo: Seq[ApplicationAttemptInfoImpl] ): SparkApplicationData = { val appId = "application_1" val restDerivedData = SparkRestDerivedData( - new ApplicationInfoImpl(appId, name = "app", Seq.empty), + new ApplicationInfoImpl(appId, name = "app", appAttemptInfo), jobDatas = Seq.empty, stageDatas = Seq.empty, executorSummaries = executorSummaries, diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala index e02e4f2fb..a0a8d802d 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorStorageSpillHeuristicTest.scala @@ -23,6 +23,7 @@ import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerived import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl, StageDataImpl} import com.linkedin.drelephant.spark.heuristics.ExecutorStorageSpillHeuristic.Evaluator import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate +import java.util.Calendar import org.scalatest.{FunSpec, Matchers} /** @@ -114,6 +115,8 @@ object ExecutorStorageSpillHeuristicTest { totalTasks = 0, maxTasks = 10, totalDuration=0, + addTime = Calendar.getInstance().getTime, + endTime = None, totalInputBytes=0, totalShuffleRead=0, totalShuffleWrite= 0, diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala index c1996fe41..ca20ca95d 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala @@ -21,9 +21,11 @@ import scala.collection.JavaConverters import com.linkedin.drelephant.analysis.{ApplicationType, Severity, SeverityThresholds} import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl} +import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationAttemptInfoImpl, +ApplicationInfoImpl, ExecutorSummaryImpl} import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} +import java.util.{Calendar,Date} /** * Test class for Executors Heuristic. It checks whether all the values used in the heuristic are calculated correctly. @@ -40,6 +42,20 @@ class ExecutorsHeuristicTest extends FunSpec with Matchers { ) ) val executorsHeuristic = new ExecutorsHeuristic(heuristicConfigurationData) + val dateArray : Array[Date] = new Array[Date](20) + val cal: Calendar = Calendar.getInstance() + dateArray(0) = cal.getTime + cal.add(Calendar.SECOND,1000) + cal.add(Calendar.MILLISECOND,1) + for(i <- 1 to 19) + { + + dateArray(i) = cal.getTime + cal.add(Calendar.SECOND,1000) + + } + val appAttemptInfo = Seq(newFakeApplicationAttemptInfo(endTime = dateArray(4))) + val maxMemory = 5000000L @@ -48,6 +64,8 @@ class ExecutorsHeuristicTest extends FunSpec with Matchers { id = "1", memoryUsed = 1000000L, totalDuration = 1000001L, + addTime = dateArray(0), + endTime = Option(dateArray(1)), totalInputBytes = 1000002L, totalShuffleRead = 1000003L, totalShuffleWrite = 1000004L, @@ -57,6 +75,8 @@ class ExecutorsHeuristicTest extends FunSpec with Matchers { id = "2", memoryUsed = 2000000L, totalDuration = 2000001L, + addTime = dateArray(0), + endTime = Option(dateArray(2)), totalInputBytes = 2000002L, totalShuffleRead = 2000003L, totalShuffleWrite = 2000004L, @@ -66,6 +86,8 @@ class ExecutorsHeuristicTest extends FunSpec with Matchers { id = "3", memoryUsed = 3000000L, totalDuration = 3000001L, + addTime = dateArray(0), + endTime = Option(dateArray(3)), totalInputBytes = 3000002L, totalShuffleRead = 3000003L, totalShuffleWrite = 3000004L, @@ -75,6 +97,8 @@ class ExecutorsHeuristicTest extends FunSpec with Matchers { id = "4", memoryUsed = 4000000L, totalDuration = 4000001L, + addTime = dateArray(0), + endTime = None, totalInputBytes = 4000002L, totalShuffleRead = 4000003L, totalShuffleWrite = 4000004L, @@ -83,7 +107,7 @@ class ExecutorsHeuristicTest extends FunSpec with Matchers { ) describe(".apply") { - val data = newFakeSparkApplicationData(executorSummaries) + val data = newFakeSparkApplicationData(executorSummaries,appAttemptInfo) val heuristicResult = executorsHeuristic.apply(data) val heuristicResultDetails = heuristicResult.getHeuristicResultDetails @@ -150,7 +174,7 @@ class ExecutorsHeuristicTest extends FunSpec with Matchers { import ExecutorsHeuristic.Evaluator import ExecutorsHeuristic.Distribution - val data = newFakeSparkApplicationData(executorSummaries) + val data = newFakeSparkApplicationData(executorSummaries,appAttemptInfo) val evaluator = new Evaluator(executorsHeuristic, data) it("has the total storage memory allocated") { @@ -232,6 +256,8 @@ object ExecutorsHeuristicTest { id: String, memoryUsed: Long, totalDuration: Long, + addTime: Date, + endTime: Option[Date], totalInputBytes: Long, totalShuffleRead: Long, totalShuffleWrite: Long, @@ -248,6 +274,8 @@ object ExecutorsHeuristicTest { totalTasks = 0, maxTasks = 0, totalDuration, + addTime, + endTime, totalInputBytes, totalShuffleRead, totalShuffleWrite, @@ -258,12 +286,21 @@ object ExecutorsHeuristicTest { peakJvmUsedMemory = Map.empty, peakUnifiedMemory = Map.empty ) + def newFakeApplicationAttemptInfo( + endTime: Date + ): ApplicationAttemptInfoImpl = new ApplicationAttemptInfoImpl( + attemptId = Option("ATTEMPTid_1"), + startTime= Calendar.getInstance().getTime, + endTime, + sparkUser = "", + completed = true + ) - def newFakeSparkApplicationData(executorSummaries: Seq[ExecutorSummaryImpl]): SparkApplicationData = { + def newFakeSparkApplicationData(executorSummaries: Seq[ExecutorSummaryImpl],appAttemptInfo: Seq[ApplicationAttemptInfoImpl]): SparkApplicationData = { val appId = "application_1" val restDerivedData = SparkRestDerivedData( - new ApplicationInfoImpl(appId, name = "app", Seq.empty), + new ApplicationInfoImpl(appId, name = "app", appAttemptInfo), jobDatas = Seq.empty, stageDatas = Seq.empty, executorSummaries = executorSummaries, diff --git a/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala index e0dc6641f..b1a42e74d 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristicTest.scala @@ -6,6 +6,7 @@ import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerived import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl} import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} +import java.util.Calendar import scala.collection.JavaConverters @@ -90,6 +91,8 @@ object JvmUsedMemoryHeuristicTest { totalTasks = 0, maxTasks = 0, totalDuration = 0, + addTime=Calendar.getInstance().getTime, + endTime=None, totalInputBytes = 0, totalShuffleRead = 0, totalShuffleWrite = 0, diff --git a/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala index 5677e3ffa..ba0bea048 100644 --- a/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala +++ b/test/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristicTest.scala @@ -7,6 +7,7 @@ import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, import com.linkedin.drelephant.spark.heuristics.UnifiedMemoryHeuristic.Evaluator import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate import org.scalatest.{FunSpec, Matchers} +import java.util.Calendar import scala.collection.JavaConverters @@ -134,6 +135,8 @@ object UnifiedMemoryHeuristicTest { totalTasks = 0, maxTasks = 0, totalDuration = 0, + addTime= Calendar.getInstance().getTime, + endTime= None, totalInputBytes = 0, totalShuffleRead = 0, totalShuffleWrite = 0, From e03fcfbad3abba1ec5837c1f5956791824b98b55 Mon Sep 17 00:00:00 2001 From: Himani Jha Date: Mon, 24 Sep 2018 01:25:04 -0700 Subject: [PATCH 14/15] Indent changes --- app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala | 2 -- app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala | 1 - .../drelephant/spark/legacydata/LegacyDataConverters.scala | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala b/app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala index be6f072a6..c8216cdf4 100644 --- a/app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala +++ b/app/com/linkedin/drelephant/spark/SparkMetricsAggregator.scala @@ -1,4 +1,3 @@ -/* /* * Copyright 2016 LinkedIn Corp. * @@ -14,7 +13,6 @@ * License for the specific language governing permissions and limitations under * the License. */ - * */ package com.linkedin.drelephant.spark diff --git a/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala b/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala index 823b5a6c2..a00f5d5b4 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala @@ -112,7 +112,6 @@ class SparkFetcher(fetcherConfigurationData: FetcherConfigurationData) private def doFetchDataUsingRestAndLogClients(analyticJob: AnalyticJob): Future[SparkApplicationData] = Future { val appId = analyticJob.getAppId val restDerivedData = Await.result(sparkRestClient.fetchData(appId, eventLogSource == EventLogSource.Rest), DEFAULT_TIMEOUT) - val logDerivedData = eventLogSource match { case EventLogSource.None => None case EventLogSource.Rest => restDerivedData.logDerivedData diff --git a/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala b/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala index 4649129a1..155dfb562 100644 --- a/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala +++ b/app/com/linkedin/drelephant/spark/legacydata/LegacyDataConverters.scala @@ -201,7 +201,7 @@ object LegacyDataConverters { executorInfo.maxTasks, executorInfo.duration, executorInfo.addTime, - Option(executorInfo.endTime), + Option(executorInfo.endTime), executorInfo.inputBytes, executorInfo.shuffleRead, executorInfo.shuffleWrite, From 39793e0a07b44b255c2995c67b92e4e03a3b08c7 Mon Sep 17 00:00:00 2001 From: hjha1 <42731065+hjha1@users.noreply.github.com> Date: Mon, 24 Sep 2018 01:27:34 -0700 Subject: [PATCH 15/15] Update SparkExecutorData.java --- .../linkedin/drelephant/spark/legacydata/SparkExecutorData.java | 1 - 1 file changed, 1 deletion(-) diff --git a/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java b/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java index 55331daad..313c58fd8 100644 --- a/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java +++ b/app/com/linkedin/drelephant/spark/legacydata/SparkExecutorData.java @@ -17,7 +17,6 @@ package com.linkedin.drelephant.spark.legacydata; import scala.None; - import java.util.HashMap; import java.util.Map; import java.util.Set;