diff --git a/app-conf/AggregatorConf.xml b/app-conf/AggregatorConf.xml index 3fecc3300..c9564185b 100644 --- a/app-conf/AggregatorConf.xml +++ b/app-conf/AggregatorConf.xml @@ -29,14 +29,14 @@ --> - - mapreduce - com.linkedin.drelephant.mapreduce.MapReduceMetricsAggregator - - - tez - com.linkedin.drelephant.tez.TezMetricsAggregator - + + + + + + + + spark com.linkedin.drelephant.spark.SparkMetricsAggregator @@ -44,8 +44,8 @@ 0.5 - - tony - com.linkedin.drelephant.tony.TonyMetricsAggregator - + + + + diff --git a/app-conf/FetcherConf.xml b/app-conf/FetcherConf.xml index 1f5e49308..2bf8f0c2d 100644 --- a/app-conf/FetcherConf.xml +++ b/app-conf/FetcherConf.xml @@ -32,10 +32,10 @@ - - tez - com.linkedin.drelephant.tez.fetchers.TezFetcher - + + + + - + + - - spark - com.linkedin.drelephant.spark.fetchers.FSFetcher - + + + + spark com.linkedin.drelephant.spark.fetchers.SparkFetcher @@ -115,7 +116,7 @@ true - --> + - - - - tez - Tez Mapper Data Skew - com.linkedin.drelephant.tez.heuristics.MapperDataSkewHeuristic - views.html.help.tez.helpMapperDataSkew - - - - - tez - Tez Mapper GC - com.linkedin.drelephant.tez.heuristics.MapperGCHeuristic - views.html.help.tez.helpGC - - - - - tez - Tez Mapper Time - com.linkedin.drelephant.tez.heuristics.MapperTimeHeuristic - views.html.help.tez.helpMapperTime - - - - - tez - Tez Mapper Speed - com.linkedin.drelephant.tez.heuristics.MapperSpeedHeuristic - views.html.help.tez.helpMapperSpeed - - - - - tez - Tez Mapper Memory - com.linkedin.drelephant.tez.heuristics.MapperMemoryHeuristic - views.html.help.tez.helpMemory - - - - - tez - Tez Mapper Spill - com.linkedin.drelephant.tez.heuristics.MapperSpillHeuristic - views.html.help.tez.helpMapperSpill - - - - - tez - Tez Reducer Data Skew - com.linkedin.drelephant.tez.heuristics.ReducerDataSkewHeuristic - views.html.help.mapreduce.helpReducerSkew - - - - - tez - Tez Reducer GC - com.linkedin.drelephant.tez.heuristics.ReducerGCHeuristic - views.html.help.mapreduce.helpGC - - - - - tez - Tez Reducer Time - com.linkedin.drelephant.tez.heuristics.ReducerTimeHeuristic - views.html.help.tez.helpReducerTime - - - - - tez - Scope Memory - com.linkedin.drelephant.tez.heuristics.TezScopeMemoryHeuristic - views.html.help.tez.helpMemory - - - - tez - Scope Data Skew - com.linkedin.drelephant.tez.heuristics.TezScopeDataSkewHeuristic - views.html.help.tez.helpScopeTaskDataSkew - - - - tez - Scope GC - com.linkedin.drelephant.tez.heuristics.TezScopeGCHeuristic - views.html.help.tez.helpGC - - - - tez - Scope Speed - com.linkedin.drelephant.tez.heuristics.TezScopeSpeedHeuristic - views.html.help.tez.helpMapperSpeed - - - - - tez - Scope Time - com.linkedin.drelephant.tez.heuristics.TezScopeTimeHeuristic - views.html.help.tez.helpScopeTaskTime - - - - - - - mapreduce - Mapper Skew - com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic - views.html.help.mapreduce.helpMapperSkew - - - - - mapreduce - Mapper GC - com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic - views.html.help.mapreduce.helpGC - - - - - mapreduce - Mapper Time - com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic - views.html.help.mapreduce.helpMapperTime - - - - - mapreduce - Mapper Speed - com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic - views.html.help.mapreduce.helpMapperSpeed - - - - - mapreduce - Mapper Spill - com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic - views.html.help.mapreduce.helpMapperSpill - - - - - mapreduce - Mapper Memory - com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic - views.html.help.mapreduce.helpMapperMemory - - - - - mapreduce - Reducer Skew - com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic - views.html.help.mapreduce.helpReducerSkew - - - - - mapreduce - Reducer GC - com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic - views.html.help.mapreduce.helpGC - - - - - mapreduce - Reducer Time - com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic - views.html.help.mapreduce.helpReducerTime - - - - - mapreduce - Reducer Memory - com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic - views.html.help.mapreduce.helpReducerMemory - - - - - mapreduce - Shuffle & Sort - com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic - views.html.help.mapreduce.helpShuffleSort - - - - - mapreduce - Exception - com.linkedin.drelephant.mapreduce.heuristics.ExceptionHeuristic - views.html.help.mapreduce.helpException - - - - mapreduce - Distributed Cache Limit - com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic - views.html.help.mapreduce.helpDistributedCacheLimit - - 500000000 - - - - @@ -354,26 +50,4 @@ views.html.help.spark.helpExecutorGcHeuristic - - - - tony - Task Memory - com.linkedin.drelephant.tony.heuristics.TaskMemoryHeuristic - views.html.help.tony.helpTaskMemory - - - - - - - - - tony - Task GPU - com.linkedin.drelephant.tony.heuristics.TaskGPUHeuristic - views.html.help.tony.helpTaskGPU - - - diff --git a/test/resources/HeuristicConf.xml b/app-conf/HeuristicConf_bak.xml similarity index 54% rename from test/resources/HeuristicConf.xml rename to app-conf/HeuristicConf_bak.xml index 3e264e853..c14f85556 100644 --- a/test/resources/HeuristicConf.xml +++ b/app-conf/HeuristicConf_bak.xml @@ -17,6 +17,160 @@ + + + + tez + Tez Mapper Data Skew + com.linkedin.drelephant.tez.heuristics.MapperDataSkewHeuristic + views.html.help.tez.helpMapperDataSkew + + + + + tez + Tez Mapper GC + com.linkedin.drelephant.tez.heuristics.MapperGCHeuristic + views.html.help.tez.helpGC + + + + + tez + Tez Mapper Time + com.linkedin.drelephant.tez.heuristics.MapperTimeHeuristic + views.html.help.tez.helpMapperTime + + + + + tez + Tez Mapper Speed + com.linkedin.drelephant.tez.heuristics.MapperSpeedHeuristic + views.html.help.tez.helpMapperSpeed + + + + + tez + Tez Mapper Memory + com.linkedin.drelephant.tez.heuristics.MapperMemoryHeuristic + views.html.help.tez.helpMemory + + + + + tez + Tez Mapper Spill + com.linkedin.drelephant.tez.heuristics.MapperSpillHeuristic + views.html.help.tez.helpMapperSpill + + + + + tez + Tez Reducer Data Skew + com.linkedin.drelephant.tez.heuristics.ReducerDataSkewHeuristic + views.html.help.mapreduce.helpReducerSkew + + + + + tez + Tez Reducer GC + com.linkedin.drelephant.tez.heuristics.ReducerGCHeuristic + views.html.help.mapreduce.helpGC + + + + + tez + Tez Reducer Time + com.linkedin.drelephant.tez.heuristics.ReducerTimeHeuristic + views.html.help.tez.helpReducerTime + + + + + tez + Scope Memory + com.linkedin.drelephant.tez.heuristics.TezScopeMemoryHeuristic + views.html.help.tez.helpMemory + + + + tez + Scope Data Skew + com.linkedin.drelephant.tez.heuristics.TezScopeDataSkewHeuristic + views.html.help.tez.helpScopeTaskDataSkew + + + + tez + Scope GC + com.linkedin.drelephant.tez.heuristics.TezScopeGCHeuristic + views.html.help.tez.helpGC + + + + tez + Scope Speed + com.linkedin.drelephant.tez.heuristics.TezScopeSpeedHeuristic + views.html.help.tez.helpMapperSpeed + + + + + tez + Scope Time + com.linkedin.drelephant.tez.heuristics.TezScopeTimeHeuristic + views.html.help.tez.helpScopeTaskTime + + + @@ -200,4 +354,26 @@ views.html.help.spark.helpExecutorGcHeuristic + + + + tony + Task Memory + com.linkedin.drelephant.tony.heuristics.TaskMemoryHeuristic + views.html.help.tony.helpTaskMemory + + + + + + + + + tony + Task GPU + com.linkedin.drelephant.tony.heuristics.TaskGPUHeuristic + views.html.help.tony.helpTaskGPU + + + diff --git a/app-conf/elephant.conf b/app-conf/elephant.conf index bc65ddf02..b6fbe92b0 100644 --- a/app-conf/elephant.conf +++ b/app-conf/elephant.conf @@ -1,5 +1,5 @@ # Play application server port -http_port=8080 +http_port=8720 # Un-comment these configs if need to enable SSL for Dr.Elephant #https_port=8090 @@ -15,10 +15,10 @@ http_port=8080 # application_secret="changeme" # Database configuration -db_url=localhost +db_url=172.22.180.19 db_name=drelephant -db_user=root -db_password="" +db_user=salt +db_password=salt # Enable web analytics for the application. # By default analytics is not turned on. Set this property @@ -35,7 +35,8 @@ enable_analytics=false # Check https://www.playframework.com/documentation/2.2.x/ProductionConfiguration # Adding the below line for Heap Tuning and Java OPTS # Use mem for tuning Heap Memory -jvm_args="-Devolutionplugin=enabled -DapplyEvolutions.default=true -mem 1024 -J-Xloggc:$project_root../logs/elephant/dr-gc.`date +'%Y%m%d%H%M'` -J-XX:+PrintGCDetails" +#jvm_args="-Devolutionplugin=enabled -DapplyEvolutions.default=true -mem 1024 -J-Xloggc:$project_root../logs/elephant/dr-gc.`date +'%Y%m%d%H%M'` -J-XX:+PrintGCDetails" +jvm_args="-Devolutionplugin=enabled -Dplay.evolutions.db.default.autoApply=true -mem 1024 -J-Xloggc:$project_root../logs/elephant/dr-gc.`date +'%Y%m%d%H%M'` -J-XX:+PrintGCDetails" # Property enables dropwizard metrics for the application. diff --git a/app-conf/resolver.conf b/app-conf/resolver.conf new file mode 100644 index 000000000..247f5246d --- /dev/null +++ b/app-conf/resolver.conf @@ -0,0 +1,8 @@ +[repositories] + local + maven-central: https://repo1.maven.org/maven2/ + play-central: https://repo1.maven.org/maven2/com/typesafe/ + tysafe-maven: https://repo.typesafe.com/typesafe/maven-releases/ + sbt-plugin-repo: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + typesafe: http://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + \ No newline at end of file diff --git a/app/com/linkedin/drelephant/ElephantContext.java b/app/com/linkedin/drelephant/ElephantContext.java index c83866f4f..fe4e93b43 100644 --- a/app/com/linkedin/drelephant/ElephantContext.java +++ b/app/com/linkedin/drelephant/ElephantContext.java @@ -49,7 +49,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; import org.w3c.dom.Document; -import play.api.templates.Html; +//import play.api.templates.Html; +import play.twirl.api.Html; /** diff --git a/app/com/linkedin/drelephant/ElephantRunner.java b/app/com/linkedin/drelephant/ElephantRunner.java index 55e6a8fe2..2518add83 100644 --- a/app/com/linkedin/drelephant/ElephantRunner.java +++ b/app/com/linkedin/drelephant/ElephantRunner.java @@ -20,6 +20,7 @@ import com.avaje.ebean.TxRunnable; import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.gson.Gson; import com.linkedin.drelephant.analysis.AnalyticJob; import com.linkedin.drelephant.analysis.AnalyticJobGenerator; import com.linkedin.drelephant.analysis.AnalyticJobGeneratorHadoop2; @@ -188,6 +189,7 @@ public Void run() { List todos; try { todos = _analyticJobGenerator.fetchAnalyticJobs(); + logger.info("return todos size :" + todos==null?0:todos.size()); } catch (Exception e) { logger.error("Error fetching job list. Try again later...", e); //Wait for a while before retry @@ -386,6 +388,7 @@ public void run() { long applicableFinishTime = -1; long jobFinishTime = -1; FinishTimeInfo finishTimeInfo = null; + Gson gson = new Gson(); try { final AppResult result = _analyticJob.getAnalysis(); applicableFinishTime = getApplicableFinishTime(_analyticJob); @@ -396,6 +399,7 @@ public void run() { // Execute as a transaction. Ebean.execute(new TxRunnable() { public void run() { + logger.info(String.format("分析结果存储%s:%s", result.id, gson.toJson(result))); result.save(); if (backfillInfo != null) { backfillInfo.save(); diff --git a/app/com/linkedin/drelephant/analysis/AnalyticJob.java b/app/com/linkedin/drelephant/analysis/AnalyticJob.java index b7aec1a98..fb41a56d8 100644 --- a/app/com/linkedin/drelephant/analysis/AnalyticJob.java +++ b/app/com/linkedin/drelephant/analysis/AnalyticJob.java @@ -317,9 +317,11 @@ public AppResult getAnalysis() throws Exception { logger.info("No Data Received for analytic job: " + getAppId()); analysisResults.add(HeuristicResult.NO_DATA); } else { + logger.info("getHeuristicsForApplicationType"); List heuristics = ElephantContext.instance().getHeuristicsForApplicationType(getAppType()); for (Heuristic heuristic : heuristics) { String confExcludedApps = heuristic.getHeuristicConfData().getParamMap().get(EXCLUDE_JOBTYPE); + logger.info("confExcludedApps: " + confExcludedApps + " cls:" + heuristic.getHeuristicConfData().getClassName()); if (confExcludedApps == null || confExcludedApps.length() == 0 || !Arrays.asList(confExcludedApps.split(",")).contains(jobTypeName)) { diff --git a/app/com/linkedin/drelephant/analysis/AnalyticJobGeneratorHadoop2.java b/app/com/linkedin/drelephant/analysis/AnalyticJobGeneratorHadoop2.java index 69489812f..fa128f505 100644 --- a/app/com/linkedin/drelephant/analysis/AnalyticJobGeneratorHadoop2.java +++ b/app/com/linkedin/drelephant/analysis/AnalyticJobGeneratorHadoop2.java @@ -69,8 +69,10 @@ public class AnalyticJobGeneratorHadoop2 implements AnalyticJobGenerator { private final List _secondRetryQueue = new LinkedList(); public void updateResourceManagerAddresses() { + logger.info("configuration.get(IS_RM_HA_ENABLED):" + configuration.get(IS_RM_HA_ENABLED)); if (Boolean.valueOf(configuration.get(IS_RM_HA_ENABLED))) { String resourceManagers = configuration.get(RESOURCE_MANAGER_IDS); + logger.info("updateResourceManagerAddresses_resourceManagers: " + resourceManagers); if (resourceManagers != null) { logger.info("The list of RM IDs are " + resourceManagers); List ids = Arrays.asList(resourceManagers.split(",")); @@ -144,6 +146,7 @@ public List fetchAnalyticJobs() // There is a lag of job data from AM/NM to JobHistoryServer HDFS, we shouldn't use the current time, since there // might be new jobs arriving after we fetch jobs. We provide one minute delay to address this lag. _currentTime = System.currentTimeMillis() - FETCH_DELAY; + _lastTime = _currentTime - FETCH_DELAY; updateAuthToken(); logger.info("Fetching recent finished application runs between last time: " + (_lastTime + 1) diff --git a/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala b/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala index 7ee5b47c7..b6bfcebab 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/SparkFetcher.scala @@ -189,6 +189,6 @@ object SparkFetcher { } val SPARK_EVENT_LOG_ENABLED_KEY = "spark.eventLog.enabled" - val DEFAULT_TIMEOUT = Duration(5, SECONDS) + val DEFAULT_TIMEOUT = Duration(30, SECONDS) //默认5s，暂时修改成30s val LOG_LOCATION_URI_XML_FIELD = "event_log_location_uri" } diff --git a/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala b/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala index abeac17ab..e056d8fe0 100644 --- a/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala +++ b/app/com/linkedin/drelephant/spark/fetchers/SparkRestClient.scala @@ -285,9 +285,9 @@ object SparkRestClient { val HISTORY_SERVER_ADDRESS_KEY = "spark.yarn.historyServer.address" val API_V1_MOUNT_PATH = "api/v1" val IN_PROGRESS = ".inprogress" - val DEFAULT_TIMEOUT = Duration(5, SECONDS); - val CONNECTION_TIMEOUT = 5000 - val READ_TIMEOUT = 5000 + val DEFAULT_TIMEOUT = Duration(10, SECONDS); + val CONNECTION_TIMEOUT = 10000 + val READ_TIMEOUT = 10000 val SparkRestObjectMapper = { val dateFormat = { diff --git a/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala index 0c0193ef5..c7f3d9701 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala @@ -148,7 +148,7 @@ object ConfigurationHeuristic { val SPARK_YARN_JARS = "spark.yarn.secondary.jars" val SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD = "spark.yarn.executor.memoryOverhead" val SPARK_YARN_DRIVER_MEMORY_OVERHEAD = "spark.yarn.driver.memoryOverhead" - val THRESHOLD_MIN_EXECUTORS: Int = 1 + val THRESHOLD_MIN_EXECUTORS: Int = 2 //默认1 val THRESHOLD_MAX_EXECUTORS: Int = 900 val SPARK_OVERHEAD_MEMORY_THRESHOLD_KEY = "spark.overheadMemory.thresholds.key" val DEFAULT_SPARK_OVERHEAD_MEMORY_THRESHOLDS = @@ -158,6 +158,9 @@ object ConfigurationHeuristic { class Evaluator(configurationHeuristic: ConfigurationHeuristic, data: SparkApplicationData) { lazy val appConfigurationProperties: Map[String, String] = data.appConfigurationProperties + println("start====================") + appConfigurationProperties.foreach(x => println(x._1, x._2)) + println("end====================") lazy val driverMemoryBytes: Option[Long] = Try(getProperty(SPARK_DRIVER_MEMORY_KEY).map(MemoryFormatUtils.stringToBytes)).getOrElse(None) @@ -187,6 +190,7 @@ object ConfigurationHeuristic { } lazy val sparkYarnJars: String = getProperty(SPARK_YARN_JARS).getOrElse("") + println("sparkYarnJars=" + sparkYarnJars) lazy val jarsSeverity: Severity = if (sparkYarnJars.contains("*")) { Severity.CRITICAL @@ -235,6 +239,8 @@ object ConfigurationHeuristic { val severityExecutorMemoryOverhead = configurationHeuristic.sparkOverheadMemoryThreshold.severityOf(MemoryFormatUtils.stringToBytes(sparkYarnExecutorMemoryOverhead)) val severityDriverMemoryOverhead = configurationHeuristic.sparkOverheadMemoryThreshold.severityOf(MemoryFormatUtils.stringToBytes(sparkYarnDriverMemoryOverhead)) + println("sparkYarnExecutorMemoryOverhead=" + MemoryFormatUtils.stringToBytes(sparkYarnExecutorMemoryOverhead) + " severityExecutorMemoryOverhead=" + severityExecutorMemoryOverhead) + println("sparkYarnDriverMemoryOverhead=" + MemoryFormatUtils.stringToBytes(sparkYarnDriverMemoryOverhead) + " severityDriverMemoryOverhead=" + severityDriverMemoryOverhead) //Severity for the configuration thresholds val severityConfThresholds: Severity = Severity.max(severityDriverCores, severityDriverMemory, severityExecutorCores, severityExecutorMemory, diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala index dae604124..6088cc7f2 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristic.scala @@ -214,6 +214,8 @@ object ExecutorsHeuristic { private lazy val ignoreMaxBytesLessThanThreshold = executorsHeuristic.ignoreMaxBytesLessThanThreshold private lazy val ignoreMaxMillisLessThanThreshold = executorsHeuristic.ignoreMaxMillisLessThanThreshold + + println( "maxToMedianRatioSeverityThresholds=" + maxToMedianRatioSeverityThresholds) } case class Distribution(min: Long, p25: Long, median: Long, p75: Long, max: Long) diff --git a/app/com/linkedin/drelephant/spark/legacydata/SparkStorageData.java b/app/com/linkedin/drelephant/spark/legacydata/SparkStorageData.java index 0145848a3..5151c80e0 100644 --- a/app/com/linkedin/drelephant/spark/legacydata/SparkStorageData.java +++ b/app/com/linkedin/drelephant/spark/legacydata/SparkStorageData.java @@ -17,8 +17,10 @@ package com.linkedin.drelephant.spark.legacydata; import java.util.List; + +import org.apache.spark.status.api.v1.RDDStorageInfo; import org.apache.spark.storage.RDDInfo; -import org.apache.spark.storage.StorageStatus; +//import org.apache.spark.storage.StorageStatus; /** @@ -26,7 +28,8 @@ */ public class SparkStorageData { private List _rddInfoList; - private List _storageStatusList; + private List _rddStorageInfoList; +// private List _storageStatusList; public List getRddInfoList() { return _rddInfoList; @@ -36,11 +39,19 @@ public void setRddInfoList(List rddInfoList) { _rddInfoList = rddInfoList; } - public List getStorageStatusList() { - return _storageStatusList; +// public List getStorageStatusList() { +// return _storageStatusList; +// } +// +// public void setStorageStatusList(List storageStatusList) { +// _storageStatusList = storageStatusList; +// } + + public List getRddStorageInfoList() { + return _rddStorageInfoList; } - public void setStorageStatusList(List storageStatusList) { - _storageStatusList = storageStatusList; + public void setRddStorageInfoList(List rddStorageInfoList) { + this._rddStorageInfoList = rddStorageInfoList; } } diff --git a/app/com/linkedin/drelephant/tony/TonyMetricsAggregator.java b/app/com/linkedin/drelephant/tony/TonyMetricsAggregator.java index 540e54d22..d5ec39d65 100644 --- a/app/com/linkedin/drelephant/tony/TonyMetricsAggregator.java +++ b/app/com/linkedin/drelephant/tony/TonyMetricsAggregator.java @@ -66,7 +66,7 @@ public void aggregate(HadoopApplicationData data) { for (TonyTaskData taskData : entry.getValue().values()) { long taskDurationSec = (taskData.getTaskEndTime() - taskData.getTaskStartTime()) / Statistics.SECOND_IN_MS; if (taskDurationSec < 0) { - // Most likely TASK_FINISHED and APPLICATION_FINISHED events are missing for the task. + // Most likely TASK_FINISHED and APPLICATION_FINISHED events.scala are missing for the task. continue; } mbSecUsed += mbRequested * taskDurationSec; diff --git a/app/com/linkedin/drelephant/tony/data/TonyApplicationData.java b/app/com/linkedin/drelephant/tony/data/TonyApplicationData.java index 7021da0a5..a450b71b2 100644 --- a/app/com/linkedin/drelephant/tony/data/TonyApplicationData.java +++ b/app/com/linkedin/drelephant/tony/data/TonyApplicationData.java @@ -40,7 +40,7 @@ public class TonyApplicationData implements HadoopApplicationData { * @param appId application id * @param appType application type (should be TONY) * @param configuration the configuration for this application - * @param events the events emitted by this application + * @param events the events.scala emitted by this application */ public TonyApplicationData(String appId, ApplicationType appType, Configuration configuration, List events) { _appId = appId; diff --git a/app/com/linkedin/drelephant/tony/fetchers/TonyFetcher.java b/app/com/linkedin/drelephant/tony/fetchers/TonyFetcher.java index 82e271bb0..a7d580ee0 100644 --- a/app/com/linkedin/drelephant/tony/fetchers/TonyFetcher.java +++ b/app/com/linkedin/drelephant/tony/fetchers/TonyFetcher.java @@ -99,7 +99,7 @@ public TonyApplicationData fetchData(AnalyticJob job) throws Exception { conf.addResource(_fs.open(confFile)); } - // Parse events. For a list of event types, see + // Parse events.scala. For a list of event types, see // https://github.com/linkedin/TonY/blob/master/tony-core/src/main/avro/EventType.avsc. // We get the task start time from the TASK_STARTED event and the finish time and metrics from the TASK_FINISHED // event. diff --git a/app/com/linkedin/drelephant/util/Utils.java b/app/com/linkedin/drelephant/util/Utils.java index f88d17c46..541665d6c 100644 --- a/app/com/linkedin/drelephant/util/Utils.java +++ b/app/com/linkedin/drelephant/util/Utils.java @@ -157,6 +157,7 @@ public static Map parseJavaOptions(String str) { public static double[] getParam(String rawLimits, int thresholdLevels) { double[] parsedLimits = null; + logger.info(String.format("rawLimits=%s, thresholdLevels=%s", rawLimits, thresholdLevels)); if (rawLimits != null && !rawLimits.isEmpty()) { String[] thresholds = rawLimits.split(","); if (thresholds.length != thresholdLevels) { @@ -177,6 +178,7 @@ public static double[] getParam(String rawLimits, int thresholdLevels) { } } } + logger.info(String.format("parsedLimits=%s", parsedLimits==null?"":parsedLimits.toString())); return parsedLimits; } @@ -258,7 +260,8 @@ public static Map parseCsKeyValue(String str) { */ public static String truncateField(String field, int limit, String context) { if (field != null && limit > TRUNCATE_SUFFIX.length() && field.length() > limit) { - logger.info("Truncating " + field + " to " + limit + " characters for " + context); +// logger.info("Truncating " + field + " to " + limit + " characters for " + context); + logger.info("Truncating " + limit + " characters for " + context); field = field.substring(0, limit - 3) + "..."; } return field; diff --git a/app/controllers/Application.java b/app/controllers/Application.java index 9e94539eb..b59135023 100644 --- a/app/controllers/Application.java +++ b/app/controllers/Application.java @@ -50,7 +50,8 @@ import org.apache.http.message.BasicNameValuePair; import org.apache.log4j.Logger; -import play.api.templates.Html; +//import play.api.templates.Html; +import play.twirl.api.Html; import play.data.DynamicForm; import play.data.Form; import play.libs.Json; diff --git a/app/models/AppHeuristicResult.java b/app/models/AppHeuristicResult.java index 96db29e19..5d45d5179 100644 --- a/app/models/AppHeuristicResult.java +++ b/app/models/AppHeuristicResult.java @@ -33,7 +33,7 @@ import com.linkedin.drelephant.analysis.Severity; import com.linkedin.drelephant.util.Utils; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/AppHeuristicResultDetails.java b/app/models/AppHeuristicResultDetails.java index 3af96cb24..ae8704204 100644 --- a/app/models/AppHeuristicResultDetails.java +++ b/app/models/AppHeuristicResultDetails.java @@ -28,8 +28,7 @@ import com.fasterxml.jackson.annotation.JsonBackReference; -import play.db.ebean.Model; - +import com.avaje.ebean.Model; @Entity @Table(name = "yarn_app_heuristic_result_details") diff --git a/app/models/AppResult.java b/app/models/AppResult.java index cca925007..1ff2905e9 100644 --- a/app/models/AppResult.java +++ b/app/models/AppResult.java @@ -16,11 +16,11 @@ package models; +import com.avaje.ebean.Model; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.linkedin.drelephant.analysis.Severity; import com.linkedin.drelephant.util.Utils; -import play.db.ebean.Model; import java.util.List; diff --git a/app/models/BackfillInfo.java b/app/models/BackfillInfo.java index 98bc5c97f..13d3336fc 100644 --- a/app/models/BackfillInfo.java +++ b/app/models/BackfillInfo.java @@ -21,7 +21,7 @@ import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.Table; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity @Table(name = "backfill_info") diff --git a/app/models/FlowDefinition.java b/app/models/FlowDefinition.java index d01a347b1..101220bea 100644 --- a/app/models/FlowDefinition.java +++ b/app/models/FlowDefinition.java @@ -25,7 +25,7 @@ import javax.persistence.Id; import javax.persistence.Table; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/FlowExecution.java b/app/models/FlowExecution.java index 0bcee835f..0443c83dc 100644 --- a/app/models/FlowExecution.java +++ b/app/models/FlowExecution.java @@ -29,7 +29,7 @@ import javax.persistence.ManyToOne; import javax.persistence.Table; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/JobDefinition.java b/app/models/JobDefinition.java index 387b7ff12..c2f05126f 100644 --- a/app/models/JobDefinition.java +++ b/app/models/JobDefinition.java @@ -31,7 +31,7 @@ import com.avaje.ebean.annotation.UpdatedTimestamp; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/JobExecution.java b/app/models/JobExecution.java index 2f67f0c88..f2289128b 100644 --- a/app/models/JobExecution.java +++ b/app/models/JobExecution.java @@ -33,7 +33,7 @@ import com.avaje.ebean.annotation.UpdatedTimestamp; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/JobSavedState.java b/app/models/JobSavedState.java index f7ca624ae..f0d055870 100644 --- a/app/models/JobSavedState.java +++ b/app/models/JobSavedState.java @@ -20,7 +20,7 @@ import javax.persistence.Column; -import play.db.ebean.Model; +import com.avaje.ebean.Model; import javax.persistence.Entity; import javax.persistence.Id; diff --git a/app/models/JobSuggestedParamSet.java b/app/models/JobSuggestedParamSet.java index bad279932..f3dc21c4a 100644 --- a/app/models/JobSuggestedParamSet.java +++ b/app/models/JobSuggestedParamSet.java @@ -32,7 +32,7 @@ import javax.persistence.OneToOne; import javax.persistence.Table; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/JobSuggestedParamValue.java b/app/models/JobSuggestedParamValue.java index d78ba21ba..23420920f 100644 --- a/app/models/JobSuggestedParamValue.java +++ b/app/models/JobSuggestedParamValue.java @@ -30,7 +30,7 @@ import javax.persistence.ManyToOne; import javax.persistence.Table; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/TuningAlgorithm.java b/app/models/TuningAlgorithm.java index 69b94c9b7..601612c7b 100644 --- a/app/models/TuningAlgorithm.java +++ b/app/models/TuningAlgorithm.java @@ -29,7 +29,7 @@ import com.avaje.ebean.annotation.UpdatedTimestamp; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/TuningJobDefinition.java b/app/models/TuningJobDefinition.java index 27c7c61d3..0908d8c47 100644 --- a/app/models/TuningJobDefinition.java +++ b/app/models/TuningJobDefinition.java @@ -28,7 +28,7 @@ import com.avaje.ebean.annotation.UpdatedTimestamp; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/models/TuningJobExecutionParamSet.java b/app/models/TuningJobExecutionParamSet.java index 1be048e17..4a40b47fe 100644 --- a/app/models/TuningJobExecutionParamSet.java +++ b/app/models/TuningJobExecutionParamSet.java @@ -25,7 +25,7 @@ import javax.persistence.JoinTable; import javax.persistence.OneToOne; import javax.persistence.Table; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity @Table(name = "tuning_job_execution_param_set") diff --git a/app/models/TuningParameter.java b/app/models/TuningParameter.java index 7ecce1acc..7097d9c01 100644 --- a/app/models/TuningParameter.java +++ b/app/models/TuningParameter.java @@ -31,7 +31,7 @@ import com.avaje.ebean.annotation.UpdatedTimestamp; -import play.db.ebean.Model; +import com.avaje.ebean.Model; @Entity diff --git a/app/org/apache/spark/deploy/history/SparkDataCollection.scala b/app/org/apache/spark/deploy/history/SparkDataCollection.scala index f60fcfa19..d0d2494ec 100644 --- a/app/org/apache/spark/deploy/history/SparkDataCollection.scala +++ b/app/org/apache/spark/deploy/history/SparkDataCollection.scala @@ -16,23 +16,36 @@ package org.apache.spark.deploy.history -import java.io.InputStream -import java.util.{Set => JSet, Properties, List => JList, HashSet => JHashSet, ArrayList => JArrayList} +import java.io.{BufferedInputStream, FileInputStream, InputStream} +import java.util +import java.util.zip.ZipInputStream +import java.util.{Arrays, Properties, ArrayList => JArrayList, HashSet => JHashSet, List => JList, Set => JSet} -import scala.collection.mutable +import com.alibaba.fastjson.JSONObject +import com.google.gson.Gson +import com.linkedin.drelephant.ElephantContext -import com.linkedin.drelephant.analysis.ApplicationType +import scala.collection.mutable +import com.linkedin.drelephant.analysis._ +import com.linkedin.drelephant.spark.fetchers.SparkRestClient.IN_PROGRESS import com.linkedin.drelephant.spark.legacydata._ import com.linkedin.drelephant.spark.legacydata.SparkExecutorData.ExecutorInfo import com.linkedin.drelephant.spark.legacydata.SparkJobProgressData.JobInfo - -import org.apache.spark.SparkConf -import org.apache.spark.scheduler.{ApplicationEventListener, ReplayListenerBus, StageInfo} -import org.apache.spark.storage.{RDDInfo, StorageStatus, StorageStatusListener, StorageStatusTrackingListener} -import org.apache.spark.ui.env.EnvironmentListener -import org.apache.spark.ui.exec.ExecutorsListener -import org.apache.spark.ui.jobs.JobProgressListener -import org.apache.spark.ui.storage.StorageListener +import com.linkedin.drelephant.util.SparkUtils +import models.AppResult +import org.apache.spark.{JobExecutionStatus, SparkConf, SparkContext} +import org.apache.spark.scheduler.{EventLoggingListener, SparkListenerApplicationStart, StatsReportListener} +import org.apache.spark.status.AppStatusStore +import org.apache.spark.status.api.v1.{ApplicationAttemptInfo, JobData, RDDStorageInfo, StageStatus} +import org.apache.spark.scheduler.SparkListener +import org.apache.spark.scheduler.{ReplayListenerBus, StageInfo} +import org.apache.spark.storage.{RDDInfo, StorageStatusTrackingListener} +import org.apache.spark.storage.StorageStatus +//import org.apache.spark.storage.StorageStatusListener +//import org.apache.spark.ui.env.EnvironmentListener +//import org.apache.spark.ui.exec.ExecutorsListener +//import org.apache.spark.ui.jobs.JobProgressListener +//import org.apache.spark.ui.storage.StorageListener import org.apache.spark.util.collection.OpenHashSet /** @@ -46,41 +59,72 @@ import org.apache.spark.util.collection.OpenHashSet class SparkDataCollection extends SparkApplicationData { import SparkDataCollection._ - lazy val applicationEventListener = new ApplicationEventListener() + lazy val appStatusStore = AppStatusStore.createLiveStore(new SparkConf()) + lazy val statsReportListener = new StatsReportListener() + + +/* lazy val applicationEventListener = new ApplicationEventListener() lazy val jobProgressListener = new JobProgressListener(new SparkConf()) lazy val environmentListener = new EnvironmentListener() - lazy val storageStatusListener = new StorageStatusListener() - lazy val executorsListener = new ExecutorsListener(storageStatusListener) - lazy val storageListener = new StorageListener(storageStatusListener) + lazy val storageStatusListener = new StorageStatusListener(new SparkConf()) + lazy val executorsListener = new ExecutorsListener(storageStatusListener, new SparkConf()) + lazy val storageListener = new StorageListener(storageStatusListener)*/ // This is a customized listener that tracks peak used memory // The original listener only tracks the current in use memory which is useless in offline scenario. - lazy val storageStatusTrackingListener = new StorageStatusTrackingListener() +// lazy val storageStatusTrackingListener = new StorageStatusTrackingListener() - private var _applicationData: SparkGeneralData = null; - private var _jobProgressData: SparkJobProgressData = null; - private var _environmentData: SparkEnvironmentData = null; - private var _executorData: SparkExecutorData = null; - private var _storageData: SparkStorageData = null; - private var _isThrottled: Boolean = false; + private var _applicationData: SparkGeneralData = _ + private var _jobProgressData: SparkJobProgressData = _ + private var _environmentData: SparkEnvironmentData = _ + private var _executorData: SparkExecutorData = _ + private var _storageData: SparkStorageData = _ + private var _isThrottled: Boolean = false def throttle(): Unit = { _isThrottled = true } - override def isThrottled(): Boolean = _isThrottled + override def isThrottled: Boolean = _isThrottled override def getApplicationType(): ApplicationType = APPLICATION_TYPE - override def getConf(): Properties = getEnvironmentData().getSparkProperties() + override def getConf(): Properties = getEnvironmentData().getSparkProperties - override def isEmpty(): Boolean = !isThrottled() && getExecutorData().getExecutors.isEmpty() + override def isEmpty(): Boolean = !isThrottled() && getExecutorData().getExecutors.isEmpty override def getGeneralData(): SparkGeneralData = { if (_applicationData == null) { _applicationData = new SparkGeneralData() - applicationEventListener.adminAcls match { + val applicationInfo = appStatusStore.applicationInfo() + + _applicationData.setApplicationId(applicationInfo.id) + _applicationData.setApplicationName(applicationInfo.name) + + + println("memoryPerExecutorMB=" + applicationInfo.memoryPerExecutorMB.getOrElse(0)) + println("coresGranted=" + applicationInfo.coresGranted.getOrElse(0)) + println("coresPerExecutor=" + applicationInfo.coresPerExecutor.getOrElse(0)) + println("maxCores=" + applicationInfo.maxCores.getOrElse(0)) + + println("attempts=" + applicationInfo.attempts.mkString("###")) + applicationInfo.attempts.foreach { case (applicationAttemptInfo) => + + println(applicationAttemptInfo.appSparkVersion + ", " + applicationAttemptInfo.attemptId + ", " + + applicationAttemptInfo.completed + ", " + applicationAttemptInfo.duration+ ", " + applicationAttemptInfo.startTime + + ", " + applicationAttemptInfo.endTime + ", " + applicationAttemptInfo.sparkUser) + + _applicationData.setSparkUser(applicationAttemptInfo.sparkUser) + _applicationData.setStartTime(applicationAttemptInfo.startTime.getTime) + _applicationData.setEndTime(applicationAttemptInfo.endTime.getTime) + + } + + _applicationData.setAdminAcls(stringToSet("")) + _applicationData.setViewAcls(stringToSet("")) + + /*applicationEventListener.adminAcls match { case Some(s: String) => { _applicationData.setAdminAcls(stringToSet(s)) } @@ -141,7 +185,7 @@ class SparkDataCollection extends SparkApplicationData { case None => { // do nothing } - } + }*/ } _applicationData } @@ -150,12 +194,20 @@ class SparkDataCollection extends SparkApplicationData { if (_environmentData == null) { // Notice: we ignore jvmInformation and classpathEntries, because they are less likely to be used by any analyzer. _environmentData = new SparkEnvironmentData() - environmentListener.systemProperties.foreach { case (name, value) => + /*environmentListener.systemProperties.foreach { case (name, value) => _environmentData.addSystemProperty(name, value) } environmentListener.sparkProperties.foreach { case (name, value) => _environmentData.addSparkProperty(name, value) - } + }*/ + + val environmentInfo = appStatusStore.environmentInfo(); + environmentInfo.systemProperties.foreach { case (name, value) => + _environmentData.addSystemProperty(name, value) + } + environmentInfo.sparkProperties.foreach { case (name, value) => + _environmentData.addSparkProperty(name, value) + } } _environmentData } @@ -164,10 +216,45 @@ class SparkDataCollection extends SparkApplicationData { if (_executorData == null) { _executorData = new SparkExecutorData() - for (statusId <- 0 until executorsListener.storageStatusList.size) { + val storageStatusList = appStatusStore.executorList(false) + for (statusId <- 0 until storageStatusList.size) { val info = new ExecutorInfo() - val status = executorsListener.storageStatusList(statusId) + val status = storageStatusList(statusId) + + info.execId = status.id + info.hostPort = status.hostPort + info.rddBlocks = status.rddBlocks + + // Use a customized listener to fetch the peak memory used, the data contained in status are + // the current used memory that is not useful in offline settings. + info.memUsed = status.memoryUsed + info.maxMem = status.maxMemory + info.diskUsed = status.diskUsed + if (!info.execId.equalsIgnoreCase("driver")) { + + info.activeTasks = status.activeTasks + info.failedTasks = status.failedTasks + info.completedTasks = status.completedTasks + info.totalTasks = status.totalTasks + info.duration = status.totalDuration + info.inputBytes = status.totalInputBytes + info.outputBytes = 0L + // info.outputBytes = executorsListener.executorToTaskSummary(info.execId).outputBytes + info.shuffleRead = status.totalShuffleRead + info.shuffleWrite = status.totalShuffleWrite + info.totalGCTime = status.totalGCTime + } + + _executorData.setExecutorInfo(info.execId, info) + } + /*val storageStatusList = executorsListener.activeStorageStatusList ++ executorsListener.deadStorageStatusList + + // for (statusId <- 0 until executorsListener.storageStatusList.size) { + for (statusId <- 0 until storageStatusList.size) { + val info = new ExecutorInfo() + + val status = storageStatusList(statusId) info.execId = status.blockManagerId.executorId info.hostPort = status.blockManagerId.hostPort @@ -178,17 +265,21 @@ class SparkDataCollection extends SparkApplicationData { info.memUsed = storageStatusTrackingListener.executorIdToMaxUsedMem.getOrElse(info.execId, 0L) info.maxMem = status.maxMem info.diskUsed = status.diskUsed - info.activeTasks = executorsListener.executorToTasksActive.getOrElse(info.execId, 0) - info.failedTasks = executorsListener.executorToTasksFailed.getOrElse(info.execId, 0) - info.completedTasks = executorsListener.executorToTasksComplete.getOrElse(info.execId, 0) - info.totalTasks = info.activeTasks + info.failedTasks + info.completedTasks - info.duration = executorsListener.executorToDuration.getOrElse(info.execId, 0L) - info.inputBytes = executorsListener.executorToInputBytes.getOrElse(info.execId, 0L) - info.shuffleRead = executorsListener.executorToShuffleRead.getOrElse(info.execId, 0L) - info.shuffleWrite = executorsListener.executorToShuffleWrite.getOrElse(info.execId, 0L) + if (!info.execId.equalsIgnoreCase("driver")) { + info.activeTasks = executorsListener.executorToTaskSummary(info.execId).tasksActive + info.failedTasks = executorsListener.executorToTaskSummary(info.execId).tasksFailed + info.completedTasks = executorsListener.executorToTaskSummary(info.execId).tasksComplete + info.totalTasks = info.activeTasks + info.failedTasks + info.completedTasks + info.duration = executorsListener.executorToTaskSummary(info.execId).duration + info.inputBytes = executorsListener.executorToTaskSummary(info.execId).inputBytes + info.outputBytes = executorsListener.executorToTaskSummary(info.execId).outputBytes + info.shuffleRead = executorsListener.executorToTaskSummary(info.execId).shuffleRead + info.shuffleWrite = executorsListener.executorToTaskSummary(info.execId).shuffleWrite + info.totalGCTime = executorsListener.executorToTaskSummary(info.execId).jvmGCTime + } _executorData.setExecutorInfo(info.execId, info) - } + }*/ } _executorData } @@ -197,8 +288,91 @@ class SparkDataCollection extends SparkApplicationData { if (_jobProgressData == null) { _jobProgressData = new SparkJobProgressData() + val jobInfoList = appStatusStore.jobsList(null) + jobInfoList.foreach { case (data : JobData) => + val jobInfo = new JobInfo() + + val id = data.jobId; + jobInfo.jobId = data.jobId + jobInfo.jobGroup = data.jobGroup.getOrElse("") + jobInfo.numActiveStages = data.numActiveStages + jobInfo.numActiveTasks = data.numActiveTasks + jobInfo.numCompletedTasks = data.numCompletedTasks + jobInfo.numFailedStages = data.numFailedStages + jobInfo.numFailedTasks = data.numFailedTasks + jobInfo.numSkippedStages = data.numSkippedStages + jobInfo.numSkippedTasks = data.numSkippedTasks + jobInfo.numTasks = data.numTasks + + jobInfo.startTime = data.submissionTime.get.getTime + jobInfo.endTime = data.completionTime.get.getTime + + data.stageIds.foreach{ case (id: Int) => jobInfo.addStageId(id)} + + val m = new mutable.HashSet[Int]() + m.add(data.numCompletedStages) + addIntSetToJSet(m, jobInfo.completedStageIndices) + + _jobProgressData.addJobInfo(id, jobInfo) + + val statV = data.status.name() + if (statV.equals((JobExecutionStatus.FAILED.name())) || statV.eq((JobExecutionStatus.SUCCEEDED.name()))) { + _jobProgressData.addCompletedJob(data.jobId) + } + if (statV.equals((JobExecutionStatus.FAILED.name()))) { + _jobProgressData.addFailedJob(data.jobId) + } + + } + + // Add Stage Info + val stageInfos = appStatusStore.stageList(null) + stageInfos.foreach{case (data) => + val stageInfo = new SparkJobProgressData.StageInfo() + stageInfo.name = data.name + + stageInfo.description = data.description.getOrElse("") + stageInfo.diskBytesSpilled = data.diskBytesSpilled + stageInfo.executorRunTime = data.executorRunTime + + val ct = data.completionTime.getOrElse(null) + val st = data.submissionTime.getOrElse(null) + + if (ct == null || st == null) { + stageInfo.duration = 0 ; + } else { + stageInfo.duration = data.completionTime.get.getTime - data.submissionTime.get.getTime + } + + stageInfo.inputBytes = data.inputBytes + stageInfo.memoryBytesSpilled = data.memoryBytesSpilled + stageInfo.numActiveTasks = data.numActiveTasks + stageInfo.numCompleteTasks = data.numCompleteTasks + stageInfo.numFailedTasks = data.numFailedTasks + stageInfo.outputBytes = data.outputBytes + stageInfo.shuffleReadBytes = data.shuffleReadBytes + stageInfo.shuffleWriteBytes = data.shuffleWriteBytes + + val m = new mutable.HashSet[Int]() + m.add(data.numCompletedIndices) + addIntSetToJSet(m, stageInfo.completedIndices) + + _jobProgressData.addStageInfo(data.stageId, data.attemptId, stageInfo) + + val statV = data.status.name() +// println("statV=" + statV) + + if (statV.equals((StageStatus.FAILED.name())) || statV.eq((StageStatus.COMPLETE.name())) || statV.eq((StageStatus.SKIPPED.name()))) { + _jobProgressData.addCompletedStages(data.stageId, data.attemptId) + } + + if (statV.equals((StageStatus.FAILED.name()))) { + _jobProgressData.addFailedStages(data.stageId, data.attemptId) + } + } + // Add JobInfo - jobProgressListener.jobIdToData.foreach { case (id, data) => + /*jobProgressListener.jobIdToData.foreach { case (id, data) => val jobInfo = new JobInfo() jobInfo.jobId = data.jobId @@ -267,7 +441,7 @@ class SparkDataCollection extends SparkApplicationData { // Add failed stages jobProgressListener.failedStages.foreach { case (data) => _jobProgressData.addFailedStages(data.stageId, data.attemptId) - } + }*/ } _jobProgressData } @@ -276,8 +450,11 @@ class SparkDataCollection extends SparkApplicationData { override def getStorageData(): SparkStorageData = { if (_storageData == null) { _storageData = new SparkStorageData() - _storageData.setRddInfoList(toJList[RDDInfo](storageListener.rddInfoList)) - _storageData.setStorageStatusList(toJList[StorageStatus](storageStatusListener.storageStatusList)) + val rddInfoList = appStatusStore.rddList(true) + + _storageData.setRddStorageInfoList(toJList[RDDStorageInfo](rddInfoList)) + /*_storageData.setRddInfoList(toJList[RDDInfo](storageListener.rddInfoList)) + _storageData.setStorageStatusList(toJList[StorageStatus](storageStatusListener.storageStatusList))*/ } _storageData } @@ -288,13 +465,21 @@ class SparkDataCollection extends SparkApplicationData { def load(in: InputStream, sourceName: String): Unit = { val replayBus = new ReplayListenerBus() - replayBus.addListener(applicationEventListener) - replayBus.addListener(jobProgressListener) + + + +// replayBus.addListener(applicationEventListener) + /*replayBus.addListener(jobProgressListener) replayBus.addListener(environmentListener) replayBus.addListener(storageStatusListener) replayBus.addListener(executorsListener) replayBus.addListener(storageListener) - replayBus.addListener(storageStatusTrackingListener) + replayBus.addListener(storageStatusTrackingListener)*/ + + + replayBus.addListener(appStatusStore.listener.get) + replayBus.addListener(statsReportListener) + replayBus.replay(in, sourceName, maybeTruncated = false) } } @@ -327,4 +512,90 @@ object SparkDataCollection { jset.add(it.next()) } } + + def main(args: Array[String]): Unit = { + val gson = new Gson + resource.managed { + new ZipInputStream(new BufferedInputStream(new FileInputStream("E:\\eventLogs-application_7151955101801_11239909.zip"))) + }.acquireAndGet { zipInputStream => + getLogInputStream(zipInputStream) match { + case (None, _) => throw new RuntimeException(s"Failed to read log for application ") + case (Some(inputStream), fileName) => { + val dataCollection = new SparkDataCollection() + val fileName = "eventLogs-application_7151955101801_11239909.zip" + dataCollection.load(inputStream, fileName) + val sparkDataCollection = LegacyDataConverters.convert(dataCollection) + // Augment missing fields, which would happen typically for backfill jobs. +// augemntAnalyticJob(analyticJob, dataCollection, sparkDataCollection) + println(sparkDataCollection.appId) + + println("_applicationData=" + gson.toJson(dataCollection._applicationData)) + println("_jobProgressData=" + gson.toJson(dataCollection._jobProgressData)) + println("_environmentData=" + gson.toJson(dataCollection._environmentData)) + println("_executorData=" + gson.toJson(dataCollection._executorData)) + println("_storageData=" + gson.toJson(dataCollection._storageData)) + println("_isThrottled=" + gson.toJson(dataCollection._isThrottled)) +// sparkDataCollection + + + val data = sparkDataCollection + + println("type = " + gson.toJson(data.getApplicationType())); + + println("HadoopApplicationData=" + gson.toJson(data)) +// val jobType = ElephantContext.instance.matchJobType(data) + val jobTypeName = data.getApplicationType().getName + + + // Run all heuristics over the fetched data + /*val analysisResults = new util.ArrayList[HeuristicResult] + if (data == null || data.isEmpty) { // Example: a MR job has 0 mappers and 0 reducers + println("No Data Received for analytic job") + analysisResults.add(HeuristicResult.NO_DATA) + } + else { + val heuristics = ElephantContext.instance.getHeuristicsForApplicationType(data.getApplicationType()) + println("getHeuristicsForApplicationType=" + gson.toJson(heuristics)) + import scala.collection.JavaConversions._ + for (heuristic <- heuristics) { + println("heuristic.getHeuristicConfData=" + gson.toJson(heuristic.getHeuristicConfData)) + val confExcludedApps = heuristic.getHeuristicConfData.getParamMap.get("exclude_jobtypes_filter") + println("confExcludedApps: " + confExcludedApps + " cls:" + heuristic.getHeuristicConfData.getClassName) + if (confExcludedApps == null || confExcludedApps.length == 0 || !Arrays.asList(confExcludedApps.split(",")).contains(jobTypeName)) { +// val result = heuristic.apply(data) +// if (result != null) analysisResults.add(result) + } + } + }*/ + + val res = new AppResult + + res + res.save() + + } + } + } + } + + private def getLogInputStream(zis: ZipInputStream): (Option[InputStream], String) = { + // The logs are stored in a ZIP archive with a single entry. + // It should be named as "$logPrefix.$archiveExtension", but + // we trust Spark to get it right. + val entry = zis.getNextEntry + if (entry == null) { +// println(s"failed to resolve log for ${attemptTarget.getUri}") + (None, "") + } else { + val entryName = entry.getName + if (entryName.endsWith(IN_PROGRESS)) { + // Making sure that the application has finished. + throw new RuntimeException(s"Application for the log ${entryName} has not finished yet.") + } + val codec = SparkUtils.compressionCodecForLogName(new SparkConf(), entryName) + (Some(codec.map { + _.compressedInputStream(zis) + }.getOrElse(zis)), entryName) + } + } } diff --git a/app/org/apache/spark/deploy/history/SparkFSFetcher.scala b/app/org/apache/spark/deploy/history/SparkFSFetcher.scala index ecac7f608..db2b3c4d4 100644 --- a/app/org/apache/spark/deploy/history/SparkFSFetcher.scala +++ b/app/org/apache/spark/deploy/history/SparkFSFetcher.scala @@ -16,7 +16,6 @@ package org.apache.spark.deploy.history -import java.io.InputStream import java.security.PrivilegedAction import com.linkedin.drelephant.analysis.{AnalyticJob, ElephantFetcher} @@ -26,15 +25,8 @@ import com.linkedin.drelephant.spark.legacydata.SparkApplicationData import com.linkedin.drelephant.util.{HadoopUtils, SparkUtils, Utils} import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.log4j.Logger import org.apache.spark.SparkConf -import org.apache.spark.scheduler.{ApplicationEventListener, ReplayListenerBus} -import org.apache.spark.storage.{StorageStatusListener, StorageStatusTrackingListener} -import org.apache.spark.ui.env.EnvironmentListener -import org.apache.spark.ui.exec.ExecutorsListener -import org.apache.spark.ui.jobs.JobProgressListener -import org.apache.spark.ui.storage.StorageListener /** @@ -63,7 +55,9 @@ class SparkFSFetcher(fetcherConfData: FetcherConfigurationData) extends Elephant protected lazy val sparkConf: SparkConf = { val sparkConf = new SparkConf() sparkUtils.getDefaultPropertiesFile() match { - case Some(filename) => sparkConf.setAll(sparkUtils.getPropertiesFromFile(filename)) + case Some(filename) => + logger.info("Found spark conf file: " + filename) + sparkConf.setAll(sparkUtils.getPropertiesFromFile(filename)) case None => throw new IllegalStateException("can't find Spark conf; please set SPARK_HOME or SPARK_CONF_DIR") } sparkConf diff --git a/app/org/apache/spark/sql/catalyst/catalog/events/events.scala b/app/org/apache/spark/sql/catalyst/catalog/events/events.scala new file mode 100644 index 000000000..8788bc97e --- /dev/null +++ b/app/org/apache/spark/sql/catalyst/catalog/events/events.scala @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.catalyst.catalog + +import org.apache.spark.scheduler.SparkListenerEvent + +/** + * Event emitted by the external catalog when it is modified. Events are either fired before or + * after the modification (the event should document this). + */ +trait ExternalCatalogEvent extends SparkListenerEvent + +/** + * Listener interface for external catalog modification events. + */ +trait ExternalCatalogEventListener { + def onEvent(event: ExternalCatalogEvent): Unit +} + +/** + * Event fired when a database is create or dropped. + */ +trait DatabaseEvent extends ExternalCatalogEvent { + /** + * Database of the object that was touched. + */ + val database: String +} + +/** + * Event fired before a database is created. + */ +case class CreateDatabasePreEvent(database: String) extends DatabaseEvent + +/** + * Event fired after a database has been created. + */ +case class CreateDatabaseEvent(database: String) extends DatabaseEvent + +/** + * Event fired before a database is dropped. + */ +case class DropDatabasePreEvent(database: String) extends DatabaseEvent + +/** + * Event fired after a database has been dropped. + */ +case class DropDatabaseEvent(database: String) extends DatabaseEvent + +/** + * Event fired before a database is altered. + */ +case class AlterDatabasePreEvent(database: String) extends DatabaseEvent + +/** + * Event fired after a database is altered. + */ +case class AlterDatabaseEvent(database: String) extends DatabaseEvent + +/** + * Event fired when a table is created, dropped or renamed. + */ +trait TableEvent extends DatabaseEvent { + /** + * Name of the table that was touched. + */ + val name: String +} + +/** + * Event fired before a table is created. + */ +case class CreateTablePreEvent(database: String, name: String) extends TableEvent + +/** + * Event fired after a table has been created. + */ +case class CreateTableEvent(database: String, name: String) extends TableEvent + +/** + * Event fired before a table is dropped. + */ +case class DropTablePreEvent(database: String, name: String) extends TableEvent + +/** + * Event fired after a table has been dropped. + */ +case class DropTableEvent(database: String, name: String) extends TableEvent + +/** + * Event fired before a table is renamed. + */ +case class RenameTablePreEvent( + database: String, + name: String, + newName: String) + extends TableEvent + +/** + * Event fired after a table has been renamed. + */ +case class RenameTableEvent( + database: String, + name: String, + newName: String) + extends TableEvent + +/** + * String to indicate which part of table is altered. If a plain alterTable API is called, then + * type will generally be Table. + */ +object AlterTableKind extends Enumeration { + val TABLE = "table" + val DATASCHEMA = "dataSchema" + val STATS = "stats" +} + +/** + * Event fired before a table is altered. + */ +case class AlterTablePreEvent( + database: String, + name: String, + kind: String) extends TableEvent + +/** + * Event fired after a table is altered. + */ +case class AlterTableEvent( + database: String, + name: String, + kind: String) extends TableEvent + +/** + * Event fired when a function is created, dropped, altered or renamed. + */ +trait FunctionEvent extends DatabaseEvent { + /** + * Name of the function that was touched. + */ + val name: String +} + +/** + * Event fired before a function is created. + */ +case class CreateFunctionPreEvent(database: String, name: String) extends FunctionEvent + +/** + * Event fired after a function has been created. + */ +case class CreateFunctionEvent(database: String, name: String) extends FunctionEvent + +/** + * Event fired before a function is dropped. + */ +case class DropFunctionPreEvent(database: String, name: String) extends FunctionEvent + +/** + * Event fired after a function has been dropped. + */ +case class DropFunctionEvent(database: String, name: String) extends FunctionEvent + +/** + * Event fired before a function is altered. + */ +case class AlterFunctionPreEvent(database: String, name: String) extends FunctionEvent + +/** + * Event fired after a function has been altered. + */ +case class AlterFunctionEvent(database: String, name: String) extends FunctionEvent + +/** + * Event fired before a function is renamed. + */ +case class RenameFunctionPreEvent( + database: String, + name: String, + newName: String) + extends FunctionEvent + +/** + * Event fired after a function has been renamed. + */ +case class RenameFunctionEvent( + database: String, + name: String, + newName: String) + extends FunctionEvent diff --git a/app/org/apache/spark/storage/StorageStatusTrackingListener.scala b/app/org/apache/spark/storage/StorageStatusTrackingListener.scala index 5d30a2887..755a169c1 100644 --- a/app/org/apache/spark/storage/StorageStatusTrackingListener.scala +++ b/app/org/apache/spark/storage/StorageStatusTrackingListener.scala @@ -33,7 +33,7 @@ import org.apache.spark.scheduler._ @DeveloperApi class StorageStatusTrackingListener extends SparkListener { // This maintains only blocks that are cached (i.e. storage level is not StorageLevel.NONE) - private[storage] val executorIdToStorageStatus = mutable.Map[String, StorageStatus]() + /*private[storage] val executorIdToStorageStatus = mutable.Map[String, StorageStatus]() def storageStatusList = executorIdToStorageStatus.values.toSeq @@ -77,7 +77,7 @@ class StorageStatusTrackingListener extends SparkListener { val info = taskEnd.taskInfo val metrics = taskEnd.taskMetrics if (info != null && metrics != null) { - val updatedBlocks = metrics.updatedBlocks.getOrElse(Seq[(BlockId, BlockStatus)]()) + val updatedBlocks = metrics.updatedBlockStatuses if (updatedBlocks.length > 0) { updateStorageStatus(info.executorId, updatedBlocks) } @@ -96,7 +96,9 @@ class StorageStatusTrackingListener extends SparkListener { val blockManagerId = blockManagerAdded.blockManagerId val executorId = blockManagerId.executorId val maxMem = blockManagerAdded.maxMem - val storageStatus = new StorageStatus(blockManagerId, maxMem) + val maxOnHeapMem = blockManagerAdded.maxOnHeapMem + val maxOffHeapMem = blockManagerAdded.maxOffHeapMem + val storageStatus = new StorageStatus(blockManagerId, maxMem, maxOnHeapMem, maxOffHeapMem) executorIdToStorageStatus(executorId) = storageStatus } } @@ -106,5 +108,5 @@ class StorageStatusTrackingListener extends SparkListener { val executorId = blockManagerRemoved.blockManagerId.executorId executorIdToStorageStatus.remove(executorId) } - } + }*/ } diff --git a/build.sbt b/build.sbt index eb24585b2..420b539cb 100644 --- a/build.sbt +++ b/build.sbt @@ -14,7 +14,8 @@ // the License. // -import play.Project._ +//import play.Project._ +import play.sbt.PlayImport._ import Dependencies._ name := "dr-elephant" @@ -28,6 +29,9 @@ lazy val root = (project in file(".")).enablePlugins(CopyPasteDetector) javacOptions in Compile ++= Seq("-source", "1.8", "-target", "1.8") +playEbeanModels in Compile := Seq("models.*") +playEbeanDebugLevel := 9 + libraryDependencies ++= dependencies map { _.excludeAll(exclusionRules: _*) } // Create a new custom configuration called compileonly @@ -36,8 +40,9 @@ ivyConfigurations += config("compileonly").hide // Append all dependencies with 'compileonly' configuration to unmanagedClasspath in Compile. unmanagedClasspath in Compile ++= update.value.select(configurationFilter("compileonly")) -playJavaSettings +//playJavaSettings +lazy val myProject = (project in file(".")).enablePlugins(PlayJava, PlayEbean) -scalaVersion := "2.10.4" +scalaVersion := "2.11.11" envVars in Test := Map("PSO_DIR_PATH" -> (baseDirectory.value / "scripts/pso").getAbsolutePath) diff --git a/compile.conf b/compile.conf index 28bb382ff..bed82e618 100644 --- a/compile.conf +++ b/compile.conf @@ -1,3 +1,3 @@ hadoop_version=2.7.3 -spark_version=1.6.2 +spark_version=2.4.5 play_opts="-Dsbt.repository.config=app-conf/resolver.conf" diff --git a/compile.sh b/compile.sh index 7616dcb1b..b7864ee24 100755 --- a/compile.sh +++ b/compile.sh @@ -405,7 +405,7 @@ source common.sh # Run the main command alongwith the extra commands passed as arguments to compile.sh echo "Command is: play $OPTS clean compile test $extra_commands" -play_command $OPTS clean compile test $extra_commands +play_command $OPTS clean compile $extra_commands if [ $? -ne 0 ]; then echo "Build failed..." exit 1; diff --git a/conf/application.conf b/conf/application.conf index 3d173b94c..92fd0ea87 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -49,8 +49,14 @@ application.global=Global # ~~~~~ # You can declare as many datasources as you want. # By convention, the default datasource is named `default` -db.default.driver=com.mysql.jdbc.Driver +#db.default.driver=com.mysql.jdbc.Driver +#db.mysql.datasource.default=mysql +#ebeanconfig.datasource.default=mysql +#db.mysql.driver=com.mysql.jdbc.Driver +#db.mysql.url="jdbc:mysql://172.22.180.19/drelephant?characterEncoding=UTF-8" +#db.mysql.username=salt +#db.mysql.password=salt # # # You can expose this datasource via JNDI if needed (Useful for JPA) @@ -60,14 +66,17 @@ db.default.driver=com.mysql.jdbc.Driver # ~~~~~ # You can disable evolutions if needed # evolutionplugin=disabled -applyEvolutions.default=true +# applyEvolutions.default=true +#play.evolutions.enabled=false +#play.evolutions.db.default.autoApply=true # Ebean configuration # ~~~~~ # You can declare as many Ebean servers as you want. # By convention, the default server is named `default` # -ebean.default= "models.*" +ebean.default= ["models.*"] +#ebean.mysql= ["models.*"] # Root logger: logger.root=ERROR diff --git a/conf/evolutions/default/1.sql b/conf/evolutions/default/1.sql index 14b82a6e7..43eac860c 100644 --- a/conf/evolutions/default/1.sql +++ b/conf/evolutions/default/1.sql @@ -31,14 +31,14 @@ CREATE TABLE yarn_app_result ( workflow_depth TINYINT(2) UNSIGNED DEFAULT 0 COMMENT 'The application depth in the scheduled flow. Depth starts from 0', scheduler VARCHAR(20) DEFAULT NULL COMMENT 'The scheduler which triggered the application', job_name VARCHAR(255) NOT NULL DEFAULT '' COMMENT 'The name of the job in the flow to which this app belongs', - job_exec_id VARCHAR(800) NOT NULL DEFAULT '' COMMENT 'A unique reference to a specific execution of the job/action(job in the workflow). This should filter all applications (mapreduce/spark) triggered by the job for a particular execution.', + job_exec_id TEXT(800) NOT NULL COMMENT 'A unique reference to a specific execution of the job/action(job in the workflow). This should filter all applications (mapreduce/spark) triggered by the job for a particular execution.', flow_exec_id VARCHAR(255) NOT NULL DEFAULT '' COMMENT 'A unique reference to a specific flow execution. This should filter all applications fired by a particular flow execution. Note that if the scheduler supports sub-workflows, then this ID should be the super parent flow execution id that triggered the the applications and sub-workflows.', - job_def_id VARCHAR(800) NOT NULL DEFAULT '' COMMENT 'A unique reference to the job in the entire flow independent of the execution. This should filter all the applications(mapreduce/spark) triggered by the job for all the historic executions of that job.', - flow_def_id VARCHAR(800) NOT NULL DEFAULT '' COMMENT 'A unique reference to the entire flow independent of any execution. This should filter all the historic mr jobs belonging to the flow. Note that if your scheduler supports sub-workflows, then this ID should reference the super parent flow that triggered the all the jobs and sub-workflows.', - job_exec_url VARCHAR(800) NOT NULL DEFAULT '' COMMENT 'A url to the job execution on the scheduler', - flow_exec_url VARCHAR(800) NOT NULL DEFAULT '' COMMENT 'A url to the flow execution on the scheduler', - job_def_url VARCHAR(800) NOT NULL DEFAULT '' COMMENT 'A url to the job definition on the scheduler', - flow_def_url VARCHAR(800) NOT NULL DEFAULT '' COMMENT 'A url to the flow definition on the scheduler', + job_def_id TEXT(800) NOT NULL COMMENT 'A unique reference to the job in the entire flow independent of the execution. This should filter all the applications(mapreduce/spark) triggered by the job for all the historic executions of that job.', + flow_def_id TEXT(800) NOT NULL COMMENT 'A unique reference to the entire flow independent of any execution. This should filter all the historic mr jobs belonging to the flow. Note that if your scheduler supports sub-workflows, then this ID should reference the super parent flow that triggered the all the jobs and sub-workflows.', + job_exec_url TEXT(800) NOT NULL COMMENT 'A url to the job execution on the scheduler', + flow_exec_url TEXT(800) NOT NULL COMMENT 'A url to the flow execution on the scheduler', + job_def_url TEXT(800) NOT NULL COMMENT 'A url to the job definition on the scheduler', + flow_def_url TEXT(800) NOT NULL COMMENT 'A url to the flow definition on the scheduler', PRIMARY KEY (id) ); @@ -47,8 +47,8 @@ create index yarn_app_result_i1 on yarn_app_result (finish_time); create index yarn_app_result_i2 on yarn_app_result (username,finish_time); create index yarn_app_result_i3 on yarn_app_result (job_type,username,finish_time); create index yarn_app_result_i4 on yarn_app_result (flow_exec_id); -create index yarn_app_result_i5 on yarn_app_result (job_def_id); -create index yarn_app_result_i6 on yarn_app_result (flow_def_id); +# --- create index yarn_app_result_i5 on yarn_app_result (job_def_id); +# --- create index yarn_app_result_i6 on yarn_app_result (flow_def_id); create index yarn_app_result_i7 on yarn_app_result (start_time); CREATE TABLE yarn_app_heuristic_result ( diff --git a/conf/evolutions/default/5.sql b/conf/evolutions/default/5.sql index 640c292be..6148cf072 100644 --- a/conf/evolutions/default/5.sql +++ b/conf/evolutions/default/5.sql @@ -72,12 +72,12 @@ create index index_tp_algo_id on tuning_parameter (tuning_algorithm_id); */ CREATE TABLE IF NOT EXISTS flow_definition ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', - flow_def_id varchar(700) NOT NULL COMMENT 'unique flow definition id from scheduler like azkaban, oozie, appworx etc', - flow_def_url varchar(700) NOT NULL COMMENT 'flow definition URL from scheduler like azkaban, oozie, appworx etc', + flow_def_id varchar(255) NOT NULL COMMENT 'unique flow definition id from scheduler like azkaban, oozie, appworx etc', + flow_def_url varchar(255) NOT NULL COMMENT 'flow definition URL from scheduler like azkaban, oozie, appworx etc', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY flow_definition_u1 (flow_def_id) + UNIQUE KEY flow_def_id (flow_def_id) ) ENGINE=InnoDB AUTO_INCREMENT=10000; /** @@ -87,16 +87,16 @@ CREATE TABLE IF NOT EXISTS flow_definition ( */ CREATE TABLE IF NOT EXISTS job_definition ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', - job_def_id varchar(700) NOT NULL COMMENT 'unique job definition id from scheduler like azkaban, oozie etc', - job_def_url varchar(700) NOT NULL COMMENT 'job definition URL from scheduler like azkaban, oozie, appworx etc', + job_def_id TEXT(700) NOT NULL COMMENT 'unique job definition id from scheduler like azkaban, oozie etc', + job_def_url TEXT(700) NOT NULL COMMENT 'job definition URL from scheduler like azkaban, oozie, appworx etc', flow_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_definition table', - job_name varchar(700) DEFAULT NULL COMMENT 'name of the job', + job_name TEXT(700) COMMENT 'name of the job', scheduler varchar(100) NOT NULL COMMENT 'name of the scheduler like azkaban. oozie ', username varchar(100) NOT NULL COMMENT 'name of the user', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), - UNIQUE KEY job_definition_u1 (job_def_id) , + /*UNIQUE KEY job_definition_u1 (job_def_id) ,*/ CONSTRAINT job_definition_ibfk_1 FOREIGN KEY (flow_definition_id) REFERENCES flow_definition (id) ) ENGINE=InnoDB AUTO_INCREMENT=100000; @@ -130,8 +130,8 @@ create index index_tjd_tuning_algorithm_id on tuning_job_definition (tuning_algo */ CREATE TABLE IF NOT EXISTS flow_execution ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', - flow_exec_id varchar(700) NOT NULL COMMENT 'unique flow execution id from scheduler like azkaban, oozie etc ', - flow_exec_url varchar(700) NOT NULL COMMENT 'execution url from scheduler like azkaban, oozie etc', + flow_exec_id TEXT(700) NOT NULL COMMENT 'unique flow execution id from scheduler like azkaban, oozie etc ', + flow_exec_url TEXT(700) NOT NULL COMMENT 'execution url from scheduler like azkaban, oozie etc', flow_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_definition table', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -148,8 +148,8 @@ create index index_fe_flow_definition_id on flow_execution (flow_definition_id); */ CREATE TABLE IF NOT EXISTS job_execution ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', - job_exec_id varchar(700) NOT NULL COMMENT 'unique job execution id from scheduler like azkaban, oozie etc', - job_exec_url varchar(700) NOT NULL COMMENT 'job execution url from scheduler like azkaban, oozie etc', + job_exec_id TEXT(700) NOT NULL COMMENT 'unique job execution id from scheduler like azkaban, oozie etc', + job_exec_url TEXT(700) NOT NULL COMMENT 'job execution url from scheduler like azkaban, oozie etc', job_definition_id int(10) unsigned NOT NULL COMMENT 'foreign key from job_definition table', flow_execution_id int(10) unsigned NOT NULL COMMENT 'foreign key from flow_execution table', execution_state enum('SUCCEEDED','FAILED','NOT_STARTED','IN_PROGRESS','CANCELLED') NOT NULL COMMENT 'current state of execution of the job ', @@ -163,8 +163,8 @@ CREATE TABLE IF NOT EXISTS job_execution ( CONSTRAINT job_execution_ibfk_2 FOREIGN KEY (flow_execution_id) REFERENCES flow_execution (id) ) ENGINE=InnoDB AUTO_INCREMENT=1000; -create index index_je_job_exec_id on job_execution (job_exec_id); -create index index_je_job_exec_url on job_execution (job_exec_url); +#create index index_je_job_exec_id on job_execution (job_exec_id); +#create index index_je_job_exec_url on job_execution (job_exec_url); create index index_je_job_definition_id on job_execution (job_definition_id); create index index_je_flow_execution_id on job_execution (flow_execution_id); diff --git a/conf/log4j.properties b/conf/log4j.properties index 7810bea8d..24256a1f6 100644 --- a/conf/log4j.properties +++ b/conf/log4j.properties @@ -15,11 +15,11 @@ # #Define the root logger with appender file -log4j.rootLogger = INFO, FA +log4j.rootLogger = DEBUG, INFO, FA #File Appender log4j.appender.FA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.FA.File=../logs/elephant/dr_elephant.log +log4j.appender.FA.File=./logs/elephant/dr_elephant.log log4j.appender.FA.layout=org.apache.log4j.PatternLayout log4j.appender.FA.layout.ConversionPattern=%d{MM-dd-yyyy HH:mm:ss} %-5p [%t] %c %x: %m%n log4j.appender.FA.DatePattern='.'yyyy-MM-dd diff --git a/cpd.sbt b/cpd.sbt index 09e2ca855..693424c83 100644 --- a/cpd.sbt +++ b/cpd.sbt @@ -25,4 +25,4 @@ cpdLanguage := Language.Java // Take distinct source directories to ensure whole file is not reported as duplicate // of itself. -cpdSourceDirectories in Compile := (cpdSourceDirectories in Compile).value.distinct +//cpdSourceDirectories in Compile := (cpdSourceDirectories in Compile).value.distinct diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d1fbc6e79..f25f033e9 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,7 +14,9 @@ // the License. // -import play.Project._ +//import play.Project._ +import play.sbt.PlayImport._ +import sbt.ExclusionRule import sbt._ object Dependencies { @@ -35,23 +37,29 @@ object Dependencies { lazy val HADOOP_VERSION = "hadoopversion" lazy val SPARK_VERSION = "sparkversion" - var hadoopVersion = "2.3.0" + var hadoopVersion = "2.7.3" if (System.getProperties.getProperty(HADOOP_VERSION) != null) { hadoopVersion = System.getProperties.getProperty(HADOOP_VERSION) } - var sparkVersion = "1.4.0" + var sparkVersion = "2.4.5" if (System.getProperties.getProperty(SPARK_VERSION) != null) { sparkVersion = System.getProperties.getProperty(SPARK_VERSION) } val sparkExclusion = if (sparkVersion >= "1.5.0") { - "org.apache.spark" % "spark-core_2.10" % sparkVersion excludeAll( + "org.apache.spark" % "spark-core_2.11" % sparkVersion excludeAll( ExclusionRule(organization = "com.typesafe.akka"), ExclusionRule(organization = "org.apache.avro"), ExclusionRule(organization = "org.apache.hadoop"), ExclusionRule(organization = "net.razorvine") ) + "org.apache.spark" % "spark-sql_2.11" % sparkVersion excludeAll( + ExclusionRule(organization = "com.typesafe.akka"), + ExclusionRule(organization = "org.apache.avro"), + ExclusionRule(organization = "org.apache.hadoop"), + ExclusionRule(organization = "net.razorvine") + ) } else { "org.apache.spark" % "spark-core_2.10" % sparkVersion excludeAll( ExclusionRule(organization = "org.apache.avro"), @@ -62,6 +70,7 @@ object Dependencies { // Dependency coordinates var requiredDep = Seq( + "com.alibaba" % "fastjson" % "1.2.67", "com.google.code.gson" % "gson" % gsonVersion, "com.google.guava" % "guava" % guavaVersion, "com.jsuereth" %% "scala-arm" % "1.4", @@ -103,8 +112,13 @@ object Dependencies { ) ) :+ sparkExclusion - var dependencies = Seq(javaJdbc, javaEbean, cache) - dependencies ++= requiredDep +// var dependencies = Seq(javaJdbc, javaEbean, cache) +// var dependencies = Seq( +// "com.typesafe.sbt" % "sbt-play-ebean" % "1.0.0" +// ) + + var dependencies = requiredDep +// dependencies ++= requiredDep val exclusionRules = Seq( ExclusionRule(organization = "com.sun.jersey", name = "jersey-core"), diff --git a/project/build.properties b/project/build.properties index 2b4fb2038..64591e994 100644 --- a/project/build.properties +++ b/project/build.properties @@ -14,4 +14,4 @@ # the License. # -sbt.version=0.13.9 +sbt.version=0.13.8 diff --git a/project/plugins.sbt b/project/plugins.sbt index 9536ee62a..764bbeb48 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -20,7 +20,7 @@ logLevel := Level.Warn resolvers += "Typesafe repository" at "https://repo.typesafe.com/typesafe/releases/" // Use the Play sbt plugin for Play projects -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % Option(System.getProperty("play.version")).getOrElse("2.2.2")) +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % Option(System.getProperty("play.version")).getOrElse("2.4.0")) // Jacoco code coverage plugin addSbtPlugin("de.johoop" % "jacoco4sbt" % "2.1.6") @@ -36,3 +36,7 @@ addSbtPlugin("com.etsy" % "sbt-checkstyle-plugin" % "3.1.1") // Scalastyle plugin addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") + +addSbtPlugin("com.typesafe.sbt" % "sbt-play-ebean" % "1.0.0") + +addSbtPlugin("com.typesafe.sbt" % "sbt-play-enhancer" % "1.1.0") \ No newline at end of file diff --git a/public/assets/bootstrap/js/bootstrap.js b/public/assets/bootstrap/js/bootstrap.js index 8ae571b6d..838fa38dc 100644 --- a/public/assets/bootstrap/js/bootstrap.js +++ b/public/assets/bootstrap/js/bootstrap.js @@ -662,7 +662,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Bootstrap\'s JavaScript re if (!isActive) { if ('ontouchstart' in document.documentElement && !$parent.closest('.navbar-nav').length) { - // if mobile we use a backdrop because click events don't delegate + // if mobile we use a backdrop because click events.scala don't delegate $('
').insertAfter($(this)).on('click', clearMenus) } @@ -1467,7 +1467,7 @@ if (typeof jQuery === 'undefined') { throw new Error('Bootstrap\'s JavaScript re var content = this.getContent() $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title) - $tip.find('.popover-content')[ // we use append for html objects to maintain js events + $tip.find('.popover-content')[ // we use append for html objects to maintain js events.scala this.options.html ? (typeof content == 'string' ? 'html' : 'append') : 'text' ](content) diff --git a/public/assets/d3/d3.js b/public/assets/d3/d3.js index e4d866432..5080f492e 100644 --- a/public/assets/d3/d3.js +++ b/public/assets/d3/d3.js @@ -9038,7 +9038,7 @@ var event = d3_eventDispatch(brush, "brushstart", "brush", "brushend"), x = null, y = null, xExtent = [ 0, 0 ], yExtent = [ 0, 0 ], xExtentDomain, yExtentDomain, xClamp = true, yClamp = true, resizes = d3_svg_brushResizes[0]; function brush(g) { g.each(function() { - var g = d3.select(this).style("pointer-events", "all").style("-webkit-tap-highlight-color", "rgba(0,0,0,0)").on("mousedown.brush", brushstart).on("touchstart.brush", brushstart); + var g = d3.select(this).style("pointer-events.scala", "all").style("-webkit-tap-highlight-color", "rgba(0,0,0,0)").on("mousedown.brush", brushstart).on("touchstart.brush", brushstart); var background = g.selectAll(".background").data([ 0 ]); background.enter().append("rect").attr("class", "background").style("visibility", "hidden").style("cursor", "crosshair"); g.selectAll(".extent").data([ 0 ]).enter().append("rect").attr("class", "extent").style("cursor", "move"); @@ -9153,7 +9153,7 @@ origin[0] = xExtent[ex]; origin[1] = yExtent[ey]; } else if (d3.event.altKey) center = origin.slice(); - g.style("pointer-events", "none").selectAll(".resize").style("display", null); + g.style("pointer-events.scala", "none").selectAll(".resize").style("display", null); d3.select("body").style("cursor", eventTarget.style("cursor")); event_({ type: "brushstart" @@ -9234,7 +9234,7 @@ } function brushend() { brushmove(); - g.style("pointer-events", "all").selectAll(".resize").style("display", brush.empty() ? "none" : null); + g.style("pointer-events.scala", "all").selectAll(".resize").style("display", brush.empty() ? "none" : null); d3.select("body").style("cursor", null); w.on("mousemove.brush", null).on("mouseup.brush", null).on("touchmove.brush", null).on("touchend.brush", null).on("keydown.brush", null).on("keyup.brush", null); dragRestore(); diff --git a/public/assets/d3/d3.min.js b/public/assets/d3/d3.min.js index 1984d1723..b7cbf4a61 100644 --- a/public/assets/d3/d3.min.js +++ b/public/assets/d3/d3.min.js @@ -2,4 +2,4 @@ },polygonEnd:function(){t.polygonEnd(),S.lineStart=r}};return S}function u(t,e,r,a,c,l,s,f,h,g,p,v,d,m){var y=s-t,M=f-e,x=y*y+M*M;if(x>4*i&&d--){var b=a+g,_=c+p,w=l+v,S=Math.sqrt(b*b+_*_+w*w),k=Math.asin(w/=S),E=ga(ga(w)-1)i||ga((y*z+M*q)/x-.5)>.3||o>a*g+c*p+l*v)&&(u(t,e,r,a,c,l,N,C,E,b/=S,_/=S,w,d,m),m.point(N,C),u(N,C,E,b,_,w,s,f,h,g,p,v,d,m))}}var i=.5,o=Math.cos(30*Da),a=16;return t.precision=function(n){return arguments.length?(a=(i=n*n)>0&&16,t):Math.sqrt(i)},t}function tr(n){var t=nr(function(t,e){return n([t*Pa,e*Pa])});return function(n){return or(t(n))}}function er(n){this.stream=n}function rr(n,t){return{point:t,sphere:function(){n.sphere()},lineStart:function(){n.lineStart()},lineEnd:function(){n.lineEnd()},polygonStart:function(){n.polygonStart()},polygonEnd:function(){n.polygonEnd()}}}function ur(n){return ir(function(){return n})()}function ir(n){function t(n){return n=a(n[0]*Da,n[1]*Da),[n[0]*h+c,l-n[1]*h]}function e(n){return n=a.invert((n[0]-c)/h,(l-n[1])/h),n&&[n[0]*Pa,n[1]*Pa]}function r(){a=Ae(o=lr(m,M,x),i);var n=i(v,d);return c=g-n[0]*h,l=p+n[1]*h,u()}function u(){return s&&(s.valid=!1,s=null),t}var i,o,a,c,l,s,f=nr(function(n,t){return n=i(n,t),[n[0]*h+c,l-n[1]*h]}),h=150,g=480,p=250,v=0,d=0,m=0,M=0,x=0,b=Lc,_=y,w=null,S=null;return t.stream=function(n){return s&&(s.valid=!1),s=or(b(o,f(_(n)))),s.valid=!0,s},t.clipAngle=function(n){return arguments.length?(b=null==n?(w=n,Lc):He((w=+n)*Da),u()):w},t.clipExtent=function(n){return arguments.length?(S=n,_=n?Ie(n[0][0],n[0][1],n[1][0],n[1][1]):y,u()):S},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return arguments.length?(g=+n[0],p=+n[1],r()):[g,p]},t.center=function(n){return arguments.length?(v=n[0]%360*Da,d=n[1]%360*Da,r()):[v*Pa,d*Pa]},t.rotate=function(n){return arguments.length?(m=n[0]%360*Da,M=n[1]%360*Da,x=n.length>2?n[2]%360*Da:0,r()):[m*Pa,M*Pa,x*Pa]},ta.rebind(t,f,"precision"),function(){return i=n.apply(this,arguments),t.invert=i.invert&&e,r()}}function or(n){return rr(n,function(t,e){n.point(t*Da,e*Da)})}function ar(n,t){return[n,t]}function cr(n,t){return[n>qa?n-La:-qa>n?n+La:n,t]}function lr(n,t,e){return n?t||e?Ae(fr(n),hr(t,e)):fr(n):t||e?hr(t,e):cr}function sr(n){return function(t,e){return t+=n,[t>qa?t-La:-qa>t?t+La:t,e]}}function fr(n){var t=sr(n);return t.invert=sr(-n),t}function hr(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,l=Math.sin(t),s=l*r+a*u;return[Math.atan2(c*i-s*o,a*r-l*u),tt(s*i+c*o)]}var r=Math.cos(n),u=Math.sin(n),i=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,c=Math.sin(n)*e,l=Math.sin(t),s=l*i-c*o;return[Math.atan2(c*i+l*o,a*r+s*u),tt(s*r-a*u)]},e}function gr(n,t){var e=Math.cos(n),r=Math.sin(n);return function(u,i,o,a){var c=o*t;null!=u?(u=pr(e,u),i=pr(e,i),(o>0?i>u:u>i)&&(u+=o*La)):(u=n+o*La,i=n-.5*c);for(var l,s=u;o>0?s>i:i>s;s-=c)a.point((l=xe([e,-r*Math.cos(s),-r*Math.sin(s)]))[0],l[1])}}function pr(n,t){var e=pe(t);e[0]-=n,Me(e);var r=nt(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-Ca)%(2*Math.PI)}function vr(n,t,e){var r=ta.range(n,t-Ca,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function dr(n,t,e){var r=ta.range(n,t-Ca,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function mr(n){return n.source}function yr(n){return n.target}function Mr(n,t,e,r){var u=Math.cos(t),i=Math.sin(t),o=Math.cos(r),a=Math.sin(r),c=u*Math.cos(n),l=u*Math.sin(n),s=o*Math.cos(e),f=o*Math.sin(e),h=2*Math.asin(Math.sqrt(it(r-t)+u*o*it(e-n))),g=1/Math.sin(h),p=h?function(n){var t=Math.sin(n*=h)*g,e=Math.sin(h-n)*g,r=e*c+t*s,u=e*l+t*f,o=e*i+t*a;return[Math.atan2(u,r)*Pa,Math.atan2(o,Math.sqrt(r*r+u*u))*Pa]}:function(){return[n*Pa,t*Pa]};return p.distance=h,p}function xr(){function n(n,u){var i=Math.sin(u*=Da),o=Math.cos(u),a=ga((n*=Da)-t),c=Math.cos(a);Yc+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*i-e*o*c)*a),e*i+r*o*c),t=n,e=i,r=o}var t,e,r;Zc.point=function(u,i){t=u*Da,e=Math.sin(i*=Da),r=Math.cos(i),Zc.point=n},Zc.lineEnd=function(){Zc.point=Zc.lineEnd=b}}function br(n,t){function e(t,e){var r=Math.cos(t),u=Math.cos(e),i=n(r*u);return[i*u*Math.sin(t),i*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),u=t(r),i=Math.sin(u),o=Math.cos(u);return[Math.atan2(n*i,r*o),Math.asin(r&&e*i/r)]},e}function _r(n,t){function e(n,t){o>0?-Ra+Ca>t&&(t=-Ra+Ca):t>Ra-Ca&&(t=Ra-Ca);var e=o/Math.pow(u(t),i);return[e*Math.sin(i*n),o-e*Math.cos(i*n)]}var r=Math.cos(n),u=function(n){return Math.tan(qa/4+n/2)},i=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(u(t)/u(n)),o=r*Math.pow(u(n),i)/i;return i?(e.invert=function(n,t){var e=o-t,r=K(i)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/i,2*Math.atan(Math.pow(o/r,1/i))-Ra]},e):Sr}function wr(n,t){function e(n,t){var e=i-t;return[e*Math.sin(u*n),i-e*Math.cos(u*n)]}var r=Math.cos(n),u=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),i=r/u+n;return ga(u)u;u++){for(;r>1&&Q(n[e[r-2]],n[e[r-1]],n[u])<=0;)--r;e[r++]=u}return e.slice(0,r)}function zr(n,t){return n[0]-t[0]||n[1]-t[1]}function qr(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function Lr(n,t,e,r){var u=n[0],i=e[0],o=t[0]-u,a=r[0]-i,c=n[1],l=e[1],s=t[1]-c,f=r[1]-l,h=(a*(c-l)-f*(u-i))/(f*o-a*s);return[u+h*o,c+h*s]}function Tr(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function Rr(){tu(this),this.edge=this.site=this.circle=null}function Dr(n){var t=el.pop()||new Rr;return t.site=n,t}function Pr(n){Xr(n),Qc.remove(n),el.push(n),tu(n)}function Ur(n){var t=n.circle,e=t.x,r=t.cy,u={x:e,y:r},i=n.P,o=n.N,a=[n];Pr(n);for(var c=i;c.circle&&ga(e-c.circle.x)s;++s)l=a[s],c=a[s-1],Kr(l.edge,c.site,l.site,u);c=a[0],l=a[f-1],l.edge=Jr(c.site,l.site,null,u),Vr(c),Vr(l)}function jr(n){for(var t,e,r,u,i=n.x,o=n.y,a=Qc._;a;)if(r=Fr(a,o)-i,r>Ca)a=a.L;else{if(u=i-Hr(a,o),!(u>Ca)){r>-Ca?(t=a.P,e=a):u>-Ca?(t=a,e=a.N):t=e=a;break}if(!a.R){t=a;break}a=a.R}var c=Dr(n);if(Qc.insert(t,c),t||e){if(t===e)return Xr(t),e=Dr(t.site),Qc.insert(c,e),c.edge=e.edge=Jr(t.site,c.site),Vr(t),void Vr(e);if(!e)return void(c.edge=Jr(t.site,c.site));Xr(t),Xr(e);var l=t.site,s=l.x,f=l.y,h=n.x-s,g=n.y-f,p=e.site,v=p.x-s,d=p.y-f,m=2*(h*d-g*v),y=h*h+g*g,M=v*v+d*d,x={x:(d*y-g*M)/m+s,y:(h*M-v*y)/m+f};Kr(e.edge,l,p,x),c.edge=Jr(l,n,null,x),e.edge=Jr(n,p,null,x),Vr(t),Vr(e)}}function Fr(n,t){var e=n.site,r=e.x,u=e.y,i=u-t;if(!i)return r;var o=n.P;if(!o)return-1/0;e=o.site;var a=e.x,c=e.y,l=c-t;if(!l)return a;var s=a-r,f=1/i-1/l,h=s/l;return f?(-h+Math.sqrt(h*h-2*f*(s*s/(-2*l)-c+l/2+u-i/2)))/f+r:(r+a)/2}function Hr(n,t){var e=n.N;if(e)return Fr(e,t);var r=n.site;return r.y===t?r.x:1/0}function Or(n){this.site=n,this.edges=[]}function Ir(n){for(var t,e,r,u,i,o,a,c,l,s,f=n[0][0],h=n[1][0],g=n[0][1],p=n[1][1],v=Kc,d=v.length;d--;)if(i=v[d],i&&i.prepare())for(a=i.edges,c=a.length,o=0;c>o;)s=a[o].end(),r=s.x,u=s.y,l=a[++o%c].start(),t=l.x,e=l.y,(ga(r-t)>Ca||ga(u-e)>Ca)&&(a.splice(o,0,new Qr(Gr(i.site,s,ga(r-f)Ca?{x:f,y:ga(t-f)Ca?{x:ga(e-p)Ca?{x:h,y:ga(t-h)Ca?{x:ga(e-g)=-za)){var g=c*c+l*l,p=s*s+f*f,v=(f*g-l*p)/h,d=(c*p-s*g)/h,f=d+a,m=rl.pop()||new Zr;m.arc=n,m.site=u,m.x=v+o,m.y=f+Math.sqrt(v*v+d*d),m.cy=f,n.circle=m;for(var y=null,M=tl._;M;)if(m.yd||d>=a)return;if(h>p){if(i){if(i.y>=l)return}else i={x:d,y:c};e={x:d,y:l}}else{if(i){if(i.yr||r>1)if(h>p){if(i){if(i.y>=l)return}else i={x:(c-u)/r,y:c};e={x:(l-u)/r,y:l}}else{if(i){if(i.yg){if(i){if(i.x>=a)return}else i={x:o,y:r*o+u};e={x:a,y:r*a+u}}else{if(i){if(i.xi||f>o||r>h||u>g)){if(p=n.point){var p,v=t-n.x,d=e-n.y,m=v*v+d*d;if(c>m){var y=Math.sqrt(c=m);r=t-y,u=e-y,i=t+y,o=e+y,a=p}}for(var M=n.nodes,x=.5*(s+h),b=.5*(f+g),_=t>=x,w=e>=b,S=w<<1|_,k=S+4;k>S;++S)if(n=M[3&S])switch(3&S){case 0:l(n,s,f,x,b);break;case 1:l(n,x,f,h,b);break;case 2:l(n,s,b,x,g);break;case 3:l(n,x,b,h,g)}}}(n,r,u,i,o),a}function gu(n,t){n=ta.rgb(n),t=ta.rgb(t);var e=n.r,r=n.g,u=n.b,i=t.r-e,o=t.g-r,a=t.b-u;return function(n){return"#"+xt(Math.round(e+i*n))+xt(Math.round(r+o*n))+xt(Math.round(u+a*n))}}function pu(n,t){var e,r={},u={};for(e in n)e in t?r[e]=mu(n[e],t[e]):u[e]=n[e];for(e in t)e in n||(u[e]=t[e]);return function(n){for(e in r)u[e]=r[e](n);return u}}function vu(n,t){return n=+n,t=+t,function(e){return n*(1-e)+t*e}}function du(n,t){var e,r,u,i=il.lastIndex=ol.lastIndex=0,o=-1,a=[],c=[];for(n+="",t+="";(e=il.exec(n))&&(r=ol.exec(t));)(u=r.index)>i&&(u=t.slice(i,u),a[o]?a[o]+=u:a[++o]=u),(e=e[0])===(r=r[0])?a[o]?a[o]+=r:a[++o]=r:(a[++o]=null,c.push({i:o,x:vu(e,r)})),i=ol.lastIndex;return ir;++r)a[(e=c[r]).i]=e.x(n);return a.join("")})}function mu(n,t){for(var e,r=ta.interpolators.length;--r>=0&&!(e=ta.interpolators[r](n,t)););return e}function yu(n,t){var e,r=[],u=[],i=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e;++e)r.push(mu(n[e],t[e]));for(;i>e;++e)u[e]=n[e];for(;o>e;++e)u[e]=t[e];return function(n){for(e=0;a>e;++e)u[e]=r[e](n);return u}}function Mu(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function xu(n){return function(t){return 1-n(1-t)}}function bu(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function _u(n){return n*n}function wu(n){return n*n*n}function Su(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function ku(n){return function(t){return Math.pow(t,n)}}function Eu(n){return 1-Math.cos(n*Ra)}function Au(n){return Math.pow(2,10*(n-1))}function Nu(n){return 1-Math.sqrt(1-n*n)}function Cu(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/La*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*La/t)}}function zu(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function qu(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)*n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function Lu(n,t){n=ta.hcl(n),t=ta.hcl(t);var e=n.h,r=n.c,u=n.l,i=t.h-e,o=t.c-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return st(e+i*n,r+o*n,u+a*n)+""}}function Tu(n,t){n=ta.hsl(n),t=ta.hsl(t);var e=n.h,r=n.s,u=n.l,i=t.h-e,o=t.s-r,a=t.l-u;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(i)?(i=0,e=isNaN(e)?t.h:e):i>180?i-=360:-180>i&&(i+=360),function(n){return ct(e+i*n,r+o*n,u+a*n)+""}}function Ru(n,t){n=ta.lab(n),t=ta.lab(t);var e=n.l,r=n.a,u=n.b,i=t.l-e,o=t.a-r,a=t.b-u;return function(n){return ht(e+i*n,r+o*n,u+a*n)+""}}function Du(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function Pu(n){var t=[n.a,n.b],e=[n.c,n.d],r=ju(t),u=Uu(t,e),i=ju(Fu(e,t,-u))||0;t[0]*e[1]180?s+=360:s-l>180&&(l+=360),u.push({i:r.push(r.pop()+"rotate(",null,")")-2,x:vu(l,s)})):s&&r.push(r.pop()+"rotate("+s+")"),f!=h?u.push({i:r.push(r.pop()+"skewX(",null,")")-2,x:vu(f,h)}):h&&r.push(r.pop()+"skewX("+h+")"),g[0]!=p[0]||g[1]!=p[1]?(e=r.push(r.pop()+"scale(",null,",",null,")"),u.push({i:e-4,x:vu(g[0],p[0])},{i:e-2,x:vu(g[1],p[1])})):(1!=p[0]||1!=p[1])&&r.push(r.pop()+"scale("+p+")"),e=u.length,function(n){for(var t,i=-1;++i=0;)e.push(u[r])}function Qu(n,t){for(var e=[n],r=[];null!=(n=e.pop());)if(r.push(n),(i=n.children)&&(u=i.length))for(var u,i,o=-1;++oe;++e)(t=n[e][1])>u&&(r=e,u=t);return r}function si(n){return n.reduce(fi,0)}function fi(n,t){return n+t[1]}function hi(n,t){return gi(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function gi(n,t){for(var e=-1,r=+n[0],u=(n[1]-r)/t,i=[];++e<=t;)i[e]=u*e+r;return i}function pi(n){return[ta.min(n),ta.max(n)]}function vi(n,t){return n.value-t.value}function di(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function mi(n,t){n._pack_next=t,t._pack_prev=n}function yi(n,t){var e=t.x-n.x,r=t.y-n.y,u=n.r+t.r;return.999*u*u>e*e+r*r}function Mi(n){function t(n){s=Math.min(n.x-n.r,s),f=Math.max(n.x+n.r,f),h=Math.min(n.y-n.r,h),g=Math.max(n.y+n.r,g)}if((e=n.children)&&(l=e.length)){var e,r,u,i,o,a,c,l,s=1/0,f=-1/0,h=1/0,g=-1/0;if(e.forEach(xi),r=e[0],r.x=-r.r,r.y=0,t(r),l>1&&(u=e[1],u.x=u.r,u.y=0,t(u),l>2))for(i=e[2],wi(r,u,i),t(i),di(r,i),r._pack_prev=i,di(i,u),u=r._pack_next,o=3;l>o;o++){wi(r,u,i=e[o]);var p=0,v=1,d=1;for(a=u._pack_next;a!==u;a=a._pack_next,v++)if(yi(a,i)){p=1;break}if(1==p)for(c=r._pack_prev;c!==a._pack_prev&&!yi(c,i);c=c._pack_prev,d++);p?(d>v||v==d&&u.ro;o++)i=e[o],i.x-=m,i.y-=y,M=Math.max(M,i.r+Math.sqrt(i.x*i.x+i.y*i.y));n.r=M,e.forEach(bi)}}function xi(n){n._pack_next=n._pack_prev=n}function bi(n){delete n._pack_next,delete n._pack_prev}function _i(n,t,e,r){var u=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,u)for(var i=-1,o=u.length;++i=0;)t=u[i],t.z+=e,t.m+=e,e+=t.s+(r+=t.c)}function Ci(n,t,e){return n.a.parent===t.parent?n.a:e}function zi(n){return 1+ta.max(n,function(n){return n.y})}function qi(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Li(n){var t=n.children;return t&&t.length?Li(t[0]):n}function Ti(n){var t,e=n.children;return e&&(t=e.length)?Ti(e[t-1]):n}function Ri(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function Di(n,t){var e=n.x+t[3],r=n.y+t[0],u=n.dx-t[1]-t[3],i=n.dy-t[0]-t[2];return 0>u&&(e+=u/2,u=0),0>i&&(r+=i/2,i=0),{x:e,y:r,dx:u,dy:i}}function Pi(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function Ui(n){return n.rangeExtent?n.rangeExtent():Pi(n.range())}function ji(n,t,e,r){var u=e(n[0],n[1]),i=r(t[0],t[1]);return function(n){return i(u(n))}}function Fi(n,t){var e,r=0,u=n.length-1,i=n[r],o=n[u];return i>o&&(e=r,r=u,u=e,e=i,i=o,o=e),n[r]=t.floor(i),n[u]=t.ceil(o),n}function Hi(n){return n?{floor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:ml}function Oi(n,t,e,r){var u=[],i=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]2?Oi:ji,c=r?Iu:Ou;return o=u(n,t,c,e),a=u(t,n,c,mu),i}function i(n){return o(n)}var o,a;return i.invert=function(n){return a(n)},i.domain=function(t){return arguments.length?(n=t.map(Number),u()):n},i.range=function(n){return arguments.length?(t=n,u()):t},i.rangeRound=function(n){return i.range(n).interpolate(Du)},i.clamp=function(n){return arguments.length?(r=n,u()):r},i.interpolate=function(n){return arguments.length?(e=n,u()):e},i.ticks=function(t){return Xi(n,t)},i.tickFormat=function(t,e){return $i(n,t,e)},i.nice=function(t){return Zi(n,t),u()},i.copy=function(){return Ii(n,t,e,r)},u()}function Yi(n,t){return ta.rebind(n,t,"range","rangeRound","interpolate","clamp")}function Zi(n,t){return Fi(n,Hi(Vi(n,t)[2]))}function Vi(n,t){null==t&&(t=10);var e=Pi(n),r=e[1]-e[0],u=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),i=t/r*u;return.15>=i?u*=10:.35>=i?u*=5:.75>=i&&(u*=2),e[0]=Math.ceil(e[0]/u)*u,e[1]=Math.floor(e[1]/u)*u+.5*u,e[2]=u,e}function Xi(n,t){return ta.range.apply(ta,Vi(n,t))}function $i(n,t,e){var r=Vi(n,t);if(e){var u=ic.exec(e);if(u.shift(),"s"===u[8]){var i=ta.formatPrefix(Math.max(ga(r[0]),ga(r[1])));return u[7]||(u[7]="."+Bi(i.scale(r[2]))),u[8]="f",e=ta.format(u.join("")),function(n){return e(i.scale(n))+i.symbol}}u[7]||(u[7]="."+Wi(u[8],r)),e=u.join("")}else e=",."+Bi(r[2])+"f";return ta.format(e)}function Bi(n){return-Math.floor(Math.log(n)/Math.LN10+.01)}function Wi(n,t){var e=Bi(t[2]);return n in yl?Math.abs(e-Bi(Math.max(ga(t[0]),ga(t[1]))))+ +("e"!==n):e-2*("%"===n)}function Ji(n,t,e,r){function u(n){return(e?Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function i(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(u(t))}return o.invert=function(t){return i(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(u)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(u)),o):t},o.nice=function(){var t=Fi(r.map(u),e?Math:xl);return n.domain(t),r=t.map(i),o},o.ticks=function(){var n=Pi(r),o=[],a=n[0],c=n[1],l=Math.floor(u(a)),s=Math.ceil(u(c)),f=t%1?2:t;if(isFinite(s-l)){if(e){for(;s>l;l++)for(var h=1;f>h;h++)o.push(i(l)*h);o.push(i(l))}else for(o.push(i(l));l++0;h--)o.push(i(l)*h);for(l=0;o[l]c;s--);o=o.slice(l,s)}return o},o.tickFormat=function(n,t){if(!arguments.length)return Ml;arguments.length<2?t=Ml:"function"!=typeof t&&(t=ta.format(t));var r,a=Math.max(.1,n/o.ticks().length),c=e?(r=1e-12,Math.ceil):(r=-1e-12,Math.floor);return function(n){return n/i(c(u(n)+r))<=a?t(n):""}},o.copy=function(){return Ji(n.copy(),t,e,r)},Yi(o,n)}function Gi(n,t,e){function r(t){return n(u(t))}var u=Ki(t),i=Ki(1/t);return r.invert=function(t){return i(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(u)),r):e},r.ticks=function(n){return Xi(e,n)},r.tickFormat=function(n,t){return $i(e,n,t)},r.nice=function(n){return r.domain(Zi(e,n))},r.exponent=function(o){return arguments.length?(u=Ki(t=o),i=Ki(1/t),n.domain(e.map(u)),r):t},r.copy=function(){return Gi(n.copy(),t,e)},Yi(r,n)}function Ki(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function Qi(n,t){function e(e){return i[((u.get(e)||("range"===t.t?u.set(e,n.push(e)):0/0))-1)%i.length]}function r(t,e){return ta.range(n.length).map(function(n){return t+e*n})}var u,i,o;return e.domain=function(r){if(!arguments.length)return n;n=[],u=new l;for(var i,o=-1,a=r.length;++oe?[0/0,0/0]:[e>0?a[e-1]:n[0],et?0/0:t/i+n,[t,t+1/i]},r.copy=function(){return to(n,t,e)},u()}function eo(n,t){function e(e){return e>=e?t[ta.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return eo(n,t)},e}function ro(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return Xi(n,t)},t.tickFormat=function(t,e){return $i(n,t,e)},t.copy=function(){return ro(n)},t}function uo(){return 0}function io(n){return n.innerRadius}function oo(n){return n.outerRadius}function ao(n){return n.startAngle}function co(n){return n.endAngle}function lo(n){return n&&n.padAngle}function so(n,t,e,r){return(n-e)*t-(t-r)*n>0?0:1}function fo(n,t,e,r,u){var i=n[0]-t[0],o=n[1]-t[1],a=(u?r:-r)/Math.sqrt(i*i+o*o),c=a*o,l=-a*i,s=n[0]+c,f=n[1]+l,h=t[0]+c,g=t[1]+l,p=(s+h)/2,v=(f+g)/2,d=h-s,m=g-f,y=d*d+m*m,M=e-r,x=s*g-h*f,b=(0>m?-1:1)*Math.sqrt(M*M*y-x*x),_=(x*m-d*b)/y,w=(-x*d-m*b)/y,S=(x*m+d*b)/y,k=(-x*d+m*b)/y,E=_-p,A=w-v,N=S-p,C=k-v;return E*E+A*A>N*N+C*C&&(_=S,w=k),[[_-c,w-l],[_*e/M,w*e/M]]}function ho(n){function t(t){function o(){l.push("M",i(n(s),a))}for(var c,l=[],s=[],f=-1,h=t.length,g=Et(e),p=Et(r);++f1&&u.push("H",r[0]),u.join("")}function mo(n){for(var t=0,e=n.length,r=n[0],u=[r[0],",",r[1]];++t1){a=t[1],i=n[c],c++,r+="C"+(u[0]+o[0])+","+(u[1]+o[1])+","+(i[0]-a[0])+","+(i[1]-a[1])+","+i[0]+","+i[1];for(var l=2;l9&&(u=3*t/Math.sqrt(u),o[a]=u*e,o[a+1]=u*r));for(a=-1;++a<=c;)u=(n[Math.min(c,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),i.push([u||0,o[a]*u||0]);return i}function To(n){return n.length<3?go(n):n[0]+_o(n,Lo(n))}function Ro(n){for(var t,e,r,u=-1,i=n.length;++ur)return s();var u=i[i.active];u&&(--i.count,delete i[i.active],u.event&&u.event.interrupt.call(n,n.__data__,u.index)),i.active=r,o.event&&o.event.start.call(n,n.__data__,t),o.tween.forEach(function(e,r){(r=r.call(n,n.__data__,t))&&v.push(r)}),h=o.ease,f=o.duration,ta.timer(function(){return p.c=l(e||1)?Ne:l,1},0,a)}function l(e){if(i.active!==r)return 1;for(var u=e/f,a=h(u),c=v.length;c>0;)v[--c].call(n,a);return u>=1?(o.event&&o.event.end.call(n,n.__data__,t),s()):void 0}function s(){return--i.count?delete i[r]:delete n[e],1}var f,h,g=o.delay,p=ec,v=[];return p.t=g+a,u>=g?c(u-g):void(p.c=c)},0,a)}}function Bo(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate("+(isFinite(r)?r:e(n))+",0)"})}function Wo(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate(0,"+(isFinite(r)?r:e(n))+")"})}function Jo(n){return n.toISOString()}function Go(n,t,e){function r(t){return n(t)}function u(n,e){var r=n[1]-n[0],u=r/e,i=ta.bisect(Vl,u);return i==Vl.length?[t.year,Vi(n.map(function(n){return n/31536e6}),e)[2]]:i?t[u/Vl[i-1]1?{floor:function(t){for(;e(t=n.floor(t));)t=Ko(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=Ko(+t+1);return t}}:n))},r.ticks=function(n,t){var e=Pi(r.domain()),i=null==n?u(e,10):"number"==typeof n?u(e,n):!n.range&&[{range:n},t];return i&&(n=i[0],t=i[1]),n.range(e[0],Ko(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return Go(n.copy(),t,e)},Yi(r,n)}function Ko(n){return new Date(n)}function Qo(n){return JSON.parse(n.responseText)}function na(n){var t=ua.createRange();return t.selectNode(ua.body),t.createContextualFragment(n.responseText)}var ta={version:"3.5.6"},ea=[].slice,ra=function(n){return ea.call(n)},ua=this.document;if(ua)try{ra(ua.documentElement.childNodes)[0].nodeType}catch(ia){ra=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}if(Date.now||(Date.now=function(){return+new Date}),ua)try{ua.createElement("DIV").style.setProperty("opacity",0,"")}catch(oa){var aa=this.Element.prototype,ca=aa.setAttribute,la=aa.setAttributeNS,sa=this.CSSStyleDeclaration.prototype,fa=sa.setProperty;aa.setAttribute=function(n,t){ca.call(this,n,t+"")},aa.setAttributeNS=function(n,t,e){la.call(this,n,t,e+"")},sa.setProperty=function(n,t,e){fa.call(this,n,t+"",e)}}ta.ascending=e,ta.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:0/0},ta.min=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u=r){e=r;break}for(;++ur&&(e=r)}else{for(;++u=r){e=r;break}for(;++ur&&(e=r)}return e},ta.max=function(n,t){var e,r,u=-1,i=n.length;if(1===arguments.length){for(;++u=r){e=r;break}for(;++ue&&(e=r)}else{for(;++u=r){e=r;break}for(;++ue&&(e=r)}return e},ta.extent=function(n,t){var e,r,u,i=-1,o=n.length;if(1===arguments.length){for(;++i=r){e=u=r;break}for(;++ir&&(e=r),r>u&&(u=r))}else{for(;++i=r){e=u=r;break}for(;++ir&&(e=r),r>u&&(u=r))}return[e,u]},ta.sum=function(n,t){var e,r=0,i=n.length,o=-1;if(1===arguments.length)for(;++o1?c/(s-1):void 0},ta.deviation=function(){var n=ta.variance.apply(this,arguments);return n?Math.sqrt(n):n};var ha=i(e);ta.bisectLeft=ha.left,ta.bisect=ta.bisectRight=ha.right,ta.bisector=function(n){return i(1===n.length?function(t,r){return e(n(t),r)}:n)},ta.shuffle=function(n,t,e){(i=arguments.length)<3&&(e=n.length,2>i&&(t=0));for(var r,u,i=e-t;i;)u=Math.random()*i--|0,r=n[i+t],n[i+t]=n[u+t],n[u+t]=r;return n},ta.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},ta.pairs=function(n){for(var t,e=0,r=n.length-1,u=n[0],i=new Array(0>r?0:r);r>e;)i[e]=[t=u,u=n[++e]];return i},ta.zip=function(){if(!(r=arguments.length))return[];for(var n=-1,t=ta.min(arguments,o),e=new Array(t);++n=0;)for(r=n[u],t=r.length;--t>=0;)e[--o]=r[t];return e};var ga=Math.abs;ta.range=function(n,t,e){if(arguments.length<3&&(e=1,arguments.length<2&&(t=n,n=0)),(t-n)/e===1/0)throw new Error("infinite range");var r,u=[],i=a(ga(e)),o=-1;if(n*=i,t*=i,e*=i,0>e)for(;(r=n+e*++o)>t;)u.push(r/i);else for(;(r=n+e*++o)=i.length)return r?r.call(u,o):e?o.sort(e):o;for(var c,s,f,h,g=-1,p=o.length,v=i[a++],d=new l;++g=i.length)return n;var r=[],u=o[e++];return n.forEach(function(n,u){r.push({key:n,values:t(u,e)})}),u?r.sort(function(n,t){return u(n.key,t.key)}):r}var e,r,u={},i=[],o=[];return u.map=function(t,e){return n(e,t,0)},u.entries=function(e){return t(n(ta.map,e,0),0)},u.key=function(n){return i.push(n),u},u.sortKeys=function(n){return o[i.length-1]=n,u},u.sortValues=function(n){return e=n,u},u.rollup=function(n){return r=n,u},u},ta.set=function(n){var t=new m;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},c(m,{has:h,add:function(n){return this._[s(n+="")]=!0,n},remove:g,values:p,size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,f(t))}}),ta.behavior={},ta.rebind=function(n,t){for(var e,r=1,u=arguments.length;++r=0&&(r=n.slice(e+1),n=n.slice(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},ta.event=null,ta.requote=function(n){return n.replace(ma,"\\$&")};var ma=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,ya={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},Ma=function(n,t){return t.querySelector(n)},xa=function(n,t){return t.querySelectorAll(n)},ba=function(n,t){var e=n.matches||n[x(n,"matchesSelector")];return(ba=function(n,t){return e.call(n,t)})(n,t)};"function"==typeof Sizzle&&(Ma=function(n,t){return Sizzle(n,t)[0]||null},xa=Sizzle,ba=Sizzle.matchesSelector),ta.selection=function(){return ta.select(ua.documentElement)};var _a=ta.selection.prototype=[];_a.select=function(n){var t,e,r,u,i=[];n=N(n);for(var o=-1,a=this.length;++o=0&&(e=n.slice(0,t),n=n.slice(t+1)),wa.hasOwnProperty(e)?{space:wa[e],local:n}:n}},_a.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=ta.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(z(t,n[t]));return this}return this.each(z(n,t))},_a.classed=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node(),r=(n=T(n)).length,u=-1;if(t=e.classList){for(;++uu){if("string"!=typeof n){2>u&&(e="");for(r in n)this.each(P(r,n[r],e));return this}if(2>u){var i=this.node();return t(i).getComputedStyle(i,null).getPropertyValue(n)}r=""}return this.each(P(n,e,r))},_a.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(U(t,n[t]));return this}return this.each(U(n,t))},_a.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},_a.html=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},_a.append=function(n){return n=j(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},_a.insert=function(n,t){return n=j(n),t=N(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments)||null)})},_a.remove=function(){return this.each(F)},_a.data=function(n,t){function e(n,e){var r,u,i,o=n.length,f=e.length,h=Math.min(o,f),g=new Array(f),p=new Array(f),v=new Array(o);if(t){var d,m=new l,y=new Array(o);for(r=-1;++rr;++r)p[r]=H(e[r]);for(;o>r;++r)v[r]=n[r]}p.update=g,p.parentNode=g.parentNode=v.parentNode=n.parentNode,a.push(p),c.push(g),s.push(v)}var r,u,i=-1,o=this.length;if(!arguments.length){for(n=new Array(o=(r=this[0]).length);++ii;i++){u.push(t=[]),t.parentNode=(e=this[i]).parentNode;for(var a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a,i)&&t.push(r)}return A(u)},_a.order=function(){for(var n=-1,t=this.length;++n=0;)(e=r[u])&&(i&&i!==e.nextSibling&&i.parentNode.insertBefore(e,i),i=e);return this},_a.sort=function(n){n=I.apply(this,arguments);for(var t=-1,e=this.length;++tn;n++)for(var e=this[n],r=0,u=e.length;u>r;r++){var i=e[r];if(i)return i}return null},_a.size=function(){var n=0;return Y(this,function(){++n}),n};var Sa=[];ta.selection.enter=Z,ta.selection.enter.prototype=Sa,Sa.append=_a.append,Sa.empty=_a.empty,Sa.node=_a.node,Sa.call=_a.call,Sa.size=_a.size,Sa.select=function(n){for(var t,e,r,u,i,o=[],a=-1,c=this.length;++ar){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(X(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(X(n,t,e))};var ka=ta.map({mouseenter:"mouseover",mouseleave:"mouseout"});ua&&ka.forEach(function(n){"on"+n in ua&&ka.remove(n)});var Ea,Aa=0;ta.mouse=function(n){return J(n,k())};var Na=this.navigator&&/WebKit/.test(this.navigator.userAgent)?-1:0;ta.touch=function(n,t,e){if(arguments.length<3&&(e=t,t=k().changedTouches),t)for(var r,u=0,i=t.length;i>u;++u)if((r=t[u]).identifier===e)return J(n,r)},ta.behavior.drag=function(){function n(){this.on("mousedown.drag",i).on("touchstart.drag",o)}function e(n,t,e,i,o){return function(){function a(){var n,e,r=t(h,v);r&&(n=r[0]-M[0],e=r[1]-M[1],p|=n|e,M=r,g({type:"drag",x:r[0]+l[0],y:r[1]+l[1],dx:n,dy:e}))}function c(){t(h,v)&&(m.on(i+d,null).on(o+d,null),y(p&&ta.event.target===f),g({type:"dragend"}))}var l,s=this,f=ta.event.target,h=s.parentNode,g=r.of(s,arguments),p=0,v=n(),d=".drag"+(null==v?"":"-"+v),m=ta.select(e(f)).on(i+d,a).on(o+d,c),y=W(f),M=t(h,v);u?(l=u.apply(s,arguments),l=[l.x-M[0],l.y-M[1]]):l=[0,0],g({type:"dragstart"})}}var r=E(n,"drag","dragstart","dragend"),u=null,i=e(b,ta.mouse,t,"mousemove","mouseup"),o=e(G,ta.touch,y,"touchmove","touchend");return n.origin=function(t){return arguments.length?(u=t,n):u},ta.rebind(n,r,"on")},ta.touches=function(n,t){return arguments.length<2&&(t=k().touches),t?ra(t).map(function(t){var e=J(n,t);return e.identifier=t.identifier,e}):[]};var Ca=1e-6,za=Ca*Ca,qa=Math.PI,La=2*qa,Ta=La-Ca,Ra=qa/2,Da=qa/180,Pa=180/qa,Ua=Math.SQRT2,ja=2,Fa=4;ta.interpolateZoom=function(n,t){function e(n){var t=n*y;if(m){var e=rt(v),o=i/(ja*h)*(e*ut(Ua*t+v)-et(v));return[r+o*l,u+o*s,i*e/rt(Ua*t+v)]}return[r+n*l,u+n*s,i*Math.exp(Ua*t)]}var r=n[0],u=n[1],i=n[2],o=t[0],a=t[1],c=t[2],l=o-r,s=a-u,f=l*l+s*s,h=Math.sqrt(f),g=(c*c-i*i+Fa*f)/(2*i*ja*h),p=(c*c-i*i-Fa*f)/(2*c*ja*h),v=Math.log(Math.sqrt(g*g+1)-g),d=Math.log(Math.sqrt(p*p+1)-p),m=d-v,y=(m||Math.log(c/i))/Ua;return e.duration=1e3*y,e},ta.behavior.zoom=function(){function n(n){n.on(q,f).on(Oa+".zoom",g).on("dblclick.zoom",p).on(R,h)}function e(n){return[(n[0]-k.x)/k.k,(n[1]-k.y)/k.k]}function r(n){return[n[0]*k.k+k.x,n[1]*k.k+k.y]}function u(n){k.k=Math.max(N[0],Math.min(N[1],n))}function i(n,t){t=r(t),k.x+=n[0]-t[0],k.y+=n[1]-t[1]}function o(t,e,r,o){t.__chart__={x:k.x,y:k.y,k:k.k},u(Math.pow(2,o)),i(d=e,r),t=ta.select(t),C>0&&(t=t.transition().duration(C)),t.call(n.event)}function a(){b&&b.domain(x.range().map(function(n){return(n-k.x)/k.k}).map(x.invert)),w&&w.domain(_.range().map(function(n){return(n-k.y)/k.k}).map(_.invert))}function c(n){z++||n({type:"zoomstart"})}function l(n){a(),n({type:"zoom",scale:k.k,translate:[k.x,k.y]})}function s(n){--z||(n({type:"zoomend"}),d=null)}function f(){function n(){f=1,i(ta.mouse(u),g),l(a)}function r(){h.on(L,null).on(T,null),p(f&&ta.event.target===o),s(a)}var u=this,o=ta.event.target,a=D.of(u,arguments),f=0,h=ta.select(t(u)).on(L,n).on(T,r),g=e(ta.mouse(u)),p=W(u);Dl.call(u),c(a)}function h(){function n(){var n=ta.touches(p);return g=k.k,n.forEach(function(n){n.identifier in d&&(d[n.identifier]=e(n))}),n}function t(){var t=ta.event.target;ta.select(t).on(x,r).on(b,a),_.push(t);for(var e=ta.event.changedTouches,u=0,i=e.length;i>u;++u)d[e[u].identifier]=null;var c=n(),l=Date.now();if(1===c.length){if(500>l-M){var s=c[0];o(p,s,d[s.identifier],Math.floor(Math.log(k.k)/Math.LN2)+1),S()}M=l}else if(c.length>1){var s=c[0],f=c[1],h=s[0]-f[0],g=s[1]-f[1];m=h*h+g*g}}function r(){var n,t,e,r,o=ta.touches(p);Dl.call(p);for(var a=0,c=o.length;c>a;++a,r=null)if(e=o[a],r=d[e.identifier]){if(t)break;n=e,t=r}if(r){var s=(s=e[0]-n[0])*s+(s=e[1]-n[1])*s,f=m&&Math.sqrt(s/m);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+r[0])/2,(t[1]+r[1])/2],u(f*g)}M=null,i(n,t),l(v)}function a(){if(ta.event.touches.length){for(var t=ta.event.changedTouches,e=0,r=t.length;r>e;++e)delete d[t[e].identifier];for(var u in d)return void n()}ta.selectAll(_).on(y,null),w.on(q,f).on(R,h),E(),s(v)}var g,p=this,v=D.of(p,arguments),d={},m=0,y=".zoom-"+ta.event.changedTouches[0].identifier,x="touchmove"+y,b="touchend"+y,_=[],w=ta.select(p),E=W(p);t(),c(v),w.on(q,null).on(R,t)}function g(){var n=D.of(this,arguments);y?clearTimeout(y):(Dl.call(this),v=e(d=m||ta.mouse(this)),c(n)),y=setTimeout(function(){y=null,s(n)},50),S(),u(Math.pow(2,.002*Ha())*k.k),i(d,v),l(n)}function p(){var n=ta.mouse(this),t=Math.log(k.k)/Math.LN2;o(this,n,e(n),ta.event.shiftKey?Math.ceil(t)-1:Math.floor(t)+1)}var v,d,m,y,M,x,b,_,w,k={x:0,y:0,k:1},A=[960,500],N=Ia,C=250,z=0,q="mousedown.zoom",L="mousemove.zoom",T="mouseup.zoom",R="touchstart.zoom",D=E(n,"zoomstart","zoom","zoomend");return Oa||(Oa="onwheel"in ua?(Ha=function(){return-ta.event.deltaY*(ta.event.deltaMode?120:1)},"wheel"):"onmousewheel"in ua?(Ha=function(){return ta.event.wheelDelta},"mousewheel"):(Ha=function(){return-ta.event.detail},"MozMousePixelScroll")),n.event=function(n){n.each(function(){var n=D.of(this,arguments),t=k;Tl?ta.select(this).transition().each("start.zoom",function(){k=this.__chart__||{x:0,y:0,k:1},c(n)}).tween("zoom:zoom",function(){var e=A[0],r=A[1],u=d?d[0]:e/2,i=d?d[1]:r/2,o=ta.interpolateZoom([(u-k.x)/k.k,(i-k.y)/k.k,e/k.k],[(u-t.x)/t.k,(i-t.y)/t.k,e/t.k]);return function(t){var r=o(t),a=e/r[2];this.__chart__=k={x:u-r[0]*a,y:i-r[1]*a,k:a},l(n)}}).each("interrupt.zoom",function(){s(n)}).each("end.zoom",function(){s(n)}):(this.__chart__=k,c(n),l(n),s(n))})},n.translate=function(t){return arguments.length?(k={x:+t[0],y:+t[1],k:k.k},a(),n):[k.x,k.y]},n.scale=function(t){return arguments.length?(k={x:k.x,y:k.y,k:+t},a(),n):k.k},n.scaleExtent=function(t){return arguments.length?(N=null==t?Ia:[+t[0],+t[1]],n):N},n.center=function(t){return arguments.length?(m=t&&[+t[0],+t[1]],n):m},n.size=function(t){return arguments.length?(A=t&&[+t[0],+t[1]],n):A},n.duration=function(t){return arguments.length?(C=+t,n):C},n.x=function(t){return arguments.length?(b=t,x=t.copy(),k={x:0,y:0,k:1},n):b},n.y=function(t){return arguments.length?(w=t,_=t.copy(),k={x:0,y:0,k:1},n):w},ta.rebind(n,D,"on")};var Ha,Oa,Ia=[0,1/0];ta.color=ot,ot.prototype.toString=function(){return this.rgb()+""},ta.hsl=at;var Ya=at.prototype=new ot;Ya.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),new at(this.h,this.s,this.l/n)},Ya.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new at(this.h,this.s,n*this.l)},Ya.rgb=function(){return ct(this.h,this.s,this.l)},ta.hcl=lt;var Za=lt.prototype=new ot;Za.brighter=function(n){return new lt(this.h,this.c,Math.min(100,this.l+Va*(arguments.length?n:1)))},Za.darker=function(n){return new lt(this.h,this.c,Math.max(0,this.l-Va*(arguments.length?n:1)))},Za.rgb=function(){return st(this.h,this.c,this.l).rgb()},ta.lab=ft;var Va=18,Xa=.95047,$a=1,Ba=1.08883,Wa=ft.prototype=new ot;Wa.brighter=function(n){return new ft(Math.min(100,this.l+Va*(arguments.length?n:1)),this.a,this.b)},Wa.darker=function(n){return new ft(Math.max(0,this.l-Va*(arguments.length?n:1)),this.a,this.b)},Wa.rgb=function(){return ht(this.l,this.a,this.b)},ta.rgb=mt;var Ja=mt.prototype=new ot;Ja.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,u=30;return t||e||r?(t&&u>t&&(t=u),e&&u>e&&(e=u),r&&u>r&&(r=u),new mt(Math.min(255,t/n),Math.min(255,e/n),Math.min(255,r/n))):new mt(u,u,u)},Ja.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new mt(n*this.r,n*this.g,n*this.b)},Ja.hsl=function(){return _t(this.r,this.g,this.b)},Ja.toString=function(){return"#"+xt(this.r)+xt(this.g)+xt(this.b)};var Ga=ta.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794160,hotpink:16738740,indianred:13458524,indigo:4915330,ivory:16777200,khaki:15787660,lavender:15132410,lavenderblush:16773365,lawngreen:8190976,lemonchiffon:16775885,lightblue:11393254,lightcoral:15761536,lightcyan:14745599,lightgoldenrodyellow:16448210,lightgray:13882323,lightgreen:9498256,lightgrey:13882323,lightpink:16758465,lightsalmon:16752762,lightseagreen:2142890,lightskyblue:8900346,lightslategray:7833753,lightslategrey:7833753,lightsteelblue:11584734,lightyellow:16777184,lime:65280,limegreen:3329330,linen:16445670,magenta:16711935,maroon:8388608,mediumaquamarine:6737322,mediumblue:205,mediumorchid:12211667,mediumpurple:9662683,mediumseagreen:3978097,mediumslateblue:8087790,mediumspringgreen:64154,mediumturquoise:4772300,mediumvioletred:13047173,midnightblue:1644912,mintcream:16121850,mistyrose:16770273,moccasin:16770229,navajowhite:16768685,navy:128,oldlace:16643558,olive:8421376,olivedrab:7048739,orange:16753920,orangered:16729344,orchid:14315734,palegoldenrod:15657130,palegreen:10025880,paleturquoise:11529966,palevioletred:14381203,papayawhip:16773077,peachpuff:16767673,peru:13468991,pink:16761035,plum:14524637,powderblue:11591910,purple:8388736,rebeccapurple:6697881,red:16711680,rosybrown:12357519,royalblue:4286945,saddlebrown:9127187,salmon:16416882,sandybrown:16032864,seagreen:3050327,seashell:16774638,sienna:10506797,silver:12632256,skyblue:8900331,slateblue:6970061,slategray:7372944,slategrey:7372944,snow:16775930,springgreen:65407,steelblue:4620980,tan:13808780,teal:32896,thistle:14204888,tomato:16737095,turquoise:4251856,violet:15631086,wheat:16113331,white:16777215,whitesmoke:16119285,yellow:16776960,yellowgreen:10145074});Ga.forEach(function(n,t){Ga.set(n,yt(t))}),ta.functor=Et,ta.xhr=At(y),ta.dsv=function(n,t){function e(n,e,i){arguments.length<3&&(i=e,e=null);var o=Nt(n,t,null==e?r:u(e),i);return o.row=function(n){return arguments.length?o.response(null==(e=n)?r:u(n)):e},o}function r(n){return e.parse(n.responseText)}function u(n){return function(t){return e.parse(t.responseText,n)}}function i(t){return t.map(o).join(n)}function o(n){return a.test(n)?'"'+n.replace(/\"/g,'""')+'"':n}var a=new RegExp('["'+n+"\n]"),c=n.charCodeAt(0);return e.parse=function(n,t){var r;return e.parseRows(n,function(n,e){if(r)return r(n,e-1);var u=new Function("d","return {"+n.map(function(n,t){return JSON.stringify(n)+": d["+t+"]"}).join(",")+"}");r=t?function(n,e){return t(u(n),e)}:u})},e.parseRows=function(n,t){function e(){if(s>=l)return o;if(u)return u=!1,i;var t=s;if(34===n.charCodeAt(t)){for(var e=t;e++s;){var r=n.charCodeAt(s++),a=1;if(10===r)u=!0;else if(13===r)u=!0,10===n.charCodeAt(s)&&(++s,++a);else if(r!==c)continue;return n.slice(t,s-a)}return n.slice(t)}for(var r,u,i={},o={},a=[],l=n.length,s=0,f=0;(r=e())!==o;){for(var h=[];r!==i&&r!==o;)h.push(r),r=e();t&&null==(h=t(h,f++))||a.push(h)}return a},e.format=function(t){if(Array.isArray(t[0]))return e.formatRows(t);var r=new m,u=[];return t.forEach(function(n){for(var t in n)r.has(t)||u.push(r.add(t))}),[u.map(o).join(n)].concat(t.map(function(t){return u.map(function(n){return o(t[n])}).join(n)})).join("\n")},e.formatRows=function(n){return n.map(i).join("\n")},e},ta.csv=ta.dsv(",","text/csv"),ta.tsv=ta.dsv(" ","text/tab-separated-values");var Ka,Qa,nc,tc,ec,rc=this[x(this,"requestAnimationFrame")]||function(n){setTimeout(n,17)};ta.timer=function(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var u=e+t,i={c:n,t:u,f:!1,n:null};Qa?Qa.n=i:Ka=i,Qa=i,nc||(tc=clearTimeout(tc),nc=1,rc(qt))},ta.timer.flush=function(){Lt(),Tt()},ta.round=function(n,t){return t?Math.round(n*(t=Math.pow(10,t)))/t:Math.round(n)};var uc=["y","z","a","f","p","n","\xb5","m","","k","M","G","T","P","E","Z","Y"].map(Dt);ta.formatPrefix=function(n,t){var e=0;return n&&(0>n&&(n*=-1),t&&(n=ta.round(n,Rt(n,t))),e=1+Math.floor(1e-12+Math.log(n)/Math.LN10),e=Math.max(-24,Math.min(24,3*Math.floor((e-1)/3)))),uc[8+e/3]};var ic=/(?:([^{])?([<>=^]))?([+\- ])?([$#])?(0)?(\d+)?(,)?(\.-?\d+)?([a-z%])?/i,oc=ta.map({b:function(n){return n.toString(2)},c:function(n){return String.fromCharCode(n)},o:function(n){return n.toString(8)},x:function(n){return n.toString(16)},X:function(n){return n.toString(16).toUpperCase()},g:function(n,t){return n.toPrecision(t)},e:function(n,t){return n.toExponential(t)},f:function(n,t){return n.toFixed(t)},r:function(n,t){return(n=ta.round(n,Rt(n,t))).toFixed(Math.max(0,Math.min(20,Rt(n*(1+1e-15),t))))}}),ac=ta.time={},cc=Date;jt.prototype={getDate:function(){return this._.getUTCDate()},getDay:function(){return this._.getUTCDay()},getFullYear:function(){return this._.getUTCFullYear()},getHours:function(){return this._.getUTCHours()},getMilliseconds:function(){return this._.getUTCMilliseconds()},getMinutes:function(){return this._.getUTCMinutes()},getMonth:function(){return this._.getUTCMonth()},getSeconds:function(){return this._.getUTCSeconds()},getTime:function(){return this._.getTime()},getTimezoneOffset:function(){return 0},valueOf:function(){return this._.valueOf()},setDate:function(){lc.setUTCDate.apply(this._,arguments)},setDay:function(){lc.setUTCDay.apply(this._,arguments)},setFullYear:function(){lc.setUTCFullYear.apply(this._,arguments)},setHours:function(){lc.setUTCHours.apply(this._,arguments)},setMilliseconds:function(){lc.setUTCMilliseconds.apply(this._,arguments)},setMinutes:function(){lc.setUTCMinutes.apply(this._,arguments)},setMonth:function(){lc.setUTCMonth.apply(this._,arguments)},setSeconds:function(){lc.setUTCSeconds.apply(this._,arguments)},setTime:function(){lc.setTime.apply(this._,arguments)}};var lc=Date.prototype;ac.year=Ft(function(n){return n=ac.day(n),n.setMonth(0,1),n},function(n,t){n.setFullYear(n.getFullYear()+t)},function(n){return n.getFullYear()}),ac.years=ac.year.range,ac.years.utc=ac.year.utc.range,ac.day=Ft(function(n){var t=new cc(2e3,0);return t.setFullYear(n.getFullYear(),n.getMonth(),n.getDate()),t},function(n,t){n.setDate(n.getDate()+t)},function(n){return n.getDate()-1}),ac.days=ac.day.range,ac.days.utc=ac.day.utc.range,ac.dayOfYear=function(n){var t=ac.year(n);return Math.floor((n-t-6e4*(n.getTimezoneOffset()-t.getTimezoneOffset()))/864e5)},["sunday","monday","tuesday","wednesday","thursday","friday","saturday"].forEach(function(n,t){t=7-t;var e=ac[n]=Ft(function(n){return(n=ac.day(n)).setDate(n.getDate()-(n.getDay()+t)%7),n},function(n,t){n.setDate(n.getDate()+7*Math.floor(t))},function(n){var e=ac.year(n).getDay();return Math.floor((ac.dayOfYear(n)+(e+t)%7)/7)-(e!==t)});ac[n+"s"]=e.range,ac[n+"s"].utc=e.utc.range,ac[n+"OfYear"]=function(n){var e=ac.year(n).getDay();return Math.floor((ac.dayOfYear(n)+(e+t)%7)/7)}}),ac.week=ac.sunday,ac.weeks=ac.sunday.range,ac.weeks.utc=ac.sunday.utc.range,ac.weekOfYear=ac.sundayOfYear;var sc={"-":"",_:" ",0:"0"},fc=/^\s*\d+/,hc=/^%/;ta.locale=function(n){return{numberFormat:Pt(n),timeFormat:Ot(n)}};var gc=ta.locale({decimal:".",thousands:",",grouping:[3],currency:["$",""],dateTime:"%a %b %e %X %Y",date:"%m/%d/%Y",time:"%H:%M:%S",periods:["AM","PM"],days:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],shortDays:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],months:["January","February","March","April","May","June","July","August","September","October","November","December"],shortMonths:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]});ta.format=gc.numberFormat,ta.geo={},ce.prototype={s:0,t:0,add:function(n){le(n,this.t,pc),le(pc.s,this.s,this),this.s?this.t+=pc.t:this.s=pc.t },reset:function(){this.s=this.t=0},valueOf:function(){return this.s}};var pc=new ce;ta.geo.stream=function(n,t){n&&vc.hasOwnProperty(n.type)?vc[n.type](n,t):se(n,t)};var vc={Feature:function(n,t){se(n.geometry,t)},FeatureCollection:function(n,t){for(var e=n.features,r=-1,u=e.length;++rn?4*qa+n:n,Mc.lineStart=Mc.lineEnd=Mc.point=b}};ta.geo.bounds=function(){function n(n,t){M.push(x=[s=n,h=n]),f>t&&(f=t),t>g&&(g=t)}function t(t,e){var r=pe([t*Da,e*Da]);if(m){var u=de(m,r),i=[u[1],-u[0],0],o=de(i,u);Me(o),o=xe(o);var c=t-p,l=c>0?1:-1,v=o[0]*Pa*l,d=ga(c)>180;if(d^(v>l*p&&l*t>v)){var y=o[1]*Pa;y>g&&(g=y)}else if(v=(v+360)%360-180,d^(v>l*p&&l*t>v)){var y=-o[1]*Pa;f>y&&(f=y)}else f>e&&(f=e),e>g&&(g=e);d?p>t?a(s,t)>a(s,h)&&(h=t):a(t,h)>a(s,h)&&(s=t):h>=s?(s>t&&(s=t),t>h&&(h=t)):t>p?a(s,t)>a(s,h)&&(h=t):a(t,h)>a(s,h)&&(s=t)}else n(t,e);m=r,p=t}function e(){b.point=t}function r(){x[0]=s,x[1]=h,b.point=n,m=null}function u(n,e){if(m){var r=n-p;y+=ga(r)>180?r+(r>0?360:-360):r}else v=n,d=e;Mc.point(n,e),t(n,e)}function i(){Mc.lineStart()}function o(){u(v,d),Mc.lineEnd(),ga(y)>Ca&&(s=-(h=180)),x[0]=s,x[1]=h,m=null}function a(n,t){return(t-=n)<0?t+360:t}function c(n,t){return n[0]-t[0]}function l(n,t){return t[0]<=t[1]?t[0]<=n&&n<=t[1]:nyc?(s=-(h=180),f=-(g=90)):y>Ca?g=90:-Ca>y&&(f=-90),x[0]=s,x[1]=h}};return function(n){g=h=-(s=f=1/0),M=[],ta.geo.stream(n,b);var t=M.length;if(t){M.sort(c);for(var e,r=1,u=M[0],i=[u];t>r;++r)e=M[r],l(e[0],u)||l(e[1],u)?(a(u[0],e[1])>a(u[0],u[1])&&(u[1]=e[1]),a(e[0],u[1])>a(u[0],u[1])&&(u[0]=e[0])):i.push(u=e);for(var o,e,p=-1/0,t=i.length-1,r=0,u=i[t];t>=r;u=e,++r)e=i[r],(o=a(u[1],e[0]))>p&&(p=o,s=e[0],h=u[1])}return M=x=null,1/0===s||1/0===f?[[0/0,0/0],[0/0,0/0]]:[[s,f],[h,g]]}}(),ta.geo.centroid=function(n){xc=bc=_c=wc=Sc=kc=Ec=Ac=Nc=Cc=zc=0,ta.geo.stream(n,qc);var t=Nc,e=Cc,r=zc,u=t*t+e*e+r*r;return za>u&&(t=kc,e=Ec,r=Ac,Ca>bc&&(t=_c,e=wc,r=Sc),u=t*t+e*e+r*r,za>u)?[0/0,0/0]:[Math.atan2(e,t)*Pa,tt(r/Math.sqrt(u))*Pa]};var xc,bc,_c,wc,Sc,kc,Ec,Ac,Nc,Cc,zc,qc={sphere:b,point:_e,lineStart:Se,lineEnd:ke,polygonStart:function(){qc.lineStart=Ee},polygonEnd:function(){qc.lineStart=Se}},Lc=Le(Ne,Pe,je,[-qa,-qa/2]),Tc=1e9;ta.geo.clipExtent=function(){var n,t,e,r,u,i,o={stream:function(n){return u&&(u.valid=!1),u=i(n),u.valid=!0,u},extent:function(a){return arguments.length?(i=Ie(n=+a[0][0],t=+a[0][1],e=+a[1][0],r=+a[1][1]),u&&(u.valid=!1,u=null),o):[[n,t],[e,r]]}};return o.extent([[0,0],[960,500]])},(ta.geo.conicEqualArea=function(){return Ye(Ze)}).raw=Ze,ta.geo.albers=function(){return ta.geo.conicEqualArea().rotate([96,0]).center([-.6,38.7]).parallels([29.5,45.5]).scale(1070)},ta.geo.albersUsa=function(){function n(n){var i=n[0],o=n[1];return t=null,e(i,o),t||(r(i,o),t)||u(i,o),t}var t,e,r,u,i=ta.geo.albers(),o=ta.geo.conicEqualArea().rotate([154,0]).center([-2,58.5]).parallels([55,65]),a=ta.geo.conicEqualArea().rotate([157,0]).center([-3,19.9]).parallels([8,18]),c={point:function(n,e){t=[n,e]}};return n.invert=function(n){var t=i.scale(),e=i.translate(),r=(n[0]-e[0])/t,u=(n[1]-e[1])/t;return(u>=.12&&.234>u&&r>=-.425&&-.214>r?o:u>=.166&&.234>u&&r>=-.214&&-.115>r?a:i).invert(n)},n.stream=function(n){var t=i.stream(n),e=o.stream(n),r=a.stream(n);return{point:function(n,u){t.point(n,u),e.point(n,u),r.point(n,u)},sphere:function(){t.sphere(),e.sphere(),r.sphere()},lineStart:function(){t.lineStart(),e.lineStart(),r.lineStart()},lineEnd:function(){t.lineEnd(),e.lineEnd(),r.lineEnd()},polygonStart:function(){t.polygonStart(),e.polygonStart(),r.polygonStart()},polygonEnd:function(){t.polygonEnd(),e.polygonEnd(),r.polygonEnd()}}},n.precision=function(t){return arguments.length?(i.precision(t),o.precision(t),a.precision(t),n):i.precision()},n.scale=function(t){return arguments.length?(i.scale(t),o.scale(.35*t),a.scale(t),n.translate(i.translate())):i.scale()},n.translate=function(t){if(!arguments.length)return i.translate();var l=i.scale(),s=+t[0],f=+t[1];return e=i.translate(t).clipExtent([[s-.455*l,f-.238*l],[s+.455*l,f+.238*l]]).stream(c).point,r=o.translate([s-.307*l,f+.201*l]).clipExtent([[s-.425*l+Ca,f+.12*l+Ca],[s-.214*l-Ca,f+.234*l-Ca]]).stream(c).point,u=a.translate([s-.205*l,f+.212*l]).clipExtent([[s-.214*l+Ca,f+.166*l+Ca],[s-.115*l-Ca,f+.234*l-Ca]]).stream(c).point,n},n.scale(1070)};var Rc,Dc,Pc,Uc,jc,Fc,Hc={point:b,lineStart:b,lineEnd:b,polygonStart:function(){Dc=0,Hc.lineStart=Ve},polygonEnd:function(){Hc.lineStart=Hc.lineEnd=Hc.point=b,Rc+=ga(Dc/2)}},Oc={point:Xe,lineStart:b,lineEnd:b,polygonStart:b,polygonEnd:b},Ic={point:We,lineStart:Je,lineEnd:Ge,polygonStart:function(){Ic.lineStart=Ke},polygonEnd:function(){Ic.point=We,Ic.lineStart=Je,Ic.lineEnd=Ge}};ta.geo.path=function(){function n(n){return n&&("function"==typeof a&&i.pointRadius(+a.apply(this,arguments)),o&&o.valid||(o=u(i)),ta.geo.stream(n,o)),i.result()}function t(){return o=null,n}var e,r,u,i,o,a=4.5;return n.area=function(n){return Rc=0,ta.geo.stream(n,u(Hc)),Rc},n.centroid=function(n){return _c=wc=Sc=kc=Ec=Ac=Nc=Cc=zc=0,ta.geo.stream(n,u(Ic)),zc?[Nc/zc,Cc/zc]:Ac?[kc/Ac,Ec/Ac]:Sc?[_c/Sc,wc/Sc]:[0/0,0/0]},n.bounds=function(n){return jc=Fc=-(Pc=Uc=1/0),ta.geo.stream(n,u(Oc)),[[Pc,Uc],[jc,Fc]]},n.projection=function(n){return arguments.length?(u=(e=n)?n.stream||tr(n):y,t()):e},n.context=function(n){return arguments.length?(i=null==(r=n)?new $e:new Qe(n),"function"!=typeof a&&i.pointRadius(a),t()):r},n.pointRadius=function(t){return arguments.length?(a="function"==typeof t?t:(i.pointRadius(+t),+t),n):a},n.projection(ta.geo.albersUsa()).context(null)},ta.geo.transform=function(n){return{stream:function(t){var e=new er(t);for(var r in n)e[r]=n[r];return e}}},er.prototype={point:function(n,t){this.stream.point(n,t)},sphere:function(){this.stream.sphere()},lineStart:function(){this.stream.lineStart()},lineEnd:function(){this.stream.lineEnd()},polygonStart:function(){this.stream.polygonStart()},polygonEnd:function(){this.stream.polygonEnd()}},ta.geo.projection=ur,ta.geo.projectionMutator=ir,(ta.geo.equirectangular=function(){return ur(ar)}).raw=ar.invert=ar,ta.geo.rotation=function(n){function t(t){return t=n(t[0]*Da,t[1]*Da),t[0]*=Pa,t[1]*=Pa,t}return n=lr(n[0]%360*Da,n[1]*Da,n.length>2?n[2]*Da:0),t.invert=function(t){return t=n.invert(t[0]*Da,t[1]*Da),t[0]*=Pa,t[1]*=Pa,t},t},cr.invert=ar,ta.geo.circle=function(){function n(){var n="function"==typeof r?r.apply(this,arguments):r,t=lr(-n[0]*Da,-n[1]*Da,0).invert,u=[];return e(null,null,1,{point:function(n,e){u.push(n=t(n,e)),n[0]*=Pa,n[1]*=Pa}}),{type:"Polygon",coordinates:[u]}}var t,e,r=[0,0],u=6;return n.origin=function(t){return arguments.length?(r=t,n):r},n.angle=function(r){return arguments.length?(e=gr((t=+r)*Da,u*Da),n):t},n.precision=function(r){return arguments.length?(e=gr(t*Da,(u=+r)*Da),n):u},n.angle(90)},ta.geo.distance=function(n,t){var e,r=(t[0]-n[0])*Da,u=n[1]*Da,i=t[1]*Da,o=Math.sin(r),a=Math.cos(r),c=Math.sin(u),l=Math.cos(u),s=Math.sin(i),f=Math.cos(i);return Math.atan2(Math.sqrt((e=f*o)*e+(e=l*s-c*f*a)*e),c*s+l*f*a)},ta.geo.graticule=function(){function n(){return{type:"MultiLineString",coordinates:t()}}function t(){return ta.range(Math.ceil(i/d)*d,u,d).map(h).concat(ta.range(Math.ceil(l/m)*m,c,m).map(g)).concat(ta.range(Math.ceil(r/p)*p,e,p).filter(function(n){return ga(n%d)>Ca}).map(s)).concat(ta.range(Math.ceil(a/v)*v,o,v).filter(function(n){return ga(n%m)>Ca}).map(f))}var e,r,u,i,o,a,c,l,s,f,h,g,p=10,v=p,d=90,m=360,y=2.5;return n.lines=function(){return t().map(function(n){return{type:"LineString",coordinates:n}})},n.outline=function(){return{type:"Polygon",coordinates:[h(i).concat(g(c).slice(1),h(u).reverse().slice(1),g(l).reverse().slice(1))]}},n.extent=function(t){return arguments.length?n.majorExtent(t).minorExtent(t):n.minorExtent()},n.majorExtent=function(t){return arguments.length?(i=+t[0][0],u=+t[1][0],l=+t[0][1],c=+t[1][1],i>u&&(t=i,i=u,u=t),l>c&&(t=l,l=c,c=t),n.precision(y)):[[i,l],[u,c]]},n.minorExtent=function(t){return arguments.length?(r=+t[0][0],e=+t[1][0],a=+t[0][1],o=+t[1][1],r>e&&(t=r,r=e,e=t),a>o&&(t=a,a=o,o=t),n.precision(y)):[[r,a],[e,o]]},n.step=function(t){return arguments.length?n.majorStep(t).minorStep(t):n.minorStep()},n.majorStep=function(t){return arguments.length?(d=+t[0],m=+t[1],n):[d,m]},n.minorStep=function(t){return arguments.length?(p=+t[0],v=+t[1],n):[p,v]},n.precision=function(t){return arguments.length?(y=+t,s=vr(a,o,90),f=dr(r,e,y),h=vr(l,c,90),g=dr(i,u,y),n):y},n.majorExtent([[-180,-90+Ca],[180,90-Ca]]).minorExtent([[-180,-80-Ca],[180,80+Ca]])},ta.geo.greatArc=function(){function n(){return{type:"LineString",coordinates:[t||r.apply(this,arguments),e||u.apply(this,arguments)]}}var t,e,r=mr,u=yr;return n.distance=function(){return ta.geo.distance(t||r.apply(this,arguments),e||u.apply(this,arguments))},n.source=function(e){return arguments.length?(r=e,t="function"==typeof e?null:e,n):r},n.target=function(t){return arguments.length?(u=t,e="function"==typeof t?null:t,n):u},n.precision=function(){return arguments.length?n:0},n},ta.geo.interpolate=function(n,t){return Mr(n[0]*Da,n[1]*Da,t[0]*Da,t[1]*Da)},ta.geo.length=function(n){return Yc=0,ta.geo.stream(n,Zc),Yc};var Yc,Zc={sphere:b,point:b,lineStart:xr,lineEnd:b,polygonStart:b,polygonEnd:b},Vc=br(function(n){return Math.sqrt(2/(1+n))},function(n){return 2*Math.asin(n/2)});(ta.geo.azimuthalEqualArea=function(){return ur(Vc)}).raw=Vc;var Xc=br(function(n){var t=Math.acos(n);return t&&t/Math.sin(t)},y);(ta.geo.azimuthalEquidistant=function(){return ur(Xc)}).raw=Xc,(ta.geo.conicConformal=function(){return Ye(_r)}).raw=_r,(ta.geo.conicEquidistant=function(){return Ye(wr)}).raw=wr;var $c=br(function(n){return 1/n},Math.atan);(ta.geo.gnomonic=function(){return ur($c)}).raw=$c,Sr.invert=function(n,t){return[n,2*Math.atan(Math.exp(t))-Ra]},(ta.geo.mercator=function(){return kr(Sr)}).raw=Sr;var Bc=br(function(){return 1},Math.asin);(ta.geo.orthographic=function(){return ur(Bc)}).raw=Bc;var Wc=br(function(n){return 1/(1+n)},function(n){return 2*Math.atan(n)});(ta.geo.stereographic=function(){return ur(Wc)}).raw=Wc,Er.invert=function(n,t){return[-t,2*Math.atan(Math.exp(n))-Ra]},(ta.geo.transverseMercator=function(){var n=kr(Er),t=n.center,e=n.rotate;return n.center=function(n){return n?t([-n[1],n[0]]):(n=t(),[n[1],-n[0]])},n.rotate=function(n){return n?e([n[0],n[1],n.length>2?n[2]+90:90]):(n=e(),[n[0],n[1],n[2]-90])},e([0,0,90])}).raw=Er,ta.geom={},ta.geom.hull=function(n){function t(n){if(n.length<3)return[];var t,u=Et(e),i=Et(r),o=n.length,a=[],c=[];for(t=0;o>t;t++)a.push([+u.call(this,n[t],t),+i.call(this,n[t],t),t]);for(a.sort(zr),t=0;o>t;t++)c.push([a[t][0],-a[t][1]]);var l=Cr(a),s=Cr(c),f=s[0]===l[0],h=s[s.length-1]===l[l.length-1],g=[];for(t=l.length-1;t>=0;--t)g.push(n[a[l[t]][2]]);for(t=+f;t=r&&l.x<=i&&l.y>=u&&l.y<=o?[[r,o],[i,o],[i,u],[r,u]]:[];s.point=n[a]}),t}function e(n){return n.map(function(n,t){return{x:Math.round(i(n,t)/Ca)*Ca,y:Math.round(o(n,t)/Ca)*Ca,i:t}})}var r=Ar,u=Nr,i=r,o=u,a=ul;return n?t(n):(t.links=function(n){return iu(e(n)).edges.filter(function(n){return n.l&&n.r}).map(function(t){return{source:n[t.l.i],target:n[t.r.i]}})},t.triangles=function(n){var t=[];return iu(e(n)).cells.forEach(function(e,r){for(var u,i,o=e.site,a=e.edges.sort(Yr),c=-1,l=a.length,s=a[l-1].edge,f=s.l===o?s.r:s.l;++c=l,h=r>=s,g=h<<1|f;n.leaf=!1,n=n.nodes[g]||(n.nodes[g]=su()),f?u=l:a=l,h?o=s:c=s,i(n,t,e,r,u,o,a,c)}var s,f,h,g,p,v,d,m,y,M=Et(a),x=Et(c);if(null!=t)v=t,d=e,m=r,y=u;else if(m=y=-(v=d=1/0),f=[],h=[],p=n.length,o)for(g=0;p>g;++g)s=n[g],s.xm&&(m=s.x),s.y>y&&(y=s.y),f.push(s.x),h.push(s.y);else for(g=0;p>g;++g){var b=+M(s=n[g],g),_=+x(s,g);v>b&&(v=b),d>_&&(d=_),b>m&&(m=b),_>y&&(y=_),f.push(b),h.push(_)}var w=m-v,S=y-d;w>S?y=d+w:m=v+S;var k=su();if(k.add=function(n){i(k,n,+M(n,++g),+x(n,g),v,d,m,y)},k.visit=function(n){fu(n,k,v,d,m,y)},k.find=function(n){return hu(k,n[0],n[1],v,d,m,y)},g=-1,null==t){for(;++g=0?n.slice(0,t):n,r=t>=0?n.slice(t+1):"in";return e=cl.get(e)||al,r=ll.get(r)||y,Mu(r(e.apply(null,ea.call(arguments,1))))},ta.interpolateHcl=Lu,ta.interpolateHsl=Tu,ta.interpolateLab=Ru,ta.interpolateRound=Du,ta.transform=function(n){var t=ua.createElementNS(ta.ns.prefix.svg,"g");return(ta.transform=function(n){if(null!=n){t.setAttribute("transform",n);var e=t.transform.baseVal.consolidate()}return new Pu(e?e.matrix:sl)})(n)},Pu.prototype.toString=function(){return"translate("+this.translate+")rotate("+this.rotate+")skewX("+this.skew+")scale("+this.scale+")"};var sl={a:1,b:0,c:0,d:1,e:0,f:0};ta.interpolateTransform=Hu,ta.layout={},ta.layout.bundle=function(){return function(n){for(var t=[],e=-1,r=n.length;++ea*a/d){if(p>c){var l=t.charge/c;n.px-=i*l,n.py-=o*l}return!0}if(t.point&&c&&p>c){var l=t.pointCharge/c;n.px-=i*l,n.py-=o*l}}return!t.charge}}function t(n){n.px=ta.event.x,n.py=ta.event.y,a.resume()}var e,r,u,i,o,a={},c=ta.dispatch("start","tick","end"),l=[1,1],s=.9,f=fl,h=hl,g=-30,p=gl,v=.1,d=.64,m=[],M=[];return a.tick=function(){if((r*=.99)<.005)return c.end({type:"end",alpha:r=0}),!0;var t,e,a,f,h,p,d,y,x,b=m.length,_=M.length;for(e=0;_>e;++e)a=M[e],f=a.source,h=a.target,y=h.x-f.x,x=h.y-f.y,(p=y*y+x*x)&&(p=r*i[e]*((p=Math.sqrt(p))-u[e])/p,y*=p,x*=p,h.x-=y*(d=f.weight/(h.weight+f.weight)),h.y-=x*d,f.x+=y*(d=1-d),f.y+=x*d);if((d=r*v)&&(y=l[0]/2,x=l[1]/2,e=-1,d))for(;++e0?n:0:n>0&&(c.start({type:"start",alpha:r=n}),ta.timer(a.tick)),a):r},a.start=function(){function n(n,r){if(!e){for(e=new Array(c),a=0;c>a;++a)e[a]=[];for(a=0;s>a;++a){var u=M[a];e[u.source.index].push(u.target),e[u.target.index].push(u.source)}}for(var i,o=e[t],a=-1,l=o.length;++at;++t)(r=m[t]).index=t,r.weight=0;for(t=0;s>t;++t)r=M[t],"number"==typeof r.source&&(r.source=m[r.source]),"number"==typeof r.target&&(r.target=m[r.target]),++r.source.weight,++r.target.weight;for(t=0;c>t;++t)r=m[t],isNaN(r.x)&&(r.x=n("x",p)),isNaN(r.y)&&(r.y=n("y",v)),isNaN(r.px)&&(r.px=r.x),isNaN(r.py)&&(r.py=r.y);if(u=[],"function"==typeof f)for(t=0;s>t;++t)u[t]=+f.call(this,M[t],t);else for(t=0;s>t;++t)u[t]=f;if(i=[],"function"==typeof h)for(t=0;s>t;++t)i[t]=+h.call(this,M[t],t);else for(t=0;s>t;++t)i[t]=h;if(o=[],"function"==typeof g)for(t=0;c>t;++t)o[t]=+g.call(this,m[t],t);else for(t=0;c>t;++t)o[t]=g;return a.resume()},a.resume=function(){return a.alpha(.1)},a.stop=function(){return a.alpha(0)},a.drag=function(){return e||(e=ta.behavior.drag().origin(y).on("dragstart.force",Xu).on("drag.force",t).on("dragend.force",$u)),arguments.length?void this.on("mouseover.force",Bu).on("mouseout.force",Wu).call(e):e},ta.rebind(a,c,"on")};var fl=20,hl=1,gl=1/0;ta.layout.hierarchy=function(){function n(u){var i,o=[u],a=[];for(u.depth=0;null!=(i=o.pop());)if(a.push(i),(l=e.call(n,i,i.depth))&&(c=l.length)){for(var c,l,s;--c>=0;)o.push(s=l[c]),s.parent=i,s.depth=i.depth+1;r&&(i.value=0),i.children=l}else r&&(i.value=+r.call(n,i,i.depth)||0),delete i.children;return Qu(u,function(n){var e,u;t&&(e=n.children)&&e.sort(t),r&&(u=n.parent)&&(u.value+=n.value)}),a}var t=ei,e=ni,r=ti;return n.sort=function(e){return arguments.length?(t=e,n):t},n.children=function(t){return arguments.length?(e=t,n):e},n.value=function(t){return arguments.length?(r=t,n):r},n.revalue=function(t){return r&&(Ku(t,function(n){n.children&&(n.value=0)}),Qu(t,function(t){var e;t.children||(t.value=+r.call(n,t,t.depth)||0),(e=t.parent)&&(e.value+=t.value)})),t},n},ta.layout.partition=function(){function n(t,e,r,u){var i=t.children;if(t.x=e,t.y=t.depth*u,t.dx=r,t.dy=u,i&&(o=i.length)){var o,a,c,l=-1;for(r=t.value?r/t.value:0;++lf?-1:1),p=(f-c*g)/ta.sum(l),v=ta.range(c),d=[];return null!=e&&v.sort(e===pl?function(n,t){return l[t]-l[n]}:function(n,t){return e(o[n],o[t])}),v.forEach(function(n){d[n]={data:o[n],value:a=l[n],startAngle:s,endAngle:s+=a*p+g,padAngle:h}}),d}var t=Number,e=pl,r=0,u=La,i=0;return n.value=function(e){return arguments.length?(t=e,n):t},n.sort=function(t){return arguments.length?(e=t,n):e},n.startAngle=function(t){return arguments.length?(r=t,n):r},n.endAngle=function(t){return arguments.length?(u=t,n):u},n.padAngle=function(t){return arguments.length?(i=t,n):i},n};var pl={};ta.layout.stack=function(){function n(a,c){if(!(h=a.length))return a;var l=a.map(function(e,r){return t.call(n,e,r)}),s=l.map(function(t){return t.map(function(t,e){return[i.call(n,t,e),o.call(n,t,e)]})}),f=e.call(n,s,c);l=ta.permute(l,f),s=ta.permute(s,f);var h,g,p,v,d=r.call(n,s,c),m=l[0].length;for(p=0;m>p;++p)for(u.call(n,l[0][p],v=d[p],s[0][p][1]),g=1;h>g;++g)u.call(n,l[g][p],v+=s[g-1][p][1],s[g][p][1]);return a}var t=y,e=ai,r=ci,u=oi,i=ui,o=ii;return n.values=function(e){return arguments.length?(t=e,n):t},n.order=function(t){return arguments.length?(e="function"==typeof t?t:vl.get(t)||ai,n):e},n.offset=function(t){return arguments.length?(r="function"==typeof t?t:dl.get(t)||ci,n):r},n.x=function(t){return arguments.length?(i=t,n):i},n.y=function(t){return arguments.length?(o=t,n):o},n.out=function(t){return arguments.length?(u=t,n):u},n};var vl=ta.map({"inside-out":function(n){var t,e,r=n.length,u=n.map(li),i=n.map(si),o=ta.range(r).sort(function(n,t){return u[n]-u[t]}),a=0,c=0,l=[],s=[];for(t=0;r>t;++t)e=o[t],c>a?(a+=i[e],l.push(e)):(c+=i[e],s.push(e));return s.reverse().concat(l)},reverse:function(n){return ta.range(n.length).reverse()},"default":ai}),dl=ta.map({silhouette:function(n){var t,e,r,u=n.length,i=n[0].length,o=[],a=0,c=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];r>a&&(a=r),o.push(r)}for(e=0;i>e;++e)c[e]=(a-o[e])/2;return c},wiggle:function(n){var t,e,r,u,i,o,a,c,l,s=n.length,f=n[0],h=f.length,g=[];for(g[0]=c=l=0,e=1;h>e;++e){for(t=0,u=0;s>t;++t)u+=n[t][e][1];for(t=0,i=0,a=f[e][0]-f[e-1][0];s>t;++t){for(r=0,o=(n[t][e][1]-n[t][e-1][1])/(2*a);t>r;++r)o+=(n[r][e][1]-n[r][e-1][1])/a;i+=o*n[t][e][1]}g[e]=c-=u?i/u*a:0,l>c&&(l=c)}for(e=0;h>e;++e)g[e]-=l;return g},expand:function(n){var t,e,r,u=n.length,i=n[0].length,o=1/u,a=[];for(e=0;i>e;++e){for(t=0,r=0;u>t;t++)r+=n[t][e][1];if(r)for(t=0;u>t;t++)n[t][e][1]/=r;else for(t=0;u>t;t++)n[t][e][1]=o}for(e=0;i>e;++e)a[e]=0;return a},zero:ci});ta.layout.histogram=function(){function n(n,i){for(var o,a,c=[],l=n.map(e,this),s=r.call(this,l,i),f=u.call(this,s,l,i),i=-1,h=l.length,g=f.length-1,p=t?1:1/h;++i0)for(i=-1;++i=s[0]&&a<=s[1]&&(o=c[ta.bisect(f,a,1,g)-1],o.y+=p,o.push(n[i]));return c}var t=!0,e=Number,r=pi,u=hi;return n.value=function(t){return arguments.length?(e=t,n):e},n.range=function(t){return arguments.length?(r=Et(t),n):r},n.bins=function(t){return arguments.length?(u="number"==typeof t?function(n){return gi(n,t)}:Et(t),n):u},n.frequency=function(e){return arguments.length?(t=!!e,n):t},n},ta.layout.pack=function(){function n(n,i){var o=e.call(this,n,i),a=o[0],c=u[0],l=u[1],s=null==t?Math.sqrt:"function"==typeof t?t:function(){return t};if(a.x=a.y=0,Qu(a,function(n){n.r=+s(n.value)}),Qu(a,Mi),r){var f=r*(t?1:Math.max(2*a.r/c,2*a.r/l))/2;Qu(a,function(n){n.r+=f}),Qu(a,Mi),Qu(a,function(n){n.r-=f})}return _i(a,c/2,l/2,t?1:1/Math.max(2*a.r/c,2*a.r/l)),o}var t,e=ta.layout.hierarchy().sort(vi),r=0,u=[1,1];return n.size=function(t){return arguments.length?(u=t,n):u},n.radius=function(e){return arguments.length?(t=null==e||"function"==typeof e?e:+e,n):t},n.padding=function(t){return arguments.length?(r=+t,n):r},Gu(n,e)},ta.layout.tree=function(){function n(n,u){var s=o.call(this,n,u),f=s[0],h=t(f);if(Qu(h,e),h.parent.m=-h.z,Ku(h,r),l)Ku(f,i);else{var g=f,p=f,v=f;Ku(f,function(n){n.xp.x&&(p=n),n.depth>v.depth&&(v=n)});var d=a(g,p)/2-g.x,m=c[0]/(p.x+a(p,g)/2+d),y=c[1]/(v.depth||1);Ku(f,function(n){n.x=(n.x+d)*m,n.y=n.depth*y})}return s}function t(n){for(var t,e={A:null,children:[n]},r=[e];null!=(t=r.pop());)for(var u,i=t.children,o=0,a=i.length;a>o;++o)r.push((i[o]=u={_:i[o],parent:t,children:(u=i[o].children)&&u.slice()||[],A:null,a:null,z:0,m:0,c:0,s:0,t:null,i:o}).a=u);return e.children[0]}function e(n){var t=n.children,e=n.parent.children,r=n.i?e[n.i-1]:null;if(t.length){Ni(n);var i=(t[0].z+t[t.length-1].z)/2;r?(n.z=r.z+a(n._,r._),n.m=n.z-i):n.z=i}else r&&(n.z=r.z+a(n._,r._));n.parent.A=u(n,r,n.parent.A||e[0])}function r(n){n._.x=n.z+n.parent.m,n.m+=n.parent.m}function u(n,t,e){if(t){for(var r,u=n,i=n,o=t,c=u.parent.children[0],l=u.m,s=i.m,f=o.m,h=c.m;o=Ei(o),u=ki(u),o&&u;)c=ki(c),i=Ei(i),i.a=n,r=o.z+f-u.z-l+a(o._,u._),r>0&&(Ai(Ci(o,n,e),n,r),l+=r,s+=r),f+=o.m,l+=u.m,h+=c.m,s+=i.m;o&&!Ei(i)&&(i.t=o,i.m+=f-s),u&&!ki(c)&&(c.t=u,c.m+=l-h,e=n)}return e}function i(n){n.x*=c[0],n.y=n.depth*c[1]}var o=ta.layout.hierarchy().sort(null).value(null),a=Si,c=[1,1],l=null;return n.separation=function(t){return arguments.length?(a=t,n):a},n.size=function(t){return arguments.length?(l=null==(c=t)?i:null,n):l?null:c},n.nodeSize=function(t){return arguments.length?(l=null==(c=t)?null:i,n):l?c:null},Gu(n,o)},ta.layout.cluster=function(){function n(n,i){var o,a=t.call(this,n,i),c=a[0],l=0;Qu(c,function(n){var t=n.children;t&&t.length?(n.x=qi(t),n.y=zi(t)):(n.x=o?l+=e(n,o):0,n.y=0,o=n)});var s=Li(c),f=Ti(c),h=s.x-e(s,f)/2,g=f.x+e(f,s)/2;return Qu(c,u?function(n){n.x=(n.x-c.x)*r[0],n.y=(c.y-n.y)*r[1]}:function(n){n.x=(n.x-h)/(g-h)*r[0],n.y=(1-(c.y?n.y/c.y:1))*r[1]}),a}var t=ta.layout.hierarchy().sort(null).value(null),e=Si,r=[1,1],u=!1;return n.separation=function(t){return arguments.length?(e=t,n):e},n.size=function(t){return arguments.length?(u=null==(r=t),n):u?null:r},n.nodeSize=function(t){return arguments.length?(u=null!=(r=t),n):u?r:null},Gu(n,t)},ta.layout.treemap=function(){function n(n,t){for(var e,r,u=-1,i=n.length;++ut?0:t),e.area=isNaN(r)||0>=r?0:r}function t(e){var i=e.children;if(i&&i.length){var o,a,c,l=f(e),s=[],h=i.slice(),p=1/0,v="slice"===g?l.dx:"dice"===g?l.dy:"slice-dice"===g?1&e.depth?l.dy:l.dx:Math.min(l.dx,l.dy);for(n(h,l.dx*l.dy/e.value),s.area=0;(c=h.length)>0;)s.push(o=h[c-1]),s.area+=o.area,"squarify"!==g||(a=r(s,v))<=p?(h.pop(),p=a):(s.area-=s.pop().area,u(s,v,l,!1),v=Math.min(l.dx,l.dy),s.length=s.area=0,p=1/0);s.length&&(u(s,v,l,!0),s.length=s.area=0),i.forEach(t)}}function e(t){var r=t.children;if(r&&r.length){var i,o=f(t),a=r.slice(),c=[];for(n(a,o.dx*o.dy/t.value),c.area=0;i=a.pop();)c.push(i),c.area+=i.area,null!=i.z&&(u(c,i.z?o.dx:o.dy,o,!a.length),c.length=c.area=0);r.forEach(e)}}function r(n,t){for(var e,r=n.area,u=0,i=1/0,o=-1,a=n.length;++oe&&(i=e),e>u&&(u=e));return r*=r,t*=t,r?Math.max(t*u*p/r,r/(t*i*p)):1/0}function u(n,t,e,r){var u,i=-1,o=n.length,a=e.x,l=e.y,s=t?c(n.area/t):0;if(t==e.dx){for((r||s>e.dy)&&(s=e.dy);++ie.dx)&&(s=e.dx);++ie&&(t=1),1>e&&(n=0),function(){var e,r,u;do e=2*Math.random()-1,r=2*Math.random()-1,u=e*e+r*r;while(!u||u>1);return n+t*e*Math.sqrt(-2*Math.log(u)/u)}},logNormal:function(){var n=ta.random.normal.apply(ta,arguments);return function(){return Math.exp(n())}},bates:function(n){var t=ta.random.irwinHall(n);return function(){return t()/n}},irwinHall:function(n){return function(){for(var t=0,e=0;n>e;e++)t+=Math.random();return t}}},ta.scale={};var ml={floor:y,ceil:y};ta.scale.linear=function(){return Ii([0,1],[0,1],mu,!1)};var yl={s:1,g:1,p:1,r:1,e:1};ta.scale.log=function(){return Ji(ta.scale.linear().domain([0,1]),10,!0,[1,10])};var Ml=ta.format(".0e"),xl={floor:function(n){return-Math.ceil(-n)},ceil:function(n){return-Math.floor(-n)}};ta.scale.pow=function(){return Gi(ta.scale.linear(),1,[0,1])},ta.scale.sqrt=function(){return ta.scale.pow().exponent(.5)},ta.scale.ordinal=function(){return Qi([],{t:"range",a:[[]]})},ta.scale.category10=function(){return ta.scale.ordinal().range(bl)},ta.scale.category20=function(){return ta.scale.ordinal().range(_l)},ta.scale.category20b=function(){return ta.scale.ordinal().range(wl)},ta.scale.category20c=function(){return ta.scale.ordinal().range(Sl)};var bl=[2062260,16744206,2924588,14034728,9725885,9197131,14907330,8355711,12369186,1556175].map(Mt),_l=[2062260,11454440,16744206,16759672,2924588,10018698,14034728,16750742,9725885,12955861,9197131,12885140,14907330,16234194,8355711,13092807,12369186,14408589,1556175,10410725].map(Mt),wl=[3750777,5395619,7040719,10264286,6519097,9216594,11915115,13556636,9202993,12426809,15186514,15190932,8666169,11356490,14049643,15177372,8077683,10834324,13528509,14589654].map(Mt),Sl=[3244733,7057110,10406625,13032431,15095053,16616764,16625259,16634018,3253076,7652470,10607003,13101504,7695281,10394312,12369372,14342891,6513507,9868950,12434877,14277081].map(Mt);ta.scale.quantile=function(){return no([],[])},ta.scale.quantize=function(){return to(0,1,[0,1])},ta.scale.threshold=function(){return eo([.5],[0,1])},ta.scale.identity=function(){return ro([0,1])},ta.svg={},ta.svg.arc=function(){function n(){var n=Math.max(0,+e.apply(this,arguments)),l=Math.max(0,+r.apply(this,arguments)),s=o.apply(this,arguments)-Ra,f=a.apply(this,arguments)-Ra,h=Math.abs(f-s),g=s>f?0:1;if(n>l&&(p=l,l=n,n=p),h>=Ta)return t(l,g)+(n?t(n,1-g):"")+"Z";var p,v,d,m,y,M,x,b,_,w,S,k,E=0,A=0,N=[];if((m=(+c.apply(this,arguments)||0)/2)&&(d=i===kl?Math.sqrt(n*n+l*l):+i.apply(this,arguments),g||(A*=-1),l&&(A=tt(d/l*Math.sin(m))),n&&(E=tt(d/n*Math.sin(m)))),l){y=l*Math.cos(s+A),M=l*Math.sin(s+A),x=l*Math.cos(f-A),b=l*Math.sin(f-A);var C=Math.abs(f-s-2*A)<=qa?0:1;if(A&&so(y,M,x,b)===g^C){var z=(s+f)/2;y=l*Math.cos(z),M=l*Math.sin(z),x=b=null}}else y=M=0;if(n){_=n*Math.cos(f-E),w=n*Math.sin(f-E),S=n*Math.cos(s+E),k=n*Math.sin(s+E);var q=Math.abs(s-f+2*E)<=qa?0:1;if(E&&so(_,w,S,k)===1-g^q){var L=(s+f)/2;_=n*Math.cos(L),w=n*Math.sin(L),S=k=null}}else _=w=0;if((p=Math.min(Math.abs(l-n)/2,+u.apply(this,arguments)))>.001){v=l>n^g?0:1;var T=null==S?[_,w]:null==x?[y,M]:Lr([y,M],[S,k],[x,b],[_,w]),R=y-T[0],D=M-T[1],P=x-T[0],U=b-T[1],j=1/Math.sin(Math.acos((R*P+D*U)/(Math.sqrt(R*R+D*D)*Math.sqrt(P*P+U*U)))/2),F=Math.sqrt(T[0]*T[0]+T[1]*T[1]);if(null!=x){var H=Math.min(p,(l-F)/(j+1)),O=fo(null==S?[_,w]:[S,k],[y,M],l,H,g),I=fo([x,b],[_,w],l,H,g);p===H?N.push("M",O[0],"A",H,",",H," 0 0,",v," ",O[1],"A",l,",",l," 0 ",1-g^so(O[1][0],O[1][1],I[1][0],I[1][1]),",",g," ",I[1],"A",H,",",H," 0 0,",v," ",I[0]):N.push("M",O[0],"A",H,",",H," 0 1,",v," ",I[0])}else N.push("M",y,",",M);if(null!=S){var Y=Math.min(p,(n-F)/(j-1)),Z=fo([y,M],[S,k],n,-Y,g),V=fo([_,w],null==x?[y,M]:[x,b],n,-Y,g);p===Y?N.push("L",V[0],"A",Y,",",Y," 0 0,",v," ",V[1],"A",n,",",n," 0 ",g^so(V[1][0],V[1][1],Z[1][0],Z[1][1]),",",1-g," ",Z[1],"A",Y,",",Y," 0 0,",v," ",Z[0]):N.push("L",V[0],"A",Y,",",Y," 0 0,",v," ",Z[0])}else N.push("L",_,",",w)}else N.push("M",y,",",M),null!=x&&N.push("A",l,",",l," 0 ",C,",",g," ",x,",",b),N.push("L",_,",",w),null!=S&&N.push("A",n,",",n," 0 ",q,",",1-g," ",S,",",k);return N.push("Z"),N.join("")}function t(n,t){return"M0,"+n+"A"+n+","+n+" 0 1,"+t+" 0,"+-n+"A"+n+","+n+" 0 1,"+t+" 0,"+n}var e=io,r=oo,u=uo,i=kl,o=ao,a=co,c=lo;return n.innerRadius=function(t){return arguments.length?(e=Et(t),n):e},n.outerRadius=function(t){return arguments.length?(r=Et(t),n):r},n.cornerRadius=function(t){return arguments.length?(u=Et(t),n):u},n.padRadius=function(t){return arguments.length?(i=t==kl?kl:Et(t),n):i},n.startAngle=function(t){return arguments.length?(o=Et(t),n):o},n.endAngle=function(t){return arguments.length?(a=Et(t),n):a},n.padAngle=function(t){return arguments.length?(c=Et(t),n):c},n.centroid=function(){var n=(+e.apply(this,arguments)+ +r.apply(this,arguments))/2,t=(+o.apply(this,arguments)+ +a.apply(this,arguments))/2-Ra;return[Math.cos(t)*n,Math.sin(t)*n]},n};var kl="auto";ta.svg.line=function(){return ho(y)};var El=ta.map({linear:go,"linear-closed":po,step:vo,"step-before":mo,"step-after":yo,basis:So,"basis-open":ko,"basis-closed":Eo,bundle:Ao,cardinal:bo,"cardinal-open":Mo,"cardinal-closed":xo,monotone:To});El.forEach(function(n,t){t.key=n,t.closed=/-closed$/.test(n)});var Al=[0,2/3,1/3,0],Nl=[0,1/3,2/3,0],Cl=[0,1/6,2/3,1/6];ta.svg.line.radial=function(){var n=ho(Ro);return n.radius=n.x,delete n.x,n.angle=n.y,delete n.y,n},mo.reverse=yo,yo.reverse=mo,ta.svg.area=function(){return Do(y)},ta.svg.area.radial=function(){var n=Do(Ro);return n.radius=n.x,delete n.x,n.innerRadius=n.x0,delete n.x0,n.outerRadius=n.x1,delete n.x1,n.angle=n.y,delete n.y,n.startAngle=n.y0,delete n.y0,n.endAngle=n.y1,delete n.y1,n},ta.svg.chord=function(){function n(n,a){var c=t(this,i,n,a),l=t(this,o,n,a);return"M"+c.p0+r(c.r,c.p1,c.a1-c.a0)+(e(c,l)?u(c.r,c.p1,c.r,c.p0):u(c.r,c.p1,l.r,l.p0)+r(l.r,l.p1,l.a1-l.a0)+u(l.r,l.p1,c.r,c.p0))+"Z"}function t(n,t,e,r){var u=t.call(n,e,r),i=a.call(n,u,r),o=c.call(n,u,r)-Ra,s=l.call(n,u,r)-Ra;return{r:i,a0:o,a1:s,p0:[i*Math.cos(o),i*Math.sin(o)],p1:[i*Math.cos(s),i*Math.sin(s)]}}function e(n,t){return n.a0==t.a0&&n.a1==t.a1}function r(n,t,e){return"A"+n+","+n+" 0 "+ +(e>qa)+",1 "+t}function u(n,t,e,r){return"Q 0,0 "+r}var i=mr,o=yr,a=Po,c=ao,l=co;return n.radius=function(t){return arguments.length?(a=Et(t),n):a},n.source=function(t){return arguments.length?(i=Et(t),n):i},n.target=function(t){return arguments.length?(o=Et(t),n):o},n.startAngle=function(t){return arguments.length?(c=Et(t),n):c},n.endAngle=function(t){return arguments.length?(l=Et(t),n):l},n},ta.svg.diagonal=function(){function n(n,u){var i=t.call(this,n,u),o=e.call(this,n,u),a=(i.y+o.y)/2,c=[i,{x:i.x,y:a},{x:o.x,y:a},o];return c=c.map(r),"M"+c[0]+"C"+c[1]+" "+c[2]+" "+c[3]}var t=mr,e=yr,r=Uo;return n.source=function(e){return arguments.length?(t=Et(e),n):t},n.target=function(t){return arguments.length?(e=Et(t),n):e},n.projection=function(t){return arguments.length?(r=t,n):r},n},ta.svg.diagonal.radial=function(){var n=ta.svg.diagonal(),t=Uo,e=n.projection;return n.projection=function(n){return arguments.length?e(jo(t=n)):t},n},ta.svg.symbol=function(){function n(n,r){return(zl.get(t.call(this,n,r))||Oo)(e.call(this,n,r))}var t=Ho,e=Fo;return n.type=function(e){return arguments.length?(t=Et(e),n):t},n.size=function(t){return arguments.length?(e=Et(t),n):e},n};var zl=ta.map({circle:Oo,cross:function(n){var t=Math.sqrt(n/5)/2;return"M"+-3*t+","+-t+"H"+-t+"V"+-3*t+"H"+t+"V"+-t+"H"+3*t+"V"+t+"H"+t+"V"+3*t+"H"+-t+"V"+t+"H"+-3*t+"Z"},diamond:function(n){var t=Math.sqrt(n/(2*Ll)),e=t*Ll;return"M0,"+-t+"L"+e+",0 0,"+t+" "+-e+",0Z"},square:function(n){var t=Math.sqrt(n)/2;return"M"+-t+","+-t+"L"+t+","+-t+" "+t+","+t+" "+-t+","+t+"Z"},"triangle-down":function(n){var t=Math.sqrt(n/ql),e=t*ql/2;return"M0,"+e+"L"+t+","+-e+" "+-t+","+-e+"Z"},"triangle-up":function(n){var t=Math.sqrt(n/ql),e=t*ql/2;return"M0,"+-e+"L"+t+","+e+" "+-t+","+e+"Z"}});ta.svg.symbolTypes=zl.keys();var ql=Math.sqrt(3),Ll=Math.tan(30*Da);_a.transition=function(n){for(var t,e,r=Tl||++Ul,u=Xo(n),i=[],o=Rl||{time:Date.now(),ease:Su,delay:0,duration:250},a=-1,c=this.length;++ai;i++){u.push(t=[]);for(var e=this[i],a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a,i)&&t.push(r)}return Yo(u,this.namespace,this.id)},Pl.tween=function(n,t){var e=this.id,r=this.namespace;return arguments.length<2?this.node()[r][e].tween.get(n):Y(this,null==t?function(t){t[r][e].tween.remove(n)}:function(u){u[r][e].tween.set(n,t)})},Pl.attr=function(n,t){function e(){this.removeAttribute(a)}function r(){this.removeAttributeNS(a.space,a.local)}function u(n){return null==n?e:(n+="",function(){var t,e=this.getAttribute(a);return e!==n&&(t=o(e,n),function(n){this.setAttribute(a,t(n))})})}function i(n){return null==n?r:(n+="",function(){var t,e=this.getAttributeNS(a.space,a.local);return e!==n&&(t=o(e,n),function(n){this.setAttributeNS(a.space,a.local,t(n))})})}if(arguments.length<2){for(t in n)this.attr(t,n[t]);return this}var o="transform"==n?Hu:mu,a=ta.ns.qualify(n);return Zo(this,"attr."+n,t,a.local?i:u)},Pl.attrTween=function(n,t){function e(n,e){var r=t.call(this,n,e,this.getAttribute(u));return r&&function(n){this.setAttribute(u,r(n))}}function r(n,e){var r=t.call(this,n,e,this.getAttributeNS(u.space,u.local));return r&&function(n){this.setAttributeNS(u.space,u.local,r(n))}}var u=ta.ns.qualify(n);return this.tween("attr."+n,u.local?r:e)},Pl.style=function(n,e,r){function u(){this.style.removeProperty(n)}function i(e){return null==e?u:(e+="",function(){var u,i=t(this).getComputedStyle(this,null).getPropertyValue(n);return i!==e&&(u=mu(i,e),function(t){this.style.setProperty(n,u(t),r)})})}var o=arguments.length;if(3>o){if("string"!=typeof n){2>o&&(e="");for(r in n)this.style(r,n[r],e);return this}r=""}return Zo(this,"style."+n,e,i)},Pl.styleTween=function(n,e,r){function u(u,i){var o=e.call(this,u,i,t(this).getComputedStyle(this,null).getPropertyValue(n));return o&&function(t){this.style.setProperty(n,o(t),r)}}return arguments.length<3&&(r=""),this.tween("style."+n,u)},Pl.text=function(n){return Zo(this,"text",n,Vo)},Pl.remove=function(){var n=this.namespace;return this.each("end.transition",function(){var t;this[n].count<2&&(t=this.parentNode)&&t.removeChild(this)})},Pl.ease=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].ease:("function"!=typeof n&&(n=ta.ease.apply(ta,arguments)),Y(this,function(r){r[e][t].ease=n}))},Pl.delay=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].delay:Y(this,"function"==typeof n?function(r,u,i){r[e][t].delay=+n.call(r,r.__data__,u,i)}:(n=+n,function(r){r[e][t].delay=n}))},Pl.duration=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].duration:Y(this,"function"==typeof n?function(r,u,i){r[e][t].duration=Math.max(1,n.call(r,r.__data__,u,i))}:(n=Math.max(1,n),function(r){r[e][t].duration=n}))},Pl.each=function(n,t){var e=this.id,r=this.namespace;if(arguments.length<2){var u=Rl,i=Tl;try{Tl=e,Y(this,function(t,u,i){Rl=t[r][e],n.call(t,t.__data__,u,i)})}finally{Rl=u,Tl=i}}else Y(this,function(u){var i=u[r][e];(i.event||(i.event=ta.dispatch("start","end","interrupt"))).on(n,t)});return this},Pl.transition=function(){for(var n,t,e,r,u=this.id,i=++Ul,o=this.namespace,a=[],c=0,l=this.length;l>c;c++){a.push(n=[]);for(var t=this[c],s=0,f=t.length;f>s;s++)(e=t[s])&&(r=e[o][u],$o(e,s,o,i,{time:r.time,ease:r.ease,delay:r.delay+r.duration,duration:r.duration})),n.push(e)}return Yo(a,o,i)},ta.svg.axis=function(){function n(n){n.each(function(){var n,l=ta.select(this),s=this.__chart__||e,f=this.__chart__=e.copy(),h=null==c?f.ticks?f.ticks.apply(f,a):f.domain():c,g=null==t?f.tickFormat?f.tickFormat.apply(f,a):y:t,p=l.selectAll(".tick").data(h,f),v=p.enter().insert("g",".domain").attr("class","tick").style("opacity",Ca),d=ta.transition(p.exit()).style("opacity",Ca).remove(),m=ta.transition(p.order()).style("opacity",1),M=Math.max(u,0)+o,x=Ui(f),b=l.selectAll(".domain").data([0]),_=(b.enter().append("path").attr("class","domain"),ta.transition(b));v.append("line"),v.append("text");var w,S,k,E,A=v.select("line"),N=m.select("line"),C=p.select("text").text(g),z=v.select("text"),q=m.select("text"),L="top"===r||"left"===r?-1:1;if("bottom"===r||"top"===r?(n=Bo,w="x",k="y",S="x2",E="y2",C.attr("dy",0>L?"0em":".71em").style("text-anchor","middle"),_.attr("d","M"+x[0]+","+L*i+"V0H"+x[1]+"V"+L*i)):(n=Wo,w="y",k="x",S="y2",E="x2",C.attr("dy",".32em").style("text-anchor",0>L?"end":"start"),_.attr("d","M"+L*i+","+x[0]+"H0V"+x[1]+"H"+L*i)),A.attr(E,L*u),z.attr(k,L*M),N.attr(S,0).attr(E,L*u),q.attr(w,0).attr(k,L*M),f.rangeBand){var T=f,R=T.rangeBand()/2;s=f=function(n){return T(n)+R}}else s.rangeBand?s=f:d.call(n,f,s);v.call(n,s,f),m.call(n,f,f)})}var t,e=ta.scale.linear(),r=jl,u=6,i=6,o=3,a=[10],c=null;return n.scale=function(t){return arguments.length?(e=t,n):e},n.orient=function(t){return arguments.length?(r=t in Fl?t+"":jl,n):r},n.ticks=function(){return arguments.length?(a=arguments,n):a},n.tickValues=function(t){return arguments.length?(c=t,n):c},n.tickFormat=function(e){return arguments.length?(t=e,n):t},n.tickSize=function(t){var e=arguments.length;return e?(u=+t,i=+arguments[e-1],n):u},n.innerTickSize=function(t){return arguments.length?(u=+t,n):u},n.outerTickSize=function(t){return arguments.length?(i=+t,n):i},n.tickPadding=function(t){return arguments.length?(o=+t,n):o},n.tickSubdivide=function(){return arguments.length&&n},n};var jl="bottom",Fl={top:1,right:1,bottom:1,left:1};ta.svg.brush=function(){function n(t){t.each(function(){var t=ta.select(this).style("pointer-events","all").style("-webkit-tap-highlight-color","rgba(0,0,0,0)").on("mousedown.brush",i).on("touchstart.brush",i),o=t.selectAll(".background").data([0]);o.enter().append("rect").attr("class","background").style("visibility","hidden").style("cursor","crosshair"),t.selectAll(".extent").data([0]).enter().append("rect").attr("class","extent").style("cursor","move");var a=t.selectAll(".resize").data(v,y);a.exit().remove(),a.enter().append("g").attr("class",function(n){return"resize "+n}).style("cursor",function(n){return Hl[n]}).append("rect").attr("x",function(n){return/[ew]$/.test(n)?-3:null}).attr("y",function(n){return/^[ns]/.test(n)?-3:null}).attr("width",6).attr("height",6).style("visibility","hidden"),a.style("display",n.empty()?"none":null);var c,f=ta.transition(t),h=ta.transition(o);l&&(c=Ui(l),h.attr("x",c[0]).attr("width",c[1]-c[0]),r(f)),s&&(c=Ui(s),h.attr("y",c[0]).attr("height",c[1]-c[0]),u(f)),e(f)})}function e(n){n.selectAll(".resize").attr("transform",function(n){return"translate("+f[+/e$/.test(n)]+","+h[+/^s/.test(n)]+")"})}function r(n){n.select(".extent").attr("x",f[0]),n.selectAll(".extent,.n>rect,.s>rect").attr("width",f[1]-f[0])}function u(n){n.select(".extent").attr("y",h[0]),n.selectAll(".extent,.e>rect,.w>rect").attr("height",h[1]-h[0])}function i(){function i(){32==ta.event.keyCode&&(C||(M=null,q[0]-=f[1],q[1]-=h[1],C=2),S())}function v(){32==ta.event.keyCode&&2==C&&(q[0]+=f[1],q[1]+=h[1],C=0,S())}function d(){var n=ta.mouse(b),t=!1;x&&(n[0]+=x[0],n[1]+=x[1]),C||(ta.event.altKey?(M||(M=[(f[0]+f[1])/2,(h[0]+h[1])/2]),q[0]=f[+(n[0]s?(u=r,r=s):u=s),v[0]!=r||v[1]!=u?(e?a=null:o=null,v[0]=r,v[1]=u,!0):void 0}function y(){d(),k.style("pointer-events","all").selectAll(".resize").style("display",n.empty()?"none":null),ta.select("body").style("cursor",null),L.on("mousemove.brush",null).on("mouseup.brush",null).on("touchmove.brush",null).on("touchend.brush",null).on("keydown.brush",null).on("keyup.brush",null),z(),w({type:"brushend"})}var M,x,b=this,_=ta.select(ta.event.target),w=c.of(b,arguments),k=ta.select(b),E=_.datum(),A=!/^(n|s)$/.test(E)&&l,N=!/^(e|w)$/.test(E)&&s,C=_.classed("extent"),z=W(b),q=ta.mouse(b),L=ta.select(t(b)).on("keydown.brush",i).on("keyup.brush",v);if(ta.event.changedTouches?L.on("touchmove.brush",d).on("touchend.brush",y):L.on("mousemove.brush",d).on("mouseup.brush",y),k.interrupt().selectAll("*").interrupt(),C)q[0]=f[0]-q[0],q[1]=h[0]-q[1];else if(E){var T=+/w$/.test(E),R=+/^n/.test(E);x=[f[1-T]-q[0],h[1-R]-q[1]],q[0]=f[T],q[1]=h[R]}else ta.event.altKey&&(M=q.slice());k.style("pointer-events","none").selectAll(".resize").style("display",null),ta.select("body").style("cursor",_.style("cursor")),w({type:"brushstart"}),d()}var o,a,c=E(n,"brushstart","brush","brushend"),l=null,s=null,f=[0,0],h=[0,0],g=!0,p=!0,v=Ol[0];return n.event=function(n){n.each(function(){var n=c.of(this,arguments),t={x:f,y:h,i:o,j:a},e=this.__chart__||t;this.__chart__=t,Tl?ta.select(this).transition().each("start.brush",function(){o=e.i,a=e.j,f=e.x,h=e.y,n({type:"brushstart"})}).tween("brush:brush",function(){var e=yu(f,t.x),r=yu(h,t.y);return o=a=null,function(u){f=t.x=e(u),h=t.y=r(u),n({type:"brush",mode:"resize"})}}).each("end.brush",function(){o=t.i,a=t.j,n({type:"brush",mode:"resize"}),n({type:"brushend"})}):(n({type:"brushstart"}),n({type:"brush",mode:"resize"}),n({type:"brushend"}))})},n.x=function(t){return arguments.length?(l=t,v=Ol[!l<<1|!s],n):l},n.y=function(t){return arguments.length?(s=t,v=Ol[!l<<1|!s],n):s},n.clamp=function(t){return arguments.length?(l&&s?(g=!!t[0],p=!!t[1]):l?g=!!t:s&&(p=!!t),n):l&&s?[g,p]:l?g:s?p:null},n.extent=function(t){var e,r,u,i,c;return arguments.length?(l&&(e=t[0],r=t[1],s&&(e=e[0],r=r[0]),o=[e,r],l.invert&&(e=l(e),r=l(r)),e>r&&(c=e,e=r,r=c),(e!=f[0]||r!=f[1])&&(f=[e,r])),s&&(u=t[0],i=t[1],l&&(u=u[1],i=i[1]),a=[u,i],s.invert&&(u=s(u),i=s(i)),u>i&&(c=u,u=i,i=c),(u!=h[0]||i!=h[1])&&(h=[u,i])),n):(l&&(o?(e=o[0],r=o[1]):(e=f[0],r=f[1],l.invert&&(e=l.invert(e),r=l.invert(r)),e>r&&(c=e,e=r,r=c))),s&&(a?(u=a[0],i=a[1]):(u=h[0],i=h[1],s.invert&&(u=s.invert(u),i=s.invert(i)),u>i&&(c=u,u=i,i=c))),l&&s?[[e,u],[r,i]]:l?[e,r]:s&&[u,i])},n.clear=function(){return n.empty()||(f=[0,0],h=[0,0],o=a=null),n},n.empty=function(){return!!l&&f[0]==f[1]||!!s&&h[0]==h[1]},ta.rebind(n,c,"on")};var Hl={n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},Ol=[["n","e","s","w","nw","ne","se","sw"],["e","w"],["n","s"],[]],Il=ac.format=gc.timeFormat,Yl=Il.utc,Zl=Yl("%Y-%m-%dT%H:%M:%S.%LZ");Il.iso=Date.prototype.toISOString&&+new Date("2000-01-01T00:00:00.000Z")?Jo:Zl,Jo.parse=function(n){var t=new Date(n);return isNaN(t)?null:t},Jo.toString=Zl.toString,ac.second=Ft(function(n){return new cc(1e3*Math.floor(n/1e3))},function(n,t){n.setTime(n.getTime()+1e3*Math.floor(t))},function(n){return n.getSeconds()}),ac.seconds=ac.second.range,ac.seconds.utc=ac.second.utc.range,ac.minute=Ft(function(n){return new cc(6e4*Math.floor(n/6e4))},function(n,t){n.setTime(n.getTime()+6e4*Math.floor(t))},function(n){return n.getMinutes()}),ac.minutes=ac.minute.range,ac.minutes.utc=ac.minute.utc.range,ac.hour=Ft(function(n){var t=n.getTimezoneOffset()/60;return new cc(36e5*(Math.floor(n/36e5-t)+t))},function(n,t){n.setTime(n.getTime()+36e5*Math.floor(t))},function(n){return n.getHours()}),ac.hours=ac.hour.range,ac.hours.utc=ac.hour.utc.range,ac.month=Ft(function(n){return n=ac.day(n),n.setDate(1),n},function(n,t){n.setMonth(n.getMonth()+t)},function(n){return n.getMonth()}),ac.months=ac.month.range,ac.months.utc=ac.month.utc.range;var Vl=[1e3,5e3,15e3,3e4,6e4,3e5,9e5,18e5,36e5,108e5,216e5,432e5,864e5,1728e5,6048e5,2592e6,7776e6,31536e6],Xl=[[ac.second,1],[ac.second,5],[ac.second,15],[ac.second,30],[ac.minute,1],[ac.minute,5],[ac.minute,15],[ac.minute,30],[ac.hour,1],[ac.hour,3],[ac.hour,6],[ac.hour,12],[ac.day,1],[ac.day,2],[ac.week,1],[ac.month,1],[ac.month,3],[ac.year,1]],$l=Il.multi([[".%L",function(n){return n.getMilliseconds()}],[":%S",function(n){return n.getSeconds()}],["%I:%M",function(n){return n.getMinutes()}],["%I %p",function(n){return n.getHours()}],["%a %d",function(n){return n.getDay()&&1!=n.getDate()}],["%b %d",function(n){return 1!=n.getDate()}],["%B",function(n){return n.getMonth()}],["%Y",Ne]]),Bl={range:function(n,t,e){return ta.range(Math.ceil(n/e)*e,+t,e).map(Ko)},floor:y,ceil:y};Xl.year=ac.year,ac.scale=function(){return Go(ta.scale.linear(),Xl,$l)};var Wl=Xl.map(function(n){return[n[0].utc,n[1]]}),Jl=Yl.multi([[".%L",function(n){return n.getUTCMilliseconds()}],[":%S",function(n){return n.getUTCSeconds()}],["%I:%M",function(n){return n.getUTCMinutes()}],["%I %p",function(n){return n.getUTCHours()}],["%a %d",function(n){return n.getUTCDay()&&1!=n.getUTCDate()}],["%b %d",function(n){return 1!=n.getUTCDate()}],["%B",function(n){return n.getUTCMonth()}],["%Y",Ne]]);Wl.year=ac.year.utc,ac.scale.utc=function(){return Go(ta.scale.linear(),Wl,Jl)},ta.text=At(function(n){return n.responseText}),ta.json=function(n,t){return Nt(n,"application/json",Qo,t)},ta.html=function(n,t){return Nt(n,"text/html",na,t)},ta.xml=At(function(n){return n.responseXML}),"function"==typeof define&&define.amd?define(ta):"object"==typeof module&&module.exports&&(module.exports=ta),this.d3=ta}(); \ No newline at end of file +return i.size=function(n){return arguments.length?(l=n,i):l},i.padding=function(n){function t(t){var e=n.call(i,t,t.depth);return null==e?Ri(t):Di(t,"number"==typeof e?[e,e,e,e]:e)}function e(t){return Di(t,n)}if(!arguments.length)return s;var r;return f=null==(s=n)?Ri:"function"==(r=typeof n)?t:"number"===r?(n=[n,n,n,n],e):e,i},i.round=function(n){return arguments.length?(c=n?Math.round:Number,i):c!=Number},i.sticky=function(n){return arguments.length?(h=n,o=null,i):h},i.ratio=function(n){return arguments.length?(p=n,i):p},i.mode=function(n){return arguments.length?(g=n+"",i):g},Gu(i,a)},ta.random={normal:function(n,t){var e=arguments.length;return 2>e&&(t=1),1>e&&(n=0),function(){var e,r,u;do e=2*Math.random()-1,r=2*Math.random()-1,u=e*e+r*r;while(!u||u>1);return n+t*e*Math.sqrt(-2*Math.log(u)/u)}},logNormal:function(){var n=ta.random.normal.apply(ta,arguments);return function(){return Math.exp(n())}},bates:function(n){var t=ta.random.irwinHall(n);return function(){return t()/n}},irwinHall:function(n){return function(){for(var t=0,e=0;n>e;e++)t+=Math.random();return t}}},ta.scale={};var ml={floor:y,ceil:y};ta.scale.linear=function(){return Ii([0,1],[0,1],mu,!1)};var yl={s:1,g:1,p:1,r:1,e:1};ta.scale.log=function(){return Ji(ta.scale.linear().domain([0,1]),10,!0,[1,10])};var Ml=ta.format(".0e"),xl={floor:function(n){return-Math.ceil(-n)},ceil:function(n){return-Math.floor(-n)}};ta.scale.pow=function(){return Gi(ta.scale.linear(),1,[0,1])},ta.scale.sqrt=function(){return ta.scale.pow().exponent(.5)},ta.scale.ordinal=function(){return Qi([],{t:"range",a:[[]]})},ta.scale.category10=function(){return ta.scale.ordinal().range(bl)},ta.scale.category20=function(){return ta.scale.ordinal().range(_l)},ta.scale.category20b=function(){return ta.scale.ordinal().range(wl)},ta.scale.category20c=function(){return ta.scale.ordinal().range(Sl)};var bl=[2062260,16744206,2924588,14034728,9725885,9197131,14907330,8355711,12369186,1556175].map(Mt),_l=[2062260,11454440,16744206,16759672,2924588,10018698,14034728,16750742,9725885,12955861,9197131,12885140,14907330,16234194,8355711,13092807,12369186,14408589,1556175,10410725].map(Mt),wl=[3750777,5395619,7040719,10264286,6519097,9216594,11915115,13556636,9202993,12426809,15186514,15190932,8666169,11356490,14049643,15177372,8077683,10834324,13528509,14589654].map(Mt),Sl=[3244733,7057110,10406625,13032431,15095053,16616764,16625259,16634018,3253076,7652470,10607003,13101504,7695281,10394312,12369372,14342891,6513507,9868950,12434877,14277081].map(Mt);ta.scale.quantile=function(){return no([],[])},ta.scale.quantize=function(){return to(0,1,[0,1])},ta.scale.threshold=function(){return eo([.5],[0,1])},ta.scale.identity=function(){return ro([0,1])},ta.svg={},ta.svg.arc=function(){function n(){var n=Math.max(0,+e.apply(this,arguments)),l=Math.max(0,+r.apply(this,arguments)),s=o.apply(this,arguments)-Ra,f=a.apply(this,arguments)-Ra,h=Math.abs(f-s),g=s>f?0:1;if(n>l&&(p=l,l=n,n=p),h>=Ta)return t(l,g)+(n?t(n,1-g):"")+"Z";var p,v,d,m,y,M,x,b,_,w,S,k,E=0,A=0,N=[];if((m=(+c.apply(this,arguments)||0)/2)&&(d=i===kl?Math.sqrt(n*n+l*l):+i.apply(this,arguments),g||(A*=-1),l&&(A=tt(d/l*Math.sin(m))),n&&(E=tt(d/n*Math.sin(m)))),l){y=l*Math.cos(s+A),M=l*Math.sin(s+A),x=l*Math.cos(f-A),b=l*Math.sin(f-A);var C=Math.abs(f-s-2*A)<=qa?0:1;if(A&&so(y,M,x,b)===g^C){var z=(s+f)/2;y=l*Math.cos(z),M=l*Math.sin(z),x=b=null}}else y=M=0;if(n){_=n*Math.cos(f-E),w=n*Math.sin(f-E),S=n*Math.cos(s+E),k=n*Math.sin(s+E);var q=Math.abs(s-f+2*E)<=qa?0:1;if(E&&so(_,w,S,k)===1-g^q){var L=(s+f)/2;_=n*Math.cos(L),w=n*Math.sin(L),S=k=null}}else _=w=0;if((p=Math.min(Math.abs(l-n)/2,+u.apply(this,arguments)))>.001){v=l>n^g?0:1;var T=null==S?[_,w]:null==x?[y,M]:Lr([y,M],[S,k],[x,b],[_,w]),R=y-T[0],D=M-T[1],P=x-T[0],U=b-T[1],j=1/Math.sin(Math.acos((R*P+D*U)/(Math.sqrt(R*R+D*D)*Math.sqrt(P*P+U*U)))/2),F=Math.sqrt(T[0]*T[0]+T[1]*T[1]);if(null!=x){var H=Math.min(p,(l-F)/(j+1)),O=fo(null==S?[_,w]:[S,k],[y,M],l,H,g),I=fo([x,b],[_,w],l,H,g);p===H?N.push("M",O[0],"A",H,",",H," 0 0,",v," ",O[1],"A",l,",",l," 0 ",1-g^so(O[1][0],O[1][1],I[1][0],I[1][1]),",",g," ",I[1],"A",H,",",H," 0 0,",v," ",I[0]):N.push("M",O[0],"A",H,",",H," 0 1,",v," ",I[0])}else N.push("M",y,",",M);if(null!=S){var Y=Math.min(p,(n-F)/(j-1)),Z=fo([y,M],[S,k],n,-Y,g),V=fo([_,w],null==x?[y,M]:[x,b],n,-Y,g);p===Y?N.push("L",V[0],"A",Y,",",Y," 0 0,",v," ",V[1],"A",n,",",n," 0 ",g^so(V[1][0],V[1][1],Z[1][0],Z[1][1]),",",1-g," ",Z[1],"A",Y,",",Y," 0 0,",v," ",Z[0]):N.push("L",V[0],"A",Y,",",Y," 0 0,",v," ",Z[0])}else N.push("L",_,",",w)}else N.push("M",y,",",M),null!=x&&N.push("A",l,",",l," 0 ",C,",",g," ",x,",",b),N.push("L",_,",",w),null!=S&&N.push("A",n,",",n," 0 ",q,",",1-g," ",S,",",k);return N.push("Z"),N.join("")}function t(n,t){return"M0,"+n+"A"+n+","+n+" 0 1,"+t+" 0,"+-n+"A"+n+","+n+" 0 1,"+t+" 0,"+n}var e=io,r=oo,u=uo,i=kl,o=ao,a=co,c=lo;return n.innerRadius=function(t){return arguments.length?(e=Et(t),n):e},n.outerRadius=function(t){return arguments.length?(r=Et(t),n):r},n.cornerRadius=function(t){return arguments.length?(u=Et(t),n):u},n.padRadius=function(t){return arguments.length?(i=t==kl?kl:Et(t),n):i},n.startAngle=function(t){return arguments.length?(o=Et(t),n):o},n.endAngle=function(t){return arguments.length?(a=Et(t),n):a},n.padAngle=function(t){return arguments.length?(c=Et(t),n):c},n.centroid=function(){var n=(+e.apply(this,arguments)+ +r.apply(this,arguments))/2,t=(+o.apply(this,arguments)+ +a.apply(this,arguments))/2-Ra;return[Math.cos(t)*n,Math.sin(t)*n]},n};var kl="auto";ta.svg.line=function(){return ho(y)};var El=ta.map({linear:go,"linear-closed":po,step:vo,"step-before":mo,"step-after":yo,basis:So,"basis-open":ko,"basis-closed":Eo,bundle:Ao,cardinal:bo,"cardinal-open":Mo,"cardinal-closed":xo,monotone:To});El.forEach(function(n,t){t.key=n,t.closed=/-closed$/.test(n)});var Al=[0,2/3,1/3,0],Nl=[0,1/3,2/3,0],Cl=[0,1/6,2/3,1/6];ta.svg.line.radial=function(){var n=ho(Ro);return n.radius=n.x,delete n.x,n.angle=n.y,delete n.y,n},mo.reverse=yo,yo.reverse=mo,ta.svg.area=function(){return Do(y)},ta.svg.area.radial=function(){var n=Do(Ro);return n.radius=n.x,delete n.x,n.innerRadius=n.x0,delete n.x0,n.outerRadius=n.x1,delete n.x1,n.angle=n.y,delete n.y,n.startAngle=n.y0,delete n.y0,n.endAngle=n.y1,delete n.y1,n},ta.svg.chord=function(){function n(n,a){var c=t(this,i,n,a),l=t(this,o,n,a);return"M"+c.p0+r(c.r,c.p1,c.a1-c.a0)+(e(c,l)?u(c.r,c.p1,c.r,c.p0):u(c.r,c.p1,l.r,l.p0)+r(l.r,l.p1,l.a1-l.a0)+u(l.r,l.p1,c.r,c.p0))+"Z"}function t(n,t,e,r){var u=t.call(n,e,r),i=a.call(n,u,r),o=c.call(n,u,r)-Ra,s=l.call(n,u,r)-Ra;return{r:i,a0:o,a1:s,p0:[i*Math.cos(o),i*Math.sin(o)],p1:[i*Math.cos(s),i*Math.sin(s)]}}function e(n,t){return n.a0==t.a0&&n.a1==t.a1}function r(n,t,e){return"A"+n+","+n+" 0 "+ +(e>qa)+",1 "+t}function u(n,t,e,r){return"Q 0,0 "+r}var i=mr,o=yr,a=Po,c=ao,l=co;return n.radius=function(t){return arguments.length?(a=Et(t),n):a},n.source=function(t){return arguments.length?(i=Et(t),n):i},n.target=function(t){return arguments.length?(o=Et(t),n):o},n.startAngle=function(t){return arguments.length?(c=Et(t),n):c},n.endAngle=function(t){return arguments.length?(l=Et(t),n):l},n},ta.svg.diagonal=function(){function n(n,u){var i=t.call(this,n,u),o=e.call(this,n,u),a=(i.y+o.y)/2,c=[i,{x:i.x,y:a},{x:o.x,y:a},o];return c=c.map(r),"M"+c[0]+"C"+c[1]+" "+c[2]+" "+c[3]}var t=mr,e=yr,r=Uo;return n.source=function(e){return arguments.length?(t=Et(e),n):t},n.target=function(t){return arguments.length?(e=Et(t),n):e},n.projection=function(t){return arguments.length?(r=t,n):r},n},ta.svg.diagonal.radial=function(){var n=ta.svg.diagonal(),t=Uo,e=n.projection;return n.projection=function(n){return arguments.length?e(jo(t=n)):t},n},ta.svg.symbol=function(){function n(n,r){return(zl.get(t.call(this,n,r))||Oo)(e.call(this,n,r))}var t=Ho,e=Fo;return n.type=function(e){return arguments.length?(t=Et(e),n):t},n.size=function(t){return arguments.length?(e=Et(t),n):e},n};var zl=ta.map({circle:Oo,cross:function(n){var t=Math.sqrt(n/5)/2;return"M"+-3*t+","+-t+"H"+-t+"V"+-3*t+"H"+t+"V"+-t+"H"+3*t+"V"+t+"H"+t+"V"+3*t+"H"+-t+"V"+t+"H"+-3*t+"Z"},diamond:function(n){var t=Math.sqrt(n/(2*Ll)),e=t*Ll;return"M0,"+-t+"L"+e+",0 0,"+t+" "+-e+",0Z"},square:function(n){var t=Math.sqrt(n)/2;return"M"+-t+","+-t+"L"+t+","+-t+" "+t+","+t+" "+-t+","+t+"Z"},"triangle-down":function(n){var t=Math.sqrt(n/ql),e=t*ql/2;return"M0,"+e+"L"+t+","+-e+" "+-t+","+-e+"Z"},"triangle-up":function(n){var t=Math.sqrt(n/ql),e=t*ql/2;return"M0,"+-e+"L"+t+","+e+" "+-t+","+e+"Z"}});ta.svg.symbolTypes=zl.keys();var ql=Math.sqrt(3),Ll=Math.tan(30*Da);_a.transition=function(n){for(var t,e,r=Tl||++Ul,u=Xo(n),i=[],o=Rl||{time:Date.now(),ease:Su,delay:0,duration:250},a=-1,c=this.length;++ai;i++){u.push(t=[]);for(var e=this[i],a=0,c=e.length;c>a;a++)(r=e[a])&&n.call(r,r.__data__,a,i)&&t.push(r)}return Yo(u,this.namespace,this.id)},Pl.tween=function(n,t){var e=this.id,r=this.namespace;return arguments.length<2?this.node()[r][e].tween.get(n):Y(this,null==t?function(t){t[r][e].tween.remove(n)}:function(u){u[r][e].tween.set(n,t)})},Pl.attr=function(n,t){function e(){this.removeAttribute(a)}function r(){this.removeAttributeNS(a.space,a.local)}function u(n){return null==n?e:(n+="",function(){var t,e=this.getAttribute(a);return e!==n&&(t=o(e,n),function(n){this.setAttribute(a,t(n))})})}function i(n){return null==n?r:(n+="",function(){var t,e=this.getAttributeNS(a.space,a.local);return e!==n&&(t=o(e,n),function(n){this.setAttributeNS(a.space,a.local,t(n))})})}if(arguments.length<2){for(t in n)this.attr(t,n[t]);return this}var o="transform"==n?Hu:mu,a=ta.ns.qualify(n);return Zo(this,"attr."+n,t,a.local?i:u)},Pl.attrTween=function(n,t){function e(n,e){var r=t.call(this,n,e,this.getAttribute(u));return r&&function(n){this.setAttribute(u,r(n))}}function r(n,e){var r=t.call(this,n,e,this.getAttributeNS(u.space,u.local));return r&&function(n){this.setAttributeNS(u.space,u.local,r(n))}}var u=ta.ns.qualify(n);return this.tween("attr."+n,u.local?r:e)},Pl.style=function(n,e,r){function u(){this.style.removeProperty(n)}function i(e){return null==e?u:(e+="",function(){var u,i=t(this).getComputedStyle(this,null).getPropertyValue(n);return i!==e&&(u=mu(i,e),function(t){this.style.setProperty(n,u(t),r)})})}var o=arguments.length;if(3>o){if("string"!=typeof n){2>o&&(e="");for(r in n)this.style(r,n[r],e);return this}r=""}return Zo(this,"style."+n,e,i)},Pl.styleTween=function(n,e,r){function u(u,i){var o=e.call(this,u,i,t(this).getComputedStyle(this,null).getPropertyValue(n));return o&&function(t){this.style.setProperty(n,o(t),r)}}return arguments.length<3&&(r=""),this.tween("style."+n,u)},Pl.text=function(n){return Zo(this,"text",n,Vo)},Pl.remove=function(){var n=this.namespace;return this.each("end.transition",function(){var t;this[n].count<2&&(t=this.parentNode)&&t.removeChild(this)})},Pl.ease=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].ease:("function"!=typeof n&&(n=ta.ease.apply(ta,arguments)),Y(this,function(r){r[e][t].ease=n}))},Pl.delay=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].delay:Y(this,"function"==typeof n?function(r,u,i){r[e][t].delay=+n.call(r,r.__data__,u,i)}:(n=+n,function(r){r[e][t].delay=n}))},Pl.duration=function(n){var t=this.id,e=this.namespace;return arguments.length<1?this.node()[e][t].duration:Y(this,"function"==typeof n?function(r,u,i){r[e][t].duration=Math.max(1,n.call(r,r.__data__,u,i))}:(n=Math.max(1,n),function(r){r[e][t].duration=n}))},Pl.each=function(n,t){var e=this.id,r=this.namespace;if(arguments.length<2){var u=Rl,i=Tl;try{Tl=e,Y(this,function(t,u,i){Rl=t[r][e],n.call(t,t.__data__,u,i)})}finally{Rl=u,Tl=i}}else Y(this,function(u){var i=u[r][e];(i.event||(i.event=ta.dispatch("start","end","interrupt"))).on(n,t)});return this},Pl.transition=function(){for(var n,t,e,r,u=this.id,i=++Ul,o=this.namespace,a=[],c=0,l=this.length;l>c;c++){a.push(n=[]);for(var t=this[c],s=0,f=t.length;f>s;s++)(e=t[s])&&(r=e[o][u],$o(e,s,o,i,{time:r.time,ease:r.ease,delay:r.delay+r.duration,duration:r.duration})),n.push(e)}return Yo(a,o,i)},ta.svg.axis=function(){function n(n){n.each(function(){var n,l=ta.select(this),s=this.__chart__||e,f=this.__chart__=e.copy(),h=null==c?f.ticks?f.ticks.apply(f,a):f.domain():c,g=null==t?f.tickFormat?f.tickFormat.apply(f,a):y:t,p=l.selectAll(".tick").data(h,f),v=p.enter().insert("g",".domain").attr("class","tick").style("opacity",Ca),d=ta.transition(p.exit()).style("opacity",Ca).remove(),m=ta.transition(p.order()).style("opacity",1),M=Math.max(u,0)+o,x=Ui(f),b=l.selectAll(".domain").data([0]),_=(b.enter().append("path").attr("class","domain"),ta.transition(b));v.append("line"),v.append("text");var w,S,k,E,A=v.select("line"),N=m.select("line"),C=p.select("text").text(g),z=v.select("text"),q=m.select("text"),L="top"===r||"left"===r?-1:1;if("bottom"===r||"top"===r?(n=Bo,w="x",k="y",S="x2",E="y2",C.attr("dy",0>L?"0em":".71em").style("text-anchor","middle"),_.attr("d","M"+x[0]+","+L*i+"V0H"+x[1]+"V"+L*i)):(n=Wo,w="y",k="x",S="y2",E="x2",C.attr("dy",".32em").style("text-anchor",0>L?"end":"start"),_.attr("d","M"+L*i+","+x[0]+"H0V"+x[1]+"H"+L*i)),A.attr(E,L*u),z.attr(k,L*M),N.attr(S,0).attr(E,L*u),q.attr(w,0).attr(k,L*M),f.rangeBand){var T=f,R=T.rangeBand()/2;s=f=function(n){return T(n)+R}}else s.rangeBand?s=f:d.call(n,f,s);v.call(n,s,f),m.call(n,f,f)})}var t,e=ta.scale.linear(),r=jl,u=6,i=6,o=3,a=[10],c=null;return n.scale=function(t){return arguments.length?(e=t,n):e},n.orient=function(t){return arguments.length?(r=t in Fl?t+"":jl,n):r},n.ticks=function(){return arguments.length?(a=arguments,n):a},n.tickValues=function(t){return arguments.length?(c=t,n):c},n.tickFormat=function(e){return arguments.length?(t=e,n):t},n.tickSize=function(t){var e=arguments.length;return e?(u=+t,i=+arguments[e-1],n):u},n.innerTickSize=function(t){return arguments.length?(u=+t,n):u},n.outerTickSize=function(t){return arguments.length?(i=+t,n):i},n.tickPadding=function(t){return arguments.length?(o=+t,n):o},n.tickSubdivide=function(){return arguments.length&&n},n};var jl="bottom",Fl={top:1,right:1,bottom:1,left:1};ta.svg.brush=function(){function n(t){t.each(function(){var t=ta.select(this).style("pointer-events.scala","all").style("-webkit-tap-highlight-color","rgba(0,0,0,0)").on("mousedown.brush",i).on("touchstart.brush",i),o=t.selectAll(".background").data([0]);o.enter().append("rect").attr("class","background").style("visibility","hidden").style("cursor","crosshair"),t.selectAll(".extent").data([0]).enter().append("rect").attr("class","extent").style("cursor","move");var a=t.selectAll(".resize").data(v,y);a.exit().remove(),a.enter().append("g").attr("class",function(n){return"resize "+n}).style("cursor",function(n){return Hl[n]}).append("rect").attr("x",function(n){return/[ew]$/.test(n)?-3:null}).attr("y",function(n){return/^[ns]/.test(n)?-3:null}).attr("width",6).attr("height",6).style("visibility","hidden"),a.style("display",n.empty()?"none":null);var c,f=ta.transition(t),h=ta.transition(o);l&&(c=Ui(l),h.attr("x",c[0]).attr("width",c[1]-c[0]),r(f)),s&&(c=Ui(s),h.attr("y",c[0]).attr("height",c[1]-c[0]),u(f)),e(f)})}function e(n){n.selectAll(".resize").attr("transform",function(n){return"translate("+f[+/e$/.test(n)]+","+h[+/^s/.test(n)]+")"})}function r(n){n.select(".extent").attr("x",f[0]),n.selectAll(".extent,.n>rect,.s>rect").attr("width",f[1]-f[0])}function u(n){n.select(".extent").attr("y",h[0]),n.selectAll(".extent,.e>rect,.w>rect").attr("height",h[1]-h[0])}function i(){function i(){32==ta.event.keyCode&&(C||(M=null,q[0]-=f[1],q[1]-=h[1],C=2),S())}function v(){32==ta.event.keyCode&&2==C&&(q[0]+=f[1],q[1]+=h[1],C=0,S())}function d(){var n=ta.mouse(b),t=!1;x&&(n[0]+=x[0],n[1]+=x[1]),C||(ta.event.altKey?(M||(M=[(f[0]+f[1])/2,(h[0]+h[1])/2]),q[0]=f[+(n[0]s?(u=r,r=s):u=s),v[0]!=r||v[1]!=u?(e?a=null:o=null,v[0]=r,v[1]=u,!0):void 0}function y(){d(),k.style("pointer-events.scala","all").selectAll(".resize").style("display",n.empty()?"none":null),ta.select("body").style("cursor",null),L.on("mousemove.brush",null).on("mouseup.brush",null).on("touchmove.brush",null).on("touchend.brush",null).on("keydown.brush",null).on("keyup.brush",null),z(),w({type:"brushend"})}var M,x,b=this,_=ta.select(ta.event.target),w=c.of(b,arguments),k=ta.select(b),E=_.datum(),A=!/^(n|s)$/.test(E)&&l,N=!/^(e|w)$/.test(E)&&s,C=_.classed("extent"),z=W(b),q=ta.mouse(b),L=ta.select(t(b)).on("keydown.brush",i).on("keyup.brush",v);if(ta.event.changedTouches?L.on("touchmove.brush",d).on("touchend.brush",y):L.on("mousemove.brush",d).on("mouseup.brush",y),k.interrupt().selectAll("*").interrupt(),C)q[0]=f[0]-q[0],q[1]=h[0]-q[1];else if(E){var T=+/w$/.test(E),R=+/^n/.test(E);x=[f[1-T]-q[0],h[1-R]-q[1]],q[0]=f[T],q[1]=h[R]}else ta.event.altKey&&(M=q.slice());k.style("pointer-events.scala","none").selectAll(".resize").style("display",null),ta.select("body").style("cursor",_.style("cursor")),w({type:"brushstart"}),d()}var o,a,c=E(n,"brushstart","brush","brushend"),l=null,s=null,f=[0,0],h=[0,0],g=!0,p=!0,v=Ol[0];return n.event=function(n){n.each(function(){var n=c.of(this,arguments),t={x:f,y:h,i:o,j:a},e=this.__chart__||t;this.__chart__=t,Tl?ta.select(this).transition().each("start.brush",function(){o=e.i,a=e.j,f=e.x,h=e.y,n({type:"brushstart"})}).tween("brush:brush",function(){var e=yu(f,t.x),r=yu(h,t.y);return o=a=null,function(u){f=t.x=e(u),h=t.y=r(u),n({type:"brush",mode:"resize"})}}).each("end.brush",function(){o=t.i,a=t.j,n({type:"brush",mode:"resize"}),n({type:"brushend"})}):(n({type:"brushstart"}),n({type:"brush",mode:"resize"}),n({type:"brushend"}))})},n.x=function(t){return arguments.length?(l=t,v=Ol[!l<<1|!s],n):l},n.y=function(t){return arguments.length?(s=t,v=Ol[!l<<1|!s],n):s},n.clamp=function(t){return arguments.length?(l&&s?(g=!!t[0],p=!!t[1]):l?g=!!t:s&&(p=!!t),n):l&&s?[g,p]:l?g:s?p:null},n.extent=function(t){var e,r,u,i,c;return arguments.length?(l&&(e=t[0],r=t[1],s&&(e=e[0],r=r[0]),o=[e,r],l.invert&&(e=l(e),r=l(r)),e>r&&(c=e,e=r,r=c),(e!=f[0]||r!=f[1])&&(f=[e,r])),s&&(u=t[0],i=t[1],l&&(u=u[1],i=i[1]),a=[u,i],s.invert&&(u=s(u),i=s(i)),u>i&&(c=u,u=i,i=c),(u!=h[0]||i!=h[1])&&(h=[u,i])),n):(l&&(o?(e=o[0],r=o[1]):(e=f[0],r=f[1],l.invert&&(e=l.invert(e),r=l.invert(r)),e>r&&(c=e,e=r,r=c))),s&&(a?(u=a[0],i=a[1]):(u=h[0],i=h[1],s.invert&&(u=s.invert(u),i=s.invert(i)),u>i&&(c=u,u=i,i=c))),l&&s?[[e,u],[r,i]]:l?[e,r]:s&&[u,i])},n.clear=function(){return n.empty()||(f=[0,0],h=[0,0],o=a=null),n},n.empty=function(){return!!l&&f[0]==f[1]||!!s&&h[0]==h[1]},ta.rebind(n,c,"on")};var Hl={n:"ns-resize",e:"ew-resize",s:"ns-resize",w:"ew-resize",nw:"nwse-resize",ne:"nesw-resize",se:"nwse-resize",sw:"nesw-resize"},Ol=[["n","e","s","w","nw","ne","se","sw"],["e","w"],["n","s"],[]],Il=ac.format=gc.timeFormat,Yl=Il.utc,Zl=Yl("%Y-%m-%dT%H:%M:%S.%LZ");Il.iso=Date.prototype.toISOString&&+new Date("2000-01-01T00:00:00.000Z")?Jo:Zl,Jo.parse=function(n){var t=new Date(n);return isNaN(t)?null:t},Jo.toString=Zl.toString,ac.second=Ft(function(n){return new cc(1e3*Math.floor(n/1e3))},function(n,t){n.setTime(n.getTime()+1e3*Math.floor(t))},function(n){return n.getSeconds()}),ac.seconds=ac.second.range,ac.seconds.utc=ac.second.utc.range,ac.minute=Ft(function(n){return new cc(6e4*Math.floor(n/6e4))},function(n,t){n.setTime(n.getTime()+6e4*Math.floor(t))},function(n){return n.getMinutes()}),ac.minutes=ac.minute.range,ac.minutes.utc=ac.minute.utc.range,ac.hour=Ft(function(n){var t=n.getTimezoneOffset()/60;return new cc(36e5*(Math.floor(n/36e5-t)+t))},function(n,t){n.setTime(n.getTime()+36e5*Math.floor(t))},function(n){return n.getHours()}),ac.hours=ac.hour.range,ac.hours.utc=ac.hour.utc.range,ac.month=Ft(function(n){return n=ac.day(n),n.setDate(1),n},function(n,t){n.setMonth(n.getMonth()+t)},function(n){return n.getMonth()}),ac.months=ac.month.range,ac.months.utc=ac.month.utc.range;var Vl=[1e3,5e3,15e3,3e4,6e4,3e5,9e5,18e5,36e5,108e5,216e5,432e5,864e5,1728e5,6048e5,2592e6,7776e6,31536e6],Xl=[[ac.second,1],[ac.second,5],[ac.second,15],[ac.second,30],[ac.minute,1],[ac.minute,5],[ac.minute,15],[ac.minute,30],[ac.hour,1],[ac.hour,3],[ac.hour,6],[ac.hour,12],[ac.day,1],[ac.day,2],[ac.week,1],[ac.month,1],[ac.month,3],[ac.year,1]],$l=Il.multi([[".%L",function(n){return n.getMilliseconds()}],[":%S",function(n){return n.getSeconds()}],["%I:%M",function(n){return n.getMinutes()}],["%I %p",function(n){return n.getHours()}],["%a %d",function(n){return n.getDay()&&1!=n.getDate()}],["%b %d",function(n){return 1!=n.getDate()}],["%B",function(n){return n.getMonth()}],["%Y",Ne]]),Bl={range:function(n,t,e){return ta.range(Math.ceil(n/e)*e,+t,e).map(Ko)},floor:y,ceil:y};Xl.year=ac.year,ac.scale=function(){return Go(ta.scale.linear(),Xl,$l)};var Wl=Xl.map(function(n){return[n[0].utc,n[1]]}),Jl=Yl.multi([[".%L",function(n){return n.getUTCMilliseconds()}],[":%S",function(n){return n.getUTCSeconds()}],["%I:%M",function(n){return n.getUTCMinutes()}],["%I %p",function(n){return n.getUTCHours()}],["%a %d",function(n){return n.getUTCDay()&&1!=n.getUTCDate()}],["%b %d",function(n){return 1!=n.getUTCDate()}],["%B",function(n){return n.getUTCMonth()}],["%Y",Ne]]);Wl.year=ac.year.utc,ac.scale.utc=function(){return Go(ta.scale.linear(),Wl,Jl)},ta.text=At(function(n){return n.responseText}),ta.json=function(n,t){return Nt(n,"application/json",Qo,t)},ta.html=function(n,t){return Nt(n,"text/html",na,t)},ta.xml=At(function(n){return n.responseXML}),"function"==typeof define&&define.amd?define(ta):"object"==typeof module&&module.exports&&(module.exports=ta),this.d3=ta}(); \ No newline at end of file diff --git a/test/com/linkedin/drelephant/BackfillTest.java b/test/com/linkedin/drelephant/BackfillTest.java deleted file mode 100644 index a67fb602e..000000000 --- a/test/com/linkedin/drelephant/BackfillTest.java +++ /dev/null @@ -1,681 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant; - -import com.google.common.collect.Lists; -import com.linkedin.drelephant.analysis.AnalyticJob; -import com.linkedin.drelephant.analysis.AnalyticJobGenerator; -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.ElephantBackfillFetcher; -import com.linkedin.drelephant.analysis.JobType; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.priorityexecutor.Priority; -import com.linkedin.drelephant.util.Utils; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Future; -import models.AppResult; -import models.BackfillInfo; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.junit.Before; -import org.junit.Test; -import play.test.FakeApplication; - -import static common.DBTestUtil.initDB; -import static common.TestConstants.APPLY_EVOLUTIONS_DEFAULT_KEY; -import static common.TestConstants.APPLY_EVOLUTIONS_DEFAULT_VALUE; -import static common.TestConstants.DB_DEFAULT_DRIVER_KEY; -import static common.TestConstants.DB_DEFAULT_DRIVER_VALUE; -import static common.TestConstants.DB_DEFAULT_URL_KEY; -import static common.TestConstants.DB_DEFAULT_URL_VALUE; -import static common.TestConstants.EVOLUTION_PLUGIN_KEY; -import static common.TestConstants.EVOLUTION_PLUGIN_VALUE; -import static common.TestConstants.TEST_SERVER_PORT; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.*; -import static play.test.Helpers.fakeApplication; -import static play.test.Helpers.running; -import static play.test.Helpers.testServer; - -/** - * Tests backfilling of jobs when Dr.Elephant comes up. - */ -public class BackfillTest { - - private static FakeApplication fakeApp; - - @Before - public void setup() { - Map dbConn = new HashMap(); - dbConn.put(DB_DEFAULT_DRIVER_KEY, DB_DEFAULT_DRIVER_VALUE); - dbConn.put(DB_DEFAULT_URL_KEY, DB_DEFAULT_URL_VALUE); - dbConn.put(EVOLUTION_PLUGIN_KEY, EVOLUTION_PLUGIN_VALUE); - dbConn.put(APPLY_EVOLUTIONS_DEFAULT_KEY, APPLY_EVOLUTIONS_DEFAULT_VALUE); - - fakeApp = fakeApplication(dbConn); - } - - private void populateTestData() { - try { - initDB(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void testGetBackfillData() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - List infos = BackfillInfo.find.select("*").findList(); - assertEquals(2, infos.size()); - for (BackfillInfo info : infos) { - if (info.appType.equals("MAPREDUCE")) { - assertEquals(123456789, info.backfillTs); - } else if (info.appType.equals("SPARK")) { - assertEquals(123456779, info.backfillTs); - } else { - fail("Unexpected app type retrieved from backfill_info table"); - } - } - - // Update backfill info. - BackfillInfo info = BackfillInfo.find.byId("MAPREDUCE"); - long ts = System.currentTimeMillis(); - info.appType = "MAPREDUCE"; - info.backfillTs = ts; - info.save(); - BackfillInfo infoAfterUpdate = BackfillInfo.find.select("*"). - where().eq("app_type", "MAPREDUCE").findUnique(); - assertEquals(ts, infoAfterUpdate.backfillTs); - - // Insert in backfill info. - info = BackfillInfo.find.byId("TEZ"); - assertNull(info); - info = new BackfillInfo(); - info.appType = "TEZ"; - info.backfillTs = ts; - info.save(); - BackfillInfo infoAfterInsert = BackfillInfo.find.select("*"). - where().eq("app_type", "TEZ").findUnique(); - assertEquals(ts, infoAfterInsert.backfillTs); - } - }); - } - - private static void assertAppResultEntry(String appId, long startTime, long finishTime, String appName) { - AppResult result = AppResult.find.byId(appId); - assertNotNull(appId + " should have been found in yarn_app_result_table", result); - assertEquals(startTime, result.startTime); - assertEquals(finishTime, result.finishTime); - assertEquals("default", result.queueName); - assertEquals("user", result.username); - assertEquals(appName, result.name); - assertEquals(Utils.getJobIdFromApplicationId(appId), result.jobExecId); - assertEquals("flow_123", result.flowExecId); - assertEquals("Azkaban", result.scheduler); - assertEquals(Severity.LOW, result.severity); - assertEquals(1, result.workflowDepth); - assertEquals(100, result.score); - } - - @Test - public void testBackfillWithNoDataInDB() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - List infos = BackfillInfo.find.findList(); - assertEquals(0, infos.size()); - - // Start runner with 2 threads. - ElephantContext.instance().getGeneralConf().setInt("drelephant.analysis.thread.count", 2); - final ElephantRunner runner = spy(new ElephantRunner()); - Map appsToPriorityMap = Collections.synchronizedMap(new LinkedHashMap()); - // 4 jobs to be returned. - List jobs = new ArrayList(); - jobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0001"). - setAppType(new ApplicationType("MAPREDUCE")).setName("mapreduce_job").setQueueName("default"). - setStartTime(1526520000000L).setFinishTime(1526520000050L).setUser("user")); - jobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0002"). - setAppType(new ApplicationType("SPARK")).setName("spark_job").setQueueName("default"). - setStartTime(1526600000000L).setFinishTime(1526600000050L).setUser("user")); - jobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0003"). - setAppType(new ApplicationType("SPARK")).setName("spark_job").setQueueName("default"). - setStartTime(1526700000000L).setFinishTime(1526700000050L).setUser("user")); - jobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0004"). - setAppType(new ApplicationType("MAPREDUCE")).setName("mapreduce_job").setQueueName("default"). - setStartTime(1526800000000L).setFinishTime(1526800000050L).setUser("user")); - doReturn(new DummyAnalyticJobGenerator(appsToPriorityMap, jobs)).when(runner).getAnalyticJobGenerator(); - Thread runnerThread = new Thread(new Runnable() { - @Override - public void run() { - runner.run(); - } - }); - runnerThread.start(); - - // Wait till results have been updated in DB. - List results = AppResult.find.findList(); - int resultCount = results.size(); - int loopCount = 400; - while (resultCount < 4 && loopCount > 0) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - e.printStackTrace(); - } - results = AppResult.find.findList(); - resultCount = results.size(); - loopCount--; - } - - // Check yarn_app_result. - assertEquals(4, resultCount); - assertAppResultEntry("application_1526555215992_0001", 1526520000000L, 1526520000050L, - "mapreduce_job"); - assertAppResultEntry("application_1526555215992_0002", 1526600000000L, 1526600000050L, - "spark_job"); - assertAppResultEntry("application_1526555215992_0003", 1526700000000L, 1526700000050L, - "spark_job"); - assertAppResultEntry("application_1526555215992_0004", 1526800000000L, 1526800000050L, - "mapreduce_job"); - - // Check backfill_info. - assertEquals(2, BackfillInfo.find.findList().size()); - BackfillInfo mapReduceInfo = BackfillInfo.find.byId("MAPREDUCE"); - assertEquals(1526800000050L, mapReduceInfo.backfillTs); - BackfillInfo sparkInfo = BackfillInfo.find.byId("SPARK"); - assertEquals(1526700000050L, sparkInfo.backfillTs); - - // Verify if apps were submitted with NORMAL priority. - assertEquals(4, appsToPriorityMap.size()); - for (Priority priority : appsToPriorityMap.values()) { - assertEquals(Priority.NORMAL, priority); - } - - // Ensure cleanup is performed. - assertTrue("App to analytic map should have been empty", runner.getAppToAnalyticJobMap().isEmpty()); - assertNotNull("MAPREDUCE app type should have final info", runner.getFinishTimeInfo("MAPREDUCE")); - assertTrue("Finish times map should be cleaned up for MAPREDUCE app type", - runner.getFinishTimeInfo("MAPREDUCE")._appFinishTimesMap.isEmpty()); - assertNotNull("SPARK app type should have final info", runner.getFinishTimeInfo("SPARK")); - assertTrue("Finish times map should be cleaned up for SPARK app type", - runner.getFinishTimeInfo("SPARK")._appFinishTimesMap.isEmpty()); - - // Stop the runner and do cleanup before exiting. - runner.kill(); - runnerThread.interrupt(); - try { - runnerThread.join(); - } catch (InterruptedException e) { - // Ignore the exception. - } - } - }); - } - - private static void assertBackfillInfo(String appType, long expectedBackfillTs) { - int loopCount = 800; - BackfillInfo info = BackfillInfo.find.byId(appType); - while ((info == null || info.backfillTs != expectedBackfillTs) && loopCount > 0) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - e.printStackTrace(); - } - info = BackfillInfo.find.byId(appType); - loopCount--; - } - assertEquals(expectedBackfillTs, info.backfillTs); - } - - @Test - public void testBackfillWithDataInDB() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - long backfillTs = 1526400000000L; - List results = AppResult.find.findList(); - int initialResultCount = results.size(); - BackfillInfo mapReduceBackfillInfo = BackfillInfo.find.byId("MAPREDUCE"); - assertNotNull(mapReduceBackfillInfo); - mapReduceBackfillInfo.backfillTs = backfillTs; - mapReduceBackfillInfo.save(); - BackfillInfo sparkBackfillInfo = BackfillInfo.find.byId("SPARK"); - assertNotNull(sparkBackfillInfo); - sparkBackfillInfo.backfillTs = backfillTs; - sparkBackfillInfo.save(); - BackfillInfo tezBackfillInfo = new BackfillInfo(); - tezBackfillInfo.appType = "TEZ"; - tezBackfillInfo.backfillTs = backfillTs; - tezBackfillInfo.save(); - List infos = BackfillInfo.find.findList(); - assertEquals(3, infos.size()); - - // Enable backfill and start runner with 1 thread. - ElephantContext.instance().getGeneralConf().setInt("drelephant.analysis.thread.count", 1); - ElephantContext.instance().getGeneralConf().setBoolean("drelephant.analysis.backfill.enabled", true); - final ElephantRunner runner = spy(new ElephantRunner()); - Map appsToPriorityMap = Collections.synchronizedMap(new LinkedHashMap()); - // 2 jobs to be returned by job generator. - // For each job, set job execution priority as MIN_PRIORITY. This acts as a sentinel value and we will wait - // during analysis till the value changes to avoid race while adding to appsToPriorityMap. - List jobs = new ArrayList(); - jobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0007"). - setAppType(new ApplicationType("MAPREDUCE")).setName("mapreduce_job_3").setQueueName("default"). - setStartTime(1526800000000L).setFinishTime(1526800000050L).setUser("user")); - jobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0008"). - setAppType(new ApplicationType("SPARK")).setName("spark_job_3").setQueueName("default"). - setStartTime(1526900000000L).setFinishTime(1526900000050L).setUser("user")); - long lowestTsFromRM = 1526800000050L; - AnalyticJobGenerator jobGenerator = new DummyAnalyticJobGenerator(appsToPriorityMap, jobs); - doReturn(jobGenerator).when(runner).getAnalyticJobGenerator(); - - // 6 jobs to be backfilled. - List mapReduceBackfillJobs = new ArrayList(); - mapReduceBackfillJobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob(). - setAppId("application_1526555215992_0001").setAppType(new ApplicationType("MAPREDUCE")). - setName("mapreduce_job_1").setQueueName("default").setStartTime(1526510000000L). - setFinishTime(1526520000150L).setUser("user").setJobExecutionPriority(Priority.MIN_PRIORITY)); - mapReduceBackfillJobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob(). - setAppId("application_1526555215992_0003").setAppType(new ApplicationType("MAPREDUCE")). - setName("mapreduce_job_2").setQueueName("default").setStartTime(1526520000000L). - setFinishTime(1526520000050L).setUser("user").setJobExecutionPriority(Priority.MIN_PRIORITY)); - DummyElephantBackfillFetcher mapreduceBackfillFetcher = - new DummyElephantBackfillFetcher(mapReduceBackfillJobs, "MAPREDUCE", backfillTs, lowestTsFromRM); - when(runner.getBackfillFetcher(eq(new ApplicationType("MAPREDUCE")))).thenReturn(mapreduceBackfillFetcher); - - List sparkBackfillJobs = new ArrayList(); - sparkBackfillJobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0002"). - setAppType(new ApplicationType("SPARK")).setName("spark_job_1").setQueueName("default"). - setStartTime(1526510050000L).setFinishTime(1526700000060L).setUser("user"). - setJobExecutionPriority(Priority.MIN_PRIORITY)); - sparkBackfillJobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0006"). - setAppType(new ApplicationType("SPARK")).setName("spark_job_2").setQueueName("default"). - setStartTime(1526700000000L).setFinishTime(1526700000050L).setUser("user"). - setJobExecutionPriority(Priority.MIN_PRIORITY)); - DummyElephantBackfillFetcher sparkBackfillFetcher = - new DummyElephantBackfillFetcher(sparkBackfillJobs, "SPARK", backfillTs, lowestTsFromRM); - when(runner.getBackfillFetcher(eq(new ApplicationType("SPARK")))).thenReturn(sparkBackfillFetcher); - - // 2 jobs below will not have finish time at the time of backfill. - List tezBackfillJobs = new ArrayList(); - tezBackfillJobs.add( - ((DummyAnalyticJobGenerator.DummyAnalyticJob) new DummyAnalyticJobGenerator.DummyAnalyticJob(). - setAppId("application_1526555215992_0004").setAppType(new ApplicationType("TEZ")). - setName("tez_job_1").setQueueName("default").setUser("user").setStartTime(1526600000000L)). - setFinishTimeAfterAnalysis(1526600000050L).setJobExecutionPriority(Priority.MIN_PRIORITY)); - tezBackfillJobs.add( - ((DummyAnalyticJobGenerator.DummyAnalyticJob) new DummyAnalyticJobGenerator.DummyAnalyticJob(). - setAppId("application_1526555215992_0005").setAppType(new ApplicationType("TEZ")). - setName("tez_job_2").setQueueName("default").setUser("user").setStartTime(1526600000020L)). - setFinishTimeAfterAnalysis(1526600000040L).setJobExecutionPriority(Priority.MIN_PRIORITY)); - ApplicationType tezAppType = new ApplicationType("TEZ"); - doReturn(tezAppType).when(runner).getAppTypeForName("TEZ"); - DummyElephantBackfillFetcher tezBackfillFetcher = new DummyElephantBackfillFetcher( - tezBackfillJobs, "TEZ", backfillTs, lowestTsFromRM); - when(runner.getBackfillFetcher(eq(tezAppType))).thenReturn(tezBackfillFetcher); - - Thread runnerThread = new Thread(new Runnable() { - @Override - public void run() { - runner.run(); - } - }); - runnerThread.start(); - - // Wait till results have been updated in DB. - results = AppResult.find.findList(); - int resultCount = results.size(); - int loopCount = 400; - while (resultCount < 8 + initialResultCount && loopCount > 0) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - e.printStackTrace(); - } - results = AppResult.find.findList(); - resultCount = results.size(); - loopCount--; - } - - // Check yarn_app_result. - assertEquals(8 + initialResultCount, resultCount); - assertAppResultEntry("application_1526555215992_0001", 1526510000000L, 1526520000150L, - "mapreduce_job_1"); - assertAppResultEntry("application_1526555215992_0002", 1526510050000L, 1526700000060L, - "spark_job_1"); - assertAppResultEntry("application_1526555215992_0003", 1526520000000L, 1526520000050L, - "mapreduce_job_2"); - assertAppResultEntry("application_1526555215992_0004", 1526600000000L, 1526600000050L, - "tez_job_1"); - assertAppResultEntry("application_1526555215992_0005", 1526600000020L, 1526600000040L, - "tez_job_2"); - assertAppResultEntry("application_1526555215992_0006", 1526700000000L, 1526700000050L, - "spark_job_2"); - assertAppResultEntry("application_1526555215992_0007", 1526800000000L, 1526800000050L, - "mapreduce_job_3"); - assertAppResultEntry("application_1526555215992_0008", 1526900000000L, 1526900000050L, - "spark_job_3"); - DummyElephantBackfillFetcher[] backfillFetchers = - new DummyElephantBackfillFetcher[] {mapreduceBackfillFetcher, sparkBackfillFetcher, tezBackfillFetcher}; - for (DummyElephantBackfillFetcher backfillFetcher : backfillFetchers) { - try { - assertTrue("Jobs for backfill (of type " + backfillFetcher.getFetcherName() + ") should have been " - + "fetched based on lowest ts of apps retrieved from RM and backfill ts in DB", - backfillFetcher.getJobsFetchedFlag()); - } catch (Exception e) { - fail("Unexpected exception."); - } - } - - // Verify apps submitted. - assertEquals(8, appsToPriorityMap.size()); - assertEquals(Lists.newArrayList(Priority.NORMAL, Priority.NORMAL, Priority.LOW, Priority.LOW, Priority.LOW, - Priority.LOW, Priority.LOW, Priority.LOW), new ArrayList(appsToPriorityMap.values())); - - String[] appTypes = new String[] {"MAPREDUCE", "SPARK", "TEZ"}; - for (String appType : appTypes) { - ElephantRunner.FinishTimeInfo finishTimeInfo = runner.getFinishTimeInfo(appType); - loopCount = 400; - while ((finishTimeInfo == null || !finishTimeInfo._appFinishTimesMap.isEmpty()) && loopCount > 0) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - e.printStackTrace(); - } - loopCount--; - } - } - - // Ensure cleanup is performed. - assertTrue("App to analytic map should have been empty", runner.getAppToAnalyticJobMap().isEmpty()); - assertNotNull("MAPREDUCE app type should have final info", runner.getFinishTimeInfo("MAPREDUCE")); - assertTrue("Finish times map should be cleaned up for MAPREDUCE app type", - runner.getFinishTimeInfo("MAPREDUCE")._appFinishTimesMap.isEmpty()); - assertNotNull("SPARK app type should have final info", runner.getFinishTimeInfo("SPARK")); - assertTrue("Finish times map should be cleaned up for SPARK app type", - runner.getFinishTimeInfo("SPARK")._appFinishTimesMap.isEmpty()); - assertNotNull("TEZ app type should have final info", runner.getFinishTimeInfo("TEZ")); - assertTrue("Finish times map should be cleaned up for TEZ app type", - runner.getFinishTimeInfo("TEZ")._appFinishTimesMap.isEmpty()); - - // Check backfill_info. - assertEquals(3, BackfillInfo.find.findList().size()); - assertBackfillInfo("MAPREDUCE", 1526800000050L); - assertBackfillInfo("SPARK", 1526900000050L); - - // Stop the runner and do cleanup before exiting. - runner.kill(); - runnerThread.interrupt(); - try { - runnerThread.join(); - } catch (InterruptedException e) { - // Ignore the exception. - } - } - }); - } - - @Test - public void testPrioritizeExecution() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - List infos = BackfillInfo.find.findList(); - assertEquals(0, infos.size()); - - // Start runner with 1 thread. - ElephantContext.instance().getGeneralConf().setInt("drelephant.analysis.thread.count", 1); - final ElephantRunner runner = spy(new ElephantRunner()); - Map appsToPriorityMap = Collections.synchronizedMap(new LinkedHashMap()); - // 3 jobs to be returned. - List jobs = new ArrayList(); - // First job will sleep for 5 seconds before analysis, if not interrupted. - jobs.add(((DummyAnalyticJobGenerator.DummyAnalyticJob) new DummyAnalyticJobGenerator.DummyAnalyticJob(). - setAppId("application_1526555215992_0001").setAppType(new ApplicationType("MAPREDUCE")). - setName("mapreduce_job").setQueueName("default").setStartTime(1526520000000L).setFinishTime(1526520000050L). - setUser("user")).setSleepBeforeAnalysis()); - jobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0002"). - setAppType(new ApplicationType("SPARK")).setName("spark_job").setQueueName("default"). - setStartTime(1526600000000L).setFinishTime(1526600000050L).setUser("user")); - jobs.add(new DummyAnalyticJobGenerator.DummyAnalyticJob().setAppId("application_1526555215992_0003"). - setAppType(new ApplicationType("SPARK")).setName("spark_job").setQueueName("default"). - setStartTime(1526700000000L).setFinishTime(1526700000050L).setUser("user")); - doReturn(new DummyAnalyticJobGenerator(appsToPriorityMap, jobs)).when(runner).getAnalyticJobGenerator(); - Thread runnerThread = new Thread(new Runnable() { - @Override - public void run() { - runner.run(); - } - }); - runnerThread.start(); - - // Wait till first app has been picked up by executor. - int loopCount = 400; - while (!runner.getAppToAnalyticJobMap().containsKey("application_1526555215992_0001") && loopCount > 0) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - e.printStackTrace(); - } - loopCount--; - } - assertTrue("Executor should have picked up application_1526555215992_0001 for execution", - runner.getAppToAnalyticJobMap().containsKey("application_1526555215992_0001")); - - // Prioritize application_1526555215992_0003. - Future future = runner.prioritizeExecutionAndReturnFuture("application_1526555215992_0003"); - assertNotNull("Priortization was not successful", future); - // Ensure first job proceeds ahead from sleep by interrupting it. - runner.getAppToAnalyticJobMap().get("application_1526555215992_0001").getJobFuture().cancel(true); - runner.waitTillPrioritizedJobAnalysisFinishes(future); - assertTrue("Prioritized job should have finished.", future.isDone()); - - // Wait till results have been updated in DB. - List results = AppResult.find.findList(); - int resultCount = results.size(); - loopCount = 400; - while (resultCount < 3 && loopCount > 0) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - e.printStackTrace(); - } - results = AppResult.find.findList(); - resultCount = results.size(); - loopCount--; - } - - // Check yarn_app_result. - assertEquals("yarn_app_result table should have 3 rows", 3, resultCount); - assertAppResultEntry("application_1526555215992_0001", 1526520000000L, 1526520000050L, - "mapreduce_job"); - assertAppResultEntry("application_1526555215992_0002", 1526600000000L, 1526600000050L, - "spark_job"); - assertAppResultEntry("application_1526555215992_0003", 1526700000000L, 1526700000050L, - "spark_job"); - - // Check backfill_info. - assertEquals("backfill_info table should have 2 rows", 2, BackfillInfo.find.findList().size()); - BackfillInfo mapReduceInfo = BackfillInfo.find.byId("MAPREDUCE"); - assertEquals(1526520000050L, mapReduceInfo.backfillTs); - BackfillInfo sparkInfo = BackfillInfo.find.byId("SPARK"); - assertEquals(1526700000050L, sparkInfo.backfillTs); - - // Verify if apps were submitted in correct order based on prioritization. - assertEquals(3, appsToPriorityMap.size()); - assertEquals(Lists.newArrayList("application_1526555215992_0001", "application_1526555215992_0003", - "application_1526555215992_0002"), new ArrayList(appsToPriorityMap.keySet())); - assertEquals(Lists.newArrayList(Priority.NORMAL, Priority.HIGH, Priority.NORMAL), - new ArrayList(appsToPriorityMap.values())); - - // Stop the runner and do cleanup before exiting. - runner.kill(); - runnerThread.interrupt(); - try { - runnerThread.join(); - } catch (InterruptedException e) { - // Ignore the exception. - } - } - }); - } - - private static class DummyAnalyticJobGenerator implements AnalyticJobGenerator { - private static Map _appsToPriorityMap; - private List _analyticJobs; - - private DummyAnalyticJobGenerator(Map appsMap, List jobs) { - _appsToPriorityMap = appsMap; - _analyticJobs = jobs; - } - - @Override - public void configure(Configuration configuration) throws IOException { - } - - @Override - public void updateResourceManagerAddresses() { - } - - @Override - public List fetchAnalyticJobs() throws IOException, AuthenticationException { - return _analyticJobs; - } - - @Override - public void addIntoRetries(AnalyticJob job) { - } - - @Override - public void addIntoSecondRetryQueue(AnalyticJob job) { - } - - @Override - public String getEffectiveResourceManagerAddress() { - return "localhost:8188"; - } - - @Override - public long getFetchStartTime() { - return 0; - } - - private static class DummyAnalyticJob extends AnalyticJob { - private long _finishTimeAfterAnalysis = -1; - // For prioritization tests, sleep for 5 seconds before analysis. - // Interruption is caught and ignored. - private boolean _sleepBeforeAnalysis = false; - @Override - public AppResult getAnalysis() throws Exception { - if (_sleepBeforeAnalysis) { - try { - Thread.sleep(5000L); - } catch (InterruptedException e) { - // Ignore and continue with analysis. - } - } - AppResult result = new AppResult(); - result.id = getAppId(); - result.finishTime = getFinishTime() <= 0 ? _finishTimeAfterAnalysis : getFinishTime(); - result.jobName = getName(); - result.queueName = getQueueName(); - result.trackingUrl = getTrackingUrl() == null ? "" : getTrackingUrl(); - result.name = getName(); - result.username = getUser(); - result.startTime = getStartTime(); - List jobTypes = ElephantContext.instance().getAppTypeToJobTypes().get(getAppType()); - result.jobType = ((jobTypes == null) ? "" : jobTypes.get(0).getName()); - result.severity = Severity.LOW; - result.score = 100; - result.workflowDepth = 1; - result.scheduler = "Azkaban"; - result.jobExecId = Utils.getJobIdFromApplicationId(getAppId()); - result.flowExecId = "flow_123"; - result.jobDefId = "job_def_123"; - result.flowDefId = "flow_def_123"; - result.jobExecUrl = "job_url"; - result.flowExecUrl = "flow_url"; - result.jobDefUrl = "job_def_url"; - result.flowDefUrl = "flow_def_url"; - // Wait for execution priority to be set, if not already set, to avoid race. - int loopCount = 400; - while (getJobExecutionPriority() == Priority.MIN_PRIORITY && loopCount > 0) { - try { - Thread.sleep(50); - } catch (InterruptedException e) { - // Ignore. - } - loopCount--; - } - _appsToPriorityMap.put(getAppId(), getJobExecutionPriority()); - return result; - } - - DummyAnalyticJob setFinishTimeAfterAnalysis(long finishTime) { - _finishTimeAfterAnalysis = finishTime; - return this; - } - - DummyAnalyticJob setSleepBeforeAnalysis() { - _sleepBeforeAnalysis = true; - return this; - } - } - } - - private static class DummyElephantBackfillFetcher implements ElephantBackfillFetcher { - private List _backfillJobs; - private long _backfillTs; - private long _lowestTsFromRM; - private volatile boolean _jobsFetchedFlag = false; - private String _fetcherName; - private DummyElephantBackfillFetcher(List jobs, String fetcherName, long backfillTs, long lowestTsFromRM) { - _backfillJobs = jobs; - _backfillTs = backfillTs; - _lowestTsFromRM = lowestTsFromRM; - _fetcherName = fetcherName; - } - - @Override - public List fetchJobsForBackfill(long startTime, long endTime) throws Exception { - // Verify whether jobs were backfilled based on lowest ts of apps retrieved from RM and backfill ts in DB. - if ((startTime == _backfillTs - ElephantRunner.BACKFILL_BUFFER_TIME) && (endTime == _lowestTsFromRM)) { - _jobsFetchedFlag = true; - } - return _backfillJobs; - } - - private boolean getJobsFetchedFlag() { - return _jobsFetchedFlag; - } - - private String getFetcherName() { - return _fetcherName; - } - } -} diff --git a/test/com/linkedin/drelephant/analysis/AnalyticJobGeneratorHadoop2Test.java b/test/com/linkedin/drelephant/analysis/AnalyticJobGeneratorHadoop2Test.java deleted file mode 100644 index e0bebbfb2..000000000 --- a/test/com/linkedin/drelephant/analysis/AnalyticJobGeneratorHadoop2Test.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.analysis; - -import java.util.ArrayList; -import java.util.ConcurrentModificationException; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - - -/** - * Unit tests for class {@link AnalyticJobGeneratorHadoop2}. - */ -public class AnalyticJobGeneratorHadoop2Test { - /** - * Tests concurrent operations (fetch and add) on second retry queue. - */ - @Test - public void testSecondRetryQueueConcurrentOperations() { - final AnalyticJobGeneratorHadoop2 analyticJobGenerator = - new AnalyticJobGeneratorHadoop2(); - - // Latch to ensure operations on second retry queue from multiple threads - // run in parallel - final CountDownLatch latch = new CountDownLatch(1); - - // Add a job into second retry queue. - AnalyticJob job1 = spy(new AnalyticJob()); - // Custom answer on call to readyForSecondRetry for this job. - doAnswer(new Answer() { - @Override - public Boolean answer(final InvocationOnMock invocation) throws Throwable { - // Wait for additions to second retry queue from add jobs thread to - // begin. In case of synchronized access, wait will be for 5 sec. - // In case of unsynchronized access, this would lead to - // ConcurrentModificationException. - latch.await(5000L, TimeUnit.MILLISECONDS); - return true ; - } - }).when(job1).readyForSecondRetry(); - analyticJobGenerator.addIntoSecondRetryQueue(job1); - - // Add couple of other jobs to second retry queue. - AnalyticJob job2 = spy(new AnalyticJob()); - when(job2.readyForSecondRetry()).thenReturn(false); - analyticJobGenerator.addIntoSecondRetryQueue(job2); - - AnalyticJob job3 = spy(new AnalyticJob()); - when(job3.readyForSecondRetry()).thenReturn(true); - analyticJobGenerator.addIntoSecondRetryQueue(job3); - - final List appList = new ArrayList(); - // Flag to indicate if ConcurrentModificationException has been thrown. - final AtomicBoolean cmExceptionFlag = new AtomicBoolean(false); - // Start a fetch jobs thread which calls fetchJobsFromSecondRetryQueue - // method. - Thread fetchJobsThread = new Thread(new Runnable() { - @Override - public void run() { - try { - analyticJobGenerator.fetchJobsFromSecondRetryQueue(appList); - } catch (ConcurrentModificationException e) { - cmExceptionFlag.set(true); - } - } - }); - fetchJobsThread.start(); - - // Start a add jobs jobs thread which adds a couple of jobs into second - // retry queue while fetch jobs thread is running in parallel. - Thread addJobsThread = new Thread(new Runnable() { - @Override - public void run() { - AnalyticJob job4 = spy(new AnalyticJob()); - when(job4.readyForSecondRetry()).thenReturn(false); - analyticJobGenerator.addIntoSecondRetryQueue(job4); - - // Latch countdown to ensure fetch jobs thread can continue. - latch.countDown(); - - AnalyticJob job5 = spy(new AnalyticJob()); - when(job5.readyForSecondRetry()).thenReturn(true); - analyticJobGenerator.addIntoSecondRetryQueue(job5); - } - }); - addJobsThread.start(); - - // Wait for both the threads to finish. - try { - fetchJobsThread.join(); - addJobsThread.join(); - } catch (InterruptedException e) { - // Ignore the exception. - } - - // Concurrent operations from multiple threads should not lead to - // ConcurrentModificationException as accesses to second retry queue are - // synchronized. - assertFalse("ConcurrentModificationException should not have been thrown " + - "while fetching jobs", cmExceptionFlag.get()); - // Checking for apps >= 2 as the exact number can be 2 or 3 depending on - // order of invocation of threads. - assertTrue("Apps fetched from second retry queue should be >= 2.", - appList.size() >= 2); - - // Drain the second retry queue by fetching jobs from it. - analyticJobGenerator.fetchJobsFromSecondRetryQueue(appList); - assertEquals("Apps fetched from second retry queue should be 3.", 3, - appList.size()); - } -} diff --git a/test/com/linkedin/drelephant/analysis/AnalyticJobTest.java b/test/com/linkedin/drelephant/analysis/AnalyticJobTest.java deleted file mode 100644 index 06b5dae36..000000000 --- a/test/com/linkedin/drelephant/analysis/AnalyticJobTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.analysis; - -import com.linkedin.drelephant.ElephantContext; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.mapreduce.fetchers.MapReduceFetcherHadoop2; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic; -import common.TestUtil; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import mockit.Expectations; -import mockit.Mocked; -import models.AppResult; -import org.junit.Ignore; -import org.junit.Test; - -import static common.TestConstants.*; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - - -/** - * Test aims to exercise {@code getAnalysis()} method in {@code AnalyticJob}.
- * Dependencies to {@code ElephantContext, ElephantFetcher and Heuristics} are mocked - * out with JMockit. - */ -@Ignore -public class AnalyticJobTest { - @Mocked(stubOutClassInitialization = true) - ElephantContext elephantContext = null; - @Mocked - MapReduceFetcherHadoop2 fetcher; - - @Test - public void testGetAnalysis() - throws Exception { - try { - // Setup analytic job - final AnalyticJob analyticJob = new AnalyticJob(). - setAppId(TEST_JOB_ID1).setAppType(new ApplicationType(TEST_APP_TYPE)). - setFinishTime(1462178403).setStartTime(1462178412).setName(TEST_JOB_NAME). - setQueueName(TEST_DEFAULT_QUEUE_NAME).setUser(TEST_USERNAME).setTrackingUrl(TEST_TRACKING_URL); - - // Setup job counter data - String filePath = FILENAME_JOBCOUNTER; - MapReduceCounterData jobCounter = new MapReduceCounterData(); - setCounterData(jobCounter, filePath); - - // Setup mapper data - long[][] mapperTasksTime = {{2563, 0, 0, 0, 0}, {2562, 0, 0, 0, 0}, {2567, 0, 0, 0, 0}}; - MapReduceTaskData[] mappers = new MapReduceTaskData[3]; - for (int i = 1; i <= mappers.length; i++) { - MapReduceCounterData taskCounter = new MapReduceCounterData(); - setCounterData(taskCounter, FILENAME_MAPPERTASK.replaceFirst("\\$", Integer.toString(i))); - mappers[i - 1 ] = new MapReduceTaskData("task-id-"+(i-1), "task-attempt-id-"+(i-1)); - mappers[i - 1].setTimeAndCounter(mapperTasksTime[i - 1], taskCounter); - } - - // Setup reducer data - long[][] reducerTasksTime = {{1870, 1665, 14, 0, 0}}; - MapReduceTaskData[] reducers = new MapReduceTaskData[1]; - for (int i = 1; i <= reducers.length; i++) { - MapReduceCounterData taskCounter = new MapReduceCounterData(); - setCounterData(taskCounter, FILENAME_REDUCERTASK.replaceFirst("\\$", Integer.toString(i))); - reducers[i - 1] = new MapReduceTaskData("task-id-"+(i-1), "task-attempt-id-"+(i-1)); - reducers[i - 1].setTimeAndCounter(reducerTasksTime[i - 1], taskCounter); - } - - // Setup job configuration data - filePath = FILENAME_JOBCONF; - Properties jobConf = TestUtil.loadProperties(filePath); - - // Setup application data - final MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter). - setMapperData(mappers).setReducerData(reducers).setJobConf(jobConf).setSucceeded(true). - setDiagnosticInfo("").setUsername(TEST_USERNAME).setUrl("").setJobName(TEST_JOB_NAME). - setStartTime(1462178412).setFinishTime(1462178403).setRetry(false).setAppId(TEST_JOB_ID1); - - // Setup heuristics - final List heuristics = loadHeuristics(); - - // Setup job type - final JobType jobType = new JobType(TEST_JOB_TYPE, TEST_JOBCONF_NAME, TEST_JOBCONF_PATTERN); - - // Set expectations in JMockit - new Expectations() {{ - fetcher.fetchData(analyticJob); - result = data; - - elephantContext.getHeuristicsForApplicationType(analyticJob.getAppType()); - result = heuristics; - - elephantContext.matchJobType(data); - result = jobType; - }}; - - // Call the method under test - AppResult result = analyticJob.getAnalysis(); - - // Make assertions on result - assertTrue("Result is null", result != null); - assertTrue("Score did not match", result.score == TEST_SCORE); - assertTrue("Severity did not match", result.severity.toString().equals(TEST_SEVERITY)); - assertTrue("APP ID did not match", result.id.equals(TEST_JOB_ID1)); - assertTrue("Scheduler did not match", result.scheduler.equals(TEST_SCHEDULER)); - } catch (Exception e) { - e.printStackTrace(); - assertFalse("Test failed with exception", true); - } - } - - private void setCounterData(MapReduceCounterData counter, String filePath) - throws IOException { - Properties counterData = TestUtil.loadProperties(filePath); - - for (Object groupName : counterData.keySet()) { - String counterValueString = (String) counterData.get(groupName); - counterValueString = counterValueString.replaceAll("\\{|\\}", ""); - - StringBuilder stringBuilder = new StringBuilder(); - - for (String counterKeyValue : counterValueString.split(",")) { - stringBuilder.append(counterKeyValue.trim()).append('\n'); - } - ByteArrayInputStream inputStream = new ByteArrayInputStream(stringBuilder.toString().getBytes(DEFAULT_ENCODING)); - Properties counterProperties = new Properties(); - counterProperties.load(inputStream); - - for (Object counterKey : counterProperties.keySet()) { - long counterValue = Long.parseLong(counterProperties.get(counterKey).toString()); - counter.set(groupName.toString(), counterKey.toString(), counterValue); - } - } - } - - private List loadHeuristics() { - List heuristics = new ArrayList(); - // dummy hash map - Map paramsMap = new HashMap(); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Mapper Skew", - "com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic", - "views.html.help.mapreduce.helpMapperSkew", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic( - new HeuristicConfigurationData("Mapper GC", "com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic", - "views.html.help.mapreduce.helpGC", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Mapper Time", - "com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic", "views.html.help.mapreduce.helpMapperTime", - new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Mapper Speed", - "com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic", - "views.html.help.mapreduce.helpMapperSpeed", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Mapper Spill", - "com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic", - "views.html.help.mapreduce.helpMapperSpill", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Mapper Memory", - "com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic", - "views.html.help.mapreduce.helpMapperMemory", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Reducer Skew", - "com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic", - "views.html.help.mapreduce.helpReducerSkew", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic( - new HeuristicConfigurationData("Reducer GC", "com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic", - "views.html.help.mapreduce.helpGC", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Reducer Time", - "com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic", - "views.html.help.mapreduce.helpReducerTime", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Reducer Memory", - "com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic", - "views.html.help.mapreduce.helpReducerMemory", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic(new HeuristicConfigurationData("Shuffle & Sort", - "com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic", - "views.html.help.mapreduce.helpShuffleSort", new ApplicationType("mapreduce"), paramsMap))); - heuristics.add(new MapperSkewHeuristic( - new HeuristicConfigurationData("Exception", "com.linkedin.drelephant.mapreduce.heuristics.ExceptionHeuristic", - "views.html.help.mapreduce.helpException", new ApplicationType("mapreduce"), paramsMap))); - - return heuristics; - } -} diff --git a/test/com/linkedin/drelephant/analysis/SeverityTest.java b/test/com/linkedin/drelephant/analysis/SeverityTest.java deleted file mode 100644 index 7f54accfc..000000000 --- a/test/com/linkedin/drelephant/analysis/SeverityTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.analysis; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - - -public class SeverityTest { - - @Test - public void testSeverityMax() { - assertEquals(Severity.CRITICAL, Severity.max(Severity.CRITICAL)); - assertEquals(Severity.CRITICAL, Severity.max(Severity.CRITICAL, Severity.SEVERE)); - assertEquals(Severity.CRITICAL, Severity.max(Severity.LOW, Severity.LOW, Severity.CRITICAL)); - } - - @Test - public void testSeverityMin() { - assertEquals(Severity.NONE, Severity.min(Severity.NONE, Severity.LOW)); - assertEquals(Severity.LOW, Severity.min(Severity.LOW, Severity.LOW)); - } - - @Test - public void testSeverityAscending() { - assertEquals(Severity.CRITICAL, Severity.getSeverityAscending(8, 2, 4, 6, 8)); - assertEquals(Severity.SEVERE, Severity.getSeverityAscending(10, 2, 4, 6, 12)); - } - - @Test - public void testSeverityDescending() { - assertEquals(Severity.CRITICAL, Severity.getSeverityDescending(2, 10, 8, 4, 2)); - assertEquals(Severity.MODERATE, Severity.getSeverityDescending(5, 10, 8, 4, 2)); - } -} diff --git a/test/com/linkedin/drelephant/analysis/SeverityThresholdsTest.scala b/test/com/linkedin/drelephant/analysis/SeverityThresholdsTest.scala deleted file mode 100644 index 488118e01..000000000 --- a/test/com/linkedin/drelephant/analysis/SeverityThresholdsTest.scala +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.analysis - -import org.scalatest.{FunSpec, Matchers} - -class SeverityThresholdsTest extends FunSpec with Matchers { - describe("SeverityThresholds") { - it("can be used to represent thresholds considered in ascending order") { - val thresholds = SeverityThresholds(low = 0.2D, moderate = 0.4D, severe = 0.6D, critical = 0.8D, ascending = true) - thresholds.severityOf(0.1D) should be(Severity.NONE) - thresholds.severityOf(0.2D) should be(Severity.LOW) - thresholds.severityOf(0.3D) should be(Severity.LOW) - thresholds.severityOf(0.4D) should be(Severity.MODERATE) - thresholds.severityOf(0.5D) should be(Severity.MODERATE) - thresholds.severityOf(0.6D) should be(Severity.SEVERE) - thresholds.severityOf(0.7D) should be(Severity.SEVERE) - thresholds.severityOf(0.8D) should be(Severity.CRITICAL) - thresholds.severityOf(0.9D) should be(Severity.CRITICAL) - } - - it("can be used to represent thresholds considered in descending order") { - val thresholds = SeverityThresholds(low = 0.8D, moderate = 0.6D, severe = 0.4D, critical = 0.2D, ascending = false) - thresholds.severityOf(0.1D) should be(Severity.CRITICAL) - thresholds.severityOf(0.2D) should be(Severity.CRITICAL) - thresholds.severityOf(0.3D) should be(Severity.SEVERE) - thresholds.severityOf(0.4D) should be(Severity.SEVERE) - thresholds.severityOf(0.5D) should be(Severity.MODERATE) - thresholds.severityOf(0.6D) should be(Severity.MODERATE) - thresholds.severityOf(0.7D) should be(Severity.LOW) - thresholds.severityOf(0.8D) should be(Severity.LOW) - thresholds.severityOf(0.9D) should be(Severity.NONE) - } - - it("can be parsed as ascending thresholds from a string that can be processed by Utils.getParam") { - SeverityThresholds.parse("0.2,0.4,0.6,0.8", ascending = true) should be( - Some(SeverityThresholds(low = 0.2D, moderate = 0.4D, severe = 0.6D, critical = 0.8D, ascending = true)) - ) - } - - it("can be parsed as descending thresholds from a string that can be processed by Utils.getParam") { - SeverityThresholds.parse("0.8,0.6,0.4,0.2", ascending = false) should be( - Some(SeverityThresholds(low = 0.8D, moderate = 0.6D, severe = 0.4D, critical = 0.2D, ascending = false)) - ) - } - - it("cannot be created as ascending thresholds with unordered values") { - an[IllegalArgumentException] should be thrownBy( - SeverityThresholds(low = 0.8D, moderate = 0.6D, severe = 0.4D, critical = 0.2D, ascending = true) - ) - } - - it("cannot be created as descending thresholds with unordered values") { - an[IllegalArgumentException] should be thrownBy( - SeverityThresholds(low = 0.2D, moderate = 0.4D, severe = 0.6D, critical = 0.8D, ascending = false) - ) - } - } -} diff --git a/test/com/linkedin/drelephant/configurations/fetcher/FetcherConfigurationTest.java b/test/com/linkedin/drelephant/configurations/fetcher/FetcherConfigurationTest.java deleted file mode 100644 index 2db6431c4..000000000 --- a/test/com/linkedin/drelephant/configurations/fetcher/FetcherConfigurationTest.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.configurations.fetcher; - -import java.io.IOException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; - - -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import static org.junit.Assert.assertEquals; - - -public class FetcherConfigurationTest { - - private static Document document1 = null; - private static Document document2 = null; - private static Document document3 = null; - private static Document document4 = null; - private static Document document5 = null; - - private static final String spark = "SPARK"; - private static final String logDirField = "event_log_dir"; - private static final String logDirValue = "/custom/configured"; - private static final String logSizeField = "event_log_size_limit_in_mb"; - private static final String logSizeValue = "50"; - - - @BeforeClass - public static void runBeforeClass() { - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - document1 = builder.parse( - FetcherConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/fetcher/FetcherConfTest1.xml")); - document2 = builder.parse( - FetcherConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/fetcher/FetcherConfTest2.xml")); - document3 = builder.parse( - FetcherConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/fetcher/FetcherConfTest3.xml")); - document4 = builder.parse( - FetcherConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/fetcher/FetcherConfTest4.xml")); - document5 = builder.parse( - FetcherConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/fetcher/FetcherConfTest5.xml")); - } catch (ParserConfigurationException e) { - throw new RuntimeException("XML Parser could not be created.", e); - } catch (SAXException e) { - throw new RuntimeException("Test files are not properly formed", e); - } catch (IOException e) { - throw new RuntimeException("Unable to read test files ", e); - } - } - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - /** - * Correctly configured fetcher - */ - @Test - public void testParseFetcherConf1() { - FetcherConfiguration fetcherConf = new FetcherConfiguration(document1.getDocumentElement()); - assertEquals(fetcherConf.getFetchersConfigurationData().size(), 2); - } - - /** - * No classname field - */ - @Test - public void testParseFetcherConf2() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'classname' in fetcher 2"); - FetcherConfiguration fetcherConf = new FetcherConfiguration(document2.getDocumentElement()); - } - - /** - * Empty classname field - */ - @Test - public void testParseFetcherConf3() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("Empty tag 'classname' in fetcher 1"); - FetcherConfiguration fetcherConf = new FetcherConfiguration(document3.getDocumentElement()); - } - - /** - * No applicationtype tag - */ - @Test - public void testParseFetcherConf4() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag or invalid tag 'applicationtype' in fetcher 1" - + " classname com.linkedin.drelephant.mapreduce.fetchers.MapReduceFetcherHadoop2"); - FetcherConfiguration fetcherConf = new FetcherConfiguration(document4.getDocumentElement()); - } - - /** - * Test Spark fetcher params, Event log size and log directory - */ - @Test - public void testParseFetcherConf5() { - FetcherConfiguration fetcherConf = new FetcherConfiguration(document5.getDocumentElement()); - assertEquals(fetcherConf.getFetchersConfigurationData().size(), 1); - assertEquals(fetcherConf.getFetchersConfigurationData().get(0).getAppType().getName(), spark); - } - -} diff --git a/test/com/linkedin/drelephant/configurations/heuristic/HeuristicConfigurationTest.java b/test/com/linkedin/drelephant/configurations/heuristic/HeuristicConfigurationTest.java deleted file mode 100644 index 16c3c8a6f..000000000 --- a/test/com/linkedin/drelephant/configurations/heuristic/HeuristicConfigurationTest.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.configurations.heuristic; - -import java.io.IOException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import static org.junit.Assert.assertEquals; - - -public class HeuristicConfigurationTest { - - private static Document document1 = null; - private static Document document2 = null; - private static Document document3 = null; - private static Document document4 = null; - private static Document document5 = null; - - @BeforeClass - public static void runBeforeClass() { - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - document1 = builder.parse(HeuristicConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/heuristic/HeuristicConfTest1.xml")); - document2 = builder.parse( - HeuristicConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/heuristic/HeuristicConfTest2.xml")); - document3 = builder.parse( - HeuristicConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/heuristic/HeuristicConfTest3.xml")); - document4 = builder.parse( - HeuristicConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/heuristic/HeuristicConfTest4.xml")); - document5 = builder.parse( - HeuristicConfigurationTest.class.getClassLoader().getResourceAsStream( - "configurations/heuristic/HeuristicConfTest5.xml")); - } catch (ParserConfigurationException e) { - throw new RuntimeException("XML Parser could not be created.", e); - } catch (SAXException e) { - throw new RuntimeException("Test files are not properly formed", e); - } catch (IOException e) { - throw new RuntimeException("Unable to read test files ", e); - } - } - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - /** - * Correctly configured fetcher - */ - @Test - public void testParseFetcherConf1() { - HeuristicConfiguration heuristicConf = new HeuristicConfiguration(document1.getDocumentElement()); - assertEquals(heuristicConf.getHeuristicsConfigurationData().size(), 3); - } - - /** - * No classname tag - */ - @Test - public void testParseFetcherConf2() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'classname' in heuristic 1"); - HeuristicConfiguration heuristicConf = new HeuristicConfiguration(document2.getDocumentElement()); - } - - /** - * No heuristic name tag - */ - @Test - public void testParseFetcherConf3() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'heuristicname' in heuristic 1 classname" - + " com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic"); - HeuristicConfiguration heuristicConf = new HeuristicConfiguration(document3.getDocumentElement()); - } - - /** - * No view name tag - */ - @Test - public void testParseFetcherConf4() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'viewname' in heuristic 1 classname" - + " com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic"); - HeuristicConfiguration heuristicConf = new HeuristicConfiguration(document4.getDocumentElement()); - } - - /** - * No application type tag - */ - @Test - public void testParseFetcherConf5() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag or invalid tag 'applicationtype' in heuristic 2 classname" - + " com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic"); - HeuristicConfiguration heuristicConf = new HeuristicConfiguration(document5.getDocumentElement()); - } -} - diff --git a/test/com/linkedin/drelephant/configurations/jobtype/JobTypeConfigurationTest.java b/test/com/linkedin/drelephant/configurations/jobtype/JobTypeConfigurationTest.java deleted file mode 100644 index a99ebd9f2..000000000 --- a/test/com/linkedin/drelephant/configurations/jobtype/JobTypeConfigurationTest.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.configurations.jobtype; - -import java.io.IOException; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import static org.junit.Assert.assertEquals; - - -public class JobTypeConfigurationTest { - - private static Document document1 = null; - private static Document document2 = null; - private static Document document3 = null; - private static Document document4 = null; - private static Document document5 = null; - private static Document document6 = null; - - @BeforeClass - public static void runBeforeClass() { - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - document1 = builder.parse(JobTypeConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/jobtype/JobTypeConfTest1.xml")); - document2 = builder.parse(JobTypeConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/jobtype/JobTypeConfTest2.xml")); - document3 = builder.parse(JobTypeConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/jobtype/JobTypeConfTest3.xml")); - document4 = builder.parse(JobTypeConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/jobtype/JobTypeConfTest4.xml")); - document5 = builder.parse(JobTypeConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/jobtype/JobTypeConfTest5.xml")); - document6 = builder.parse(JobTypeConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/jobtype/JobTypeConfTest6.xml")); - } catch (ParserConfigurationException e) { - throw new RuntimeException("XML Parser could not be created.", e); - } catch (SAXException e) { - throw new RuntimeException("Test files are not properly formed", e); - } catch (IOException e) { - throw new RuntimeException("Unable to read test files ", e); - } - } - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - /** - * Correctly configured fetcher - */ - @Test - public void testParseFetcherConf1() { - JobTypeConfiguration jobTypeConf = new JobTypeConfiguration(document1.getDocumentElement()); - assertEquals(jobTypeConf.getAppTypeToJobTypeList().size(), 2); - } - - /** - * No name tag - */ - @Test - public void testParseFetcherConf2() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'jobtype' in jobtype 3"); - JobTypeConfiguration jobTypeConf = new JobTypeConfiguration(document2.getDocumentElement()); - } - - /** - * No conf tag - */ - @Test - public void testParseFetcherConf3() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'conf' in jobtype Spark"); - JobTypeConfiguration jobTypeConf = new JobTypeConfiguration(document3.getDocumentElement()); - } - - /** - * No applicationtype tag - */ - @Test - public void testParseFetcherConf4() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'applicationtype' in jobtype Pig"); - JobTypeConfiguration jobTypeConf = new JobTypeConfiguration(document4.getDocumentElement()); - } - - /** - * Wrong pattern for job type - */ - @Test - public void testParseFetcherConf5() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("Error processing this pattern. Pattern:[(voldemort) jobtype:Voldemort"); - JobTypeConfiguration jobTypeConf = new JobTypeConfiguration(document5.getDocumentElement()); - } - - /** - * Multiple default types - */ - @Test - public void testParseFetcherConf6() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("Each application type should have one and only one default job type. Duplicate default" - + " job type: jobType:Hive, for application type:mapreduce, isDefault:true, confName:hive.mapred.mode," - + " confValue:.*. for application type: MAPREDUCE"); - JobTypeConfiguration jobTypeConf = new JobTypeConfiguration(document6.getDocumentElement()); - } -} diff --git a/test/com/linkedin/drelephant/configurations/scheduler/SchedulerConfigurationTest.java b/test/com/linkedin/drelephant/configurations/scheduler/SchedulerConfigurationTest.java deleted file mode 100644 index 2c86ec561..000000000 --- a/test/com/linkedin/drelephant/configurations/scheduler/SchedulerConfigurationTest.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.configurations.scheduler; - -import com.linkedin.drelephant.configurations.scheduler.SchedulerConfiguration; -import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData; - -import java.io.IOException; -import java.util.List; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - - -public class SchedulerConfigurationTest { - - private static Document document1; - private static Document document2; - private static Document document3; - - @BeforeClass - public static void runBeforeClass() { - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - document1 = builder.parse(SchedulerConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/scheduler/SchedulerConfTest1.xml")); - document2 = builder.parse(SchedulerConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/scheduler/SchedulerConfTest2.xml")); - document3 = builder.parse(SchedulerConfigurationTest.class.getClassLoader() - .getResourceAsStream("configurations/scheduler/SchedulerConfTest3.xml")); - } catch (ParserConfigurationException e) { - throw new RuntimeException("XML Parser could not be created.", e); - } catch (SAXException e) { - throw new RuntimeException("Test files are not properly formed", e); - } catch (IOException e) { - throw new RuntimeException("Unable to read test files ", e); - } - } - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - /** - * Correctly configured scheduler - */ - @Test - public void testParseSchedulerConf1() { - SchedulerConfiguration schedulerConf = new SchedulerConfiguration(document1.getDocumentElement()); - List schedulerConfData = schedulerConf.getSchedulerConfigurationData(); - assertEquals(schedulerConfData.size(), 2); - for (SchedulerConfigurationData data : schedulerConfData) { - if (data.getSchedulerName().equals("airflow")) { - assertEquals("com.linkedin.drelephant.schedulers.AirflowScheduler", data.getClassName()); - assertEquals("http://localhost:8000", data.getParamMap().get("airflowbaseurl")); - } else { - assertEquals("azkaban", data.getSchedulerName()); - assertEquals("com.linkedin.drelephant.schedulers.AzkabanScheduler", data.getClassName()); - } - } - } - - /** - * No classname tag - */ - @Test - public void testParseSchedulerConf2() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'classname' in scheduler 1"); - new SchedulerConfiguration(document2.getDocumentElement()); - } - - /** - * No name tag - */ - @Test - public void testParseSchedulerConf3() { - expectedEx.expect(RuntimeException.class); - expectedEx.expectMessage("No tag 'name' in scheduler 2 classname com.linkedin.drelephant.schedulers.AzkabanScheduler"); - new SchedulerConfiguration(document3.getDocumentElement()); - } - -} diff --git a/test/com/linkedin/drelephant/exceptions/EventExceptionTest.java b/test/com/linkedin/drelephant/exceptions/EventExceptionTest.java deleted file mode 100644 index e0086e1be..000000000 --- a/test/com/linkedin/drelephant/exceptions/EventExceptionTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.exceptions; - -import org.apache.log4j.Logger; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; - - -public class EventExceptionTest { - private static final Logger logger = Logger.getLogger(EventExceptionTest.class); - - @Test - public void getMessageTest() { - - String rawEventException = - "java.io.FileNotFoundException: Path is not a file: /data/sample/Sample/Sample/1466675602538-PT-472724050\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:70)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:56)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1914)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1855)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1835)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1807)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:552)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:362)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:619)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.ipc.RPC$Server.call(RPC.java:962)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2044)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2040)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat java.security.AccessController.doPrivileged(Native Method)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat javax.security.auth.Subject.doAs(Subject.java:422)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038)"; - - EventException eventException = new EventException(0, rawEventException); - assertTrue("getMessageTest failed", eventException.getMessage() - .equals("Path is not a file: /data/sample/Sample/Sample/1466675602538-PT-472724050")); - logger.info("correct message" + eventException.getMessage()); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/exceptions/MRJobLogAnalyzerTest.java b/test/com/linkedin/drelephant/exceptions/MRJobLogAnalyzerTest.java deleted file mode 100644 index 8942c0b6a..000000000 --- a/test/com/linkedin/drelephant/exceptions/MRJobLogAnalyzerTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.exceptions; - -import org.junit.Test; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertEquals; - -public class MRJobLogAnalyzerTest { - - private String failedMRJobDiagnostic = "Task failed task_1466048666726_979739_r_000000\n" - + "Job failed as tasks failed. failedMaps:0 failedReduces:1"; - - private String killedMRJobDiagnostic = "Kill job job_1466048666726_978316 received from zfu@LINKEDIN.BIZ (auth:TOKEN) at 10.150.4.50\n" - + "Job received Kill while in RUNNING state."; - - private MRJobLogAnalyzer analyzedFailedJobDiagnostic; - private MRJobLogAnalyzer analyzedKilledJobDiagnostic; - - public MRJobLogAnalyzerTest(){ - analyzedFailedJobDiagnostic = new MRJobLogAnalyzer(failedMRJobDiagnostic); - analyzedKilledJobDiagnostic = new MRJobLogAnalyzer(killedMRJobDiagnostic); - } - - @Test - public void getFailedSubEventsTest(){ - assertEquals(analyzedFailedJobDiagnostic.getFailedSubEvents().size(),1 ); - assertTrue(analyzedKilledJobDiagnostic.getFailedSubEvents().isEmpty()); - } - - @Test - public void getExceptionTest(){ - - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/exceptions/azkaban/AzkabanJobLogAnalyzerTest.java b/test/com/linkedin/drelephant/exceptions/azkaban/AzkabanJobLogAnalyzerTest.java deleted file mode 100644 index b26478a95..000000000 --- a/test/com/linkedin/drelephant/exceptions/azkaban/AzkabanJobLogAnalyzerTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.exceptions.azkaban; - -import com.linkedin.drelephant.exceptions.JobState; -import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; - - -public class AzkabanJobLogAnalyzerTest { - - private String succeededAzkabanJobLog = "24-06-2016 03:12:53 PDT extractCrawlCompanyIndustryClassificationFlow_extractionFinalizer INFO - Starting job sample_sample at 1466763173873\n" - + "24-06-2016 03:12:53 PDT extractCrawlCompanyIndustryClassificationFlow_extractionFinalizer INFO - job JVM args: -Dazkaban.flowid=sampleFlow -Dazkaban.execid=557260 -Dazkaban.jobid=sample_jobr\n" - + "24-06-2016 03:12:55 PDT extractCrawlCompanyIndustryClassificationFlow_extractionFinalizer INFO - Finishing job sample_job attempt: 0 at 1466763175040 with status SUCCEEDED"; - - private String killedAzkabanJobLog = "28-06-2016 16:58:20 PDT feature-exploration_create-index-map INFO - Starting job sample at 1467158300703\n" - + "28-06-2016 17:58:05 PDT feature-exploration_create-index-map ERROR - Kill has been called.\n" - + "28-06-2016 17:58:05 PDT feature-exploration_create-index-map INFO - 16/06/29 00:58:05 INFO util.Utils: Shutdown hook called\n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map ERROR - caught error running the job\n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map INFO - Token service: sample-localhostrm01.grid.linkedin.com:8032\n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map INFO - Cancelling mr job tracker token \n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map ERROR - Job run failed!\n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map ERROR - java.lang.RuntimeException: azkaban.jobExecutor.utils.process.ProcessFailureException cause: java.lang.RuntimeException: azkaban.jobExecutor.utils.process.ProcessFailureException\n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map INFO - Finishing job feature-exploration_create-index-map attempt: 0 at 1467161886022 with status KILLED\n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map INFO - applicationIds to kill: [application_1466048666726_642278]\n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map INFO - start klling application: application_1466048666726_642278\n" - + "28-06-2016 17:58:06 PDT feature-exploration_create-index-map INFO - successfully killed application: application_1466048666726_642278"; - - private String mrLevelFailedAzkabanJobLog = "24-06-2016 03:12:19 PDT help_center_sessions INFO - Starting job help_center_sessions at 1466763139993\n" - + "24-06-2016 03:12:25 PDT help_center_sessions INFO - INFO Kind: HDFS_DELEGATION_TOKEN, Service: sample-localhostnn01.grid.linkedin.com:9000, Ident: (HDFS_DELEGATION_TOKEN token 5017233 for username)\n" - + "24-06-2016 03:12:26 PDT help_center_sessions INFO - INFO Submitted application application_1466048666726_410150\n" - + "24-06-2016 03:12:26 PDT help_center_sessions INFO - INFO Running job: job_1466048666726_410150\n" - + "24-06-2016 03:12:33 PDT help_center_sessions INFO - INFO Job job_1466048666726_410150 running in uber mode : false\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - Error: java.io.FileNotFoundException: Path is not a file: /data/databases/sample/Sample/1466675602538-PT-472724050\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:70)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1671)\n" - + "24-06-2016 03:12:40 PDT help_center_sessions INFO - \tat org.apache.hadoop.ipc.Server$Handler.run(Server.java:2038\n" - + "24-06-2016 03:13:00 PDT help_center_sessions ERROR - Job run failed!\n" - + "24-06-2016 03:13:00 PDT help_center_sessions ERROR - java.lang.RuntimeException: azkaban.jobExecutor.utils.process.ProcessFailureException cause: java.lang.RuntimeException: azkaban.jobExecutor.utils.process.ProcessFailureException\n" - + "24-06-2016 03:13:00 PDT help_center_sessions INFO - Finishing job help_center_sessions attempt: 0 at 1466763180242 with status FAILED"; - - private String scriptLevelFailedAzkabanJobLog="28-06-2016 16:23:10 PDT job_search_trigger INFO - Starting job job_search_trigger at 1467156190329\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - INFO Last attempt: false\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - Exception in thread \"main\" java.lang.reflect.UndeclaredThrowableException\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - \tat org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1686)\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - \t... 3 more\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - Caused by: java.lang.RuntimeException: Backfill requires start and end date\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - \tat com.linkedin.metrics.feeder.TriggerJob.generateDaily(TriggerJob.java:143)\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - \tat com.linkedin.metrics.feeder.TriggerJob.run(TriggerJob.java:135)\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - \t... 14 more\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - Process completed unsuccessfully in 1 seconds.\n" - + "28-06-2016 16:23:12 PDT job_search_trigger ERROR - Job run failed!\n" - + "28-06-2016 16:23:12 PDT job_search_trigger ERROR - java.lang.RuntimeException: azkaban.jobExecutor.utils.process.ProcessFailureException cause: java.lang.RuntimeException: azkaban.jobExecutor.utils.process.ProcessFailureException\n" - + "28-06-2016 16:23:12 PDT job_search_trigger INFO - Finishing job job_search_trigger attempt: 0 at 1467156192215 with status FAILED"; - - private String azkabanLevelFailedAzkabanJobLog = "28-06-2016 13:45:27 PDT feature-exploration_create-index-map INFO - Starting job feature-exploration_create-index-map at 1467146727699\n" - + "28-06-2016 13:45:27 PDT feature-exploration_create-index-map INFO - job JVM args: -Dazkaban.flowid=feature-exploration -Dazkaban.execid=593197 -Dazkaban.jobid=feature-exploration_create-index-map\n" - + "28-06-2016 13:45:27 PDT feature-exploration_create-index-map INFO - Building spark job executor. \n" - + "28-06-2016 13:45:27 PDT feature-exploration_create-index-map ERROR - Failed to build job executor for job feature-exploration_create-index-mapCould not find variable substitution for variable(s) [global.jvm.args->user.to.proxy]\n" - + "28-06-2016 13:45:27 PDT feature-exploration_create-index-map ERROR - Failed to build job type\n" - + "azkaban.jobtype.JobTypeManagerException: Failed to build job executor for job feature-exploration_create-index-map\n" - + "28-06-2016 13:45:27 PDT feature-exploration_create-index-map ERROR - Job run failed preparing the job.\n" - + "28-06-2016 13:45:27 PDT feature-exploration_create-index-map INFO - Finishing job feature-exploration_create-index-map attempt: 0 at 1467146727702 with status FAILED"; - - private AzkabanJobLogAnalyzer analyzedSucceededLog; - private AzkabanJobLogAnalyzer analyzedKilledLog; - private AzkabanJobLogAnalyzer analyzedMRLevelFailedLog; - private AzkabanJobLogAnalyzer analyzedScriptLevelFailedLog; - private AzkabanJobLogAnalyzer analyzedAzkabanLevelFailedLog; - - public AzkabanJobLogAnalyzerTest(){ - analyzedSucceededLog = new AzkabanJobLogAnalyzer(succeededAzkabanJobLog); - analyzedKilledLog = new AzkabanJobLogAnalyzer(killedAzkabanJobLog); - analyzedMRLevelFailedLog = new AzkabanJobLogAnalyzer(mrLevelFailedAzkabanJobLog); - analyzedScriptLevelFailedLog = new AzkabanJobLogAnalyzer(scriptLevelFailedAzkabanJobLog); - analyzedAzkabanLevelFailedLog = new AzkabanJobLogAnalyzer(azkabanLevelFailedAzkabanJobLog); - } - @Test - public void getStateTest(){ - assertTrue(analyzedSucceededLog.getState() == JobState.SUCCEEDED); - assertTrue(analyzedKilledLog.getState() == JobState.KILLED); - assertTrue(analyzedMRLevelFailedLog.getState() == JobState.MRFAIL); - assertTrue(analyzedScriptLevelFailedLog.getState() == JobState.SCRIPTFAIL); - assertTrue(analyzedAzkabanLevelFailedLog.getState() == JobState.SCHEDULERFAIL); - } - - @Test - public void getSubEventsTest(){ - assertTrue("Succeeded sub events test failed",analyzedSucceededLog.getSubEvents().isEmpty()); - assertTrue("Script level failed sub events test failed",analyzedScriptLevelFailedLog.getSubEvents().isEmpty()); - assertTrue("Azkaban level failed sub events test failed",analyzedAzkabanLevelFailedLog.getSubEvents().isEmpty()); - assertTrue(analyzedMRLevelFailedLog.getSubEvents().size() == 1); - assertTrue(analyzedMRLevelFailedLog.getSubEvents().iterator().next().equals("job_1466048666726_410150")); - assertTrue("Killed sub events test failed",analyzedKilledLog.getSubEvents().isEmpty()); - } - - @Test - public void getExceptionsTest(){ - assertTrue(analyzedSucceededLog.getException() == null); - assertTrue(analyzedKilledLog.getException() == null); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/mapreduce/TestTaskLevelAggregatedMetrics.java b/test/com/linkedin/drelephant/mapreduce/TestTaskLevelAggregatedMetrics.java deleted file mode 100644 index 93208e8b5..000000000 --- a/test/com/linkedin/drelephant/mapreduce/TestTaskLevelAggregatedMetrics.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce; - -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import org.junit.Assert; -import org.junit.Test; - -public class TestTaskLevelAggregatedMetrics { - - @Test - public void testZeroTasks() { - MapReduceTaskData taskData[] = {}; - TaskLevelAggregatedMetrics taskMetrics = new TaskLevelAggregatedMetrics(taskData, 0, 0); - Assert.assertEquals(taskMetrics.getDelay(), 0); - Assert.assertEquals(taskMetrics.getResourceUsed(), 0); - Assert.assertEquals(taskMetrics.getResourceWasted(), 0); - } - - @Test - public void testNullTaskArray() { - TaskLevelAggregatedMetrics taskMetrics = new TaskLevelAggregatedMetrics(null, 0, 0); - Assert.assertEquals(taskMetrics.getDelay(), 0); - Assert.assertEquals(taskMetrics.getResourceUsed(), 0); - Assert.assertEquals(taskMetrics.getResourceWasted(), 0); - } - - @Test - public void testTaskLevelData() { - MapReduceTaskData taskData[] = new MapReduceTaskData[3]; - MapReduceCounterData counterData = new MapReduceCounterData(); - counterData.set(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES, 655577088L); - counterData.set(MapReduceCounterData.CounterName.VIRTUAL_MEMORY_BYTES, 3051589632L); - long time[] = {0,0,0,1464218501117L, 1464218534148L}; - taskData[0] = new MapReduceTaskData("task", "id"); - taskData[0].setTimeAndCounter(time, counterData); - taskData[1] = new MapReduceTaskData("task", "id"); - taskData[1].setTimeAndCounter(new long[5], counterData); - // Non-sampled task, which does not contain time and counter data - taskData[2] = new MapReduceTaskData("task", "id"); - TaskLevelAggregatedMetrics taskMetrics = new TaskLevelAggregatedMetrics(taskData, 4096L, 1463218501117L); - Assert.assertEquals(taskMetrics.getDelay(), 1000000000L); - Assert.assertEquals(taskMetrics.getResourceUsed(), 135168L); - Assert.assertEquals(taskMetrics.getResourceWasted(), 66627L); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2Test.java b/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2Test.java deleted file mode 100644 index 16f3ffb8f..000000000 --- a/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2Test.java +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.fetchers; - -import com.google.common.collect.ImmutableMap; -import com.linkedin.drelephant.analysis.AnalyticJob; -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.configurations.fetcher.FetcherConfiguration; -import com.linkedin.drelephant.configurations.fetcher.FetcherConfigurationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import java.io.FileNotFoundException; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.mapreduce.Counters; -import org.apache.hadoop.mapreduce.TaskAttemptID; -import org.apache.hadoop.mapreduce.TaskID; -import org.apache.hadoop.mapreduce.TaskType; -import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.List; -import java.util.Map; -import java.util.HashMap; -import java.util.TimeZone; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - - -public class MapReduceFSFetcherHadoop2Test { - - private static Document document9 = null; - private static Document document10 = null; - private static Document document11 = null; - - @BeforeClass - public static void before() { - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - ClassLoader classLoader = MapReduceFSFetcherHadoop2Test.class.getClassLoader(); - document9 = builder.parse(classLoader.getResourceAsStream( - "configurations/fetcher/FetcherConfTest9.xml")); - document10 = builder.parse(classLoader.getResourceAsStream( - "configurations/fetcher/FetcherConfTest10.xml")); - document11 = builder.parse(classLoader.getResourceAsStream( - "configurations/fetcher/FetcherConfTest11.xml")); - } catch (ParserConfigurationException e) { - throw new RuntimeException("XML Parser could not be created.", e); - } catch (SAXException e) { - throw new RuntimeException("Test files are not properly formed", e); - } catch (IOException e) { - throw new RuntimeException("Unable to read test files ", e); - } - } - - @Test - public void testFetcherDefaultConfig() { - FetcherConfiguration fetcherConf = new FetcherConfiguration(document9.getDocumentElement()); - try { - MapReduceFSFetcherHadoop2 fetcher = new MapReduceFSFetcherHadoop2( - fetcherConf.getFetchersConfigurationData().get(0)); - Assert.assertFalse("Sampling should be disabled in default", fetcher.isSamplingEnabled()); - Assert.assertEquals(fetcher.DEFALUT_MAX_LOG_SIZE_IN_MB, fetcher.getMaxLogSizeInMB(), 0.0001); - Assert.assertEquals(TimeZone.getDefault(), fetcher.getTimeZone()); - - List list = new ArrayList(); - int listLen = fetcher.MAX_SAMPLE_SIZE * 2; - for (int i = 0; i < listLen; i++) { - list.add(0); - } - Assert.assertEquals("Should not sample task list when sampling is disabled", listLen, - fetcher.sampleAndGetSize("appId", list)); - } catch (IOException e) { - Assert.assertNull("Failed to initialize FileSystem", e); - } - } - - @Test - public void testFetcherConfig() { - FetcherConfiguration fetcherConf = new FetcherConfiguration(document10.getDocumentElement()); - try { - MapReduceFSFetcherHadoop2 fetcher = new MapReduceFSFetcherHadoop2( - fetcherConf.getFetchersConfigurationData().get(0)); - Assert.assertTrue("Failed to enable sampling", fetcher.isSamplingEnabled()); - Assert.assertEquals(200d, fetcher.getMaxLogSizeInMB(), 0.0001); - Assert.assertEquals(TimeZone.getTimeZone("PST"), fetcher.getTimeZone()); - - List list = new ArrayList(); - int listLen = fetcher.MAX_SAMPLE_SIZE * 2; - for (int i = 0; i < listLen; i++) { - list.add(0); - } - Assert.assertEquals("Should sample task list when sampling is enabled", fetcher.MAX_SAMPLE_SIZE, - fetcher.sampleAndGetSize("appId", list)); - } catch (IOException e) { - Assert.assertNull("Failed to initialize FileSystem", e); - } - } - - @Test - public void testFetcherEmptyConf() { - FetcherConfiguration fetcherConf = new FetcherConfiguration(document11.getDocumentElement()); - try { - MapReduceFSFetcherHadoop2 fetcher = new MapReduceFSFetcherHadoop2( - fetcherConf.getFetchersConfigurationData().get(0)); - Assert.assertFalse("Sampling should be disabled in default", fetcher.isSamplingEnabled()); - Assert.assertEquals(fetcher.DEFALUT_MAX_LOG_SIZE_IN_MB, fetcher.getMaxLogSizeInMB(), 0.0001); - Assert.assertEquals(TimeZone.getDefault(), fetcher.getTimeZone()); - - List list = new ArrayList(); - int listLen = fetcher.MAX_SAMPLE_SIZE * 2; - for (int i = 0; i < listLen; i++) { - list.add(0); - } - Assert.assertEquals("Should not sample task list when sampling is disabled", listLen, - fetcher.sampleAndGetSize("appId", list)); - } catch (IOException e) { - Assert.assertNull("Failed to initialize FileSystem", e); - } - } - - @Test - public void testGetHistoryDir() { - FetcherConfiguration fetcherConf = new FetcherConfiguration(document9.getDocumentElement()); - try { - MapReduceFSFetcherHadoop2 fetcher = new MapReduceFSFetcherHadoop2( - fetcherConf.getFetchersConfigurationData().get(0)); - Calendar timestamp = Calendar.getInstance(); - timestamp.set(2016, Calendar.JULY, 30); - AnalyticJob job = new AnalyticJob() - .setAppId("application_1461566847127_84624") - .setFinishTime(timestamp.getTimeInMillis()); - - String expected = StringUtils.join(new String[]{fetcher.getHistoryLocation(), "2016", "07", "30", "000084", ""}, - File.separator); - Assert.assertEquals("Error history directory", expected, fetcher.getHistoryDir(job)); - } catch (IOException e) { - Assert.assertNull("Failed to initialize FileSystem", e); - } - } - - @Test - public void testGetBackfillList() { - FetcherConfigurationData confData = new FetcherConfigurationData(MapReduceFSFetcherHadoop2.class.getName(), - new ApplicationType("MAPREDUCE"), - ImmutableMap.of(MapReduceFSFetcherHadoop2.HISTORY_SERVER_TIME_ZONE_XML_FIELD, "GMT")); - try { - MapReduceFSFetcherHadoop2 fetcher = new MapReduceFSFetcherHadoop2(confData); - List jobs = fetcher.fetchJobsForBackfill(1526520000000L, 1526600000000L); - assertEquals(2, jobs.size()); - for (AnalyticJob job : jobs) { - if (!job.getAppId().equals("application_1526555215992_0001") && - !job.getAppId().equals("application_1526555215992_0002")) { - fail("Unexpected applications returned on backfill"); - } - } - assertEquals("MAPREDUCE", jobs.get(0).getAppType().getName()); - assertEquals("default", jobs.get(0).getQueueName()); - assertEquals("user", jobs.get(0).getUser()); - - jobs = fetcher.fetchJobsForBackfill(1526555658200L, 1526555658400L); - assertEquals(1, jobs.size()); - assertEquals("application_1526555215992_0001", jobs.get(0).getAppId()); - } catch (Exception e) { - fail("Backfill failed."); - } - } - - @Test - public void testFetchData() { - FetcherConfigurationData confData = new FetcherConfigurationData(MapReduceFSFetcherHadoop2.class.getName(), - new ApplicationType("MAPREDUCE"), - ImmutableMap.of(MapReduceFSFetcherHadoop2.HISTORY_SERVER_TIME_ZONE_XML_FIELD, "GMT")); - try { - MapReduceFSFetcherHadoop2 fetcher = new MapReduceFSFetcherHadoop2(confData); - // Fetch data for a job which has history file in done directory. - MapReduceApplicationData data = fetcher.fetchData(new AnalyticJob().setAppId("application_1526555215992_0001"). - setUser("user").setQueueName("default").setStartTime(1526555642677L).setFinishTime(1526555658299L)); - assertNotNull(data); - assertEquals("application_1526555215992_0001", data.getAppId()); - assertEquals("job_1526555215992_0001", data.getJobId()); - assertEquals("MAPREDUCE", data.getApplicationType().getName()); - assertEquals("org.apache.hadoop.util.QuickSort", data.getConf().getProperty("map.sort.class")); - assertEquals(28, data.getCounters().get(MapReduceCounterData.CounterName.FILE_BYTES_READ)); - assertEquals(1, data.getMapperData().length); - assertEquals(1, data.getReducerData().length); - assertTrue("Job should have succeeded", data.getSucceeded()); - - // Fetch data for a job which has history file in intermediate directory. - data = fetcher.fetchData(new AnalyticJob().setAppId("application_1526555215992_0004").setUser("user"). - setQueueName("default").setStartTime(1526567231815L).setFinishTime(1526567243482L)); - assertNotNull(data); - assertEquals("application_1526555215992_0004", data.getAppId()); - assertEquals("job_1526555215992_0004", data.getJobId()); - assertEquals("MAPREDUCE", data.getApplicationType().getName()); - assertEquals("RECORD", data.getConf().getProperty("mapreduce.output.fileoutputformat.compress.type")); - assertEquals(346030080, data.getCounters().get(MapReduceCounterData.CounterName.COMMITTED_HEAP_BYTES)); - assertEquals(1, data.getMapperData().length); - assertEquals(1, data.getReducerData().length); - assertTrue("Job should have succeeded", data.getSucceeded()); - - // Fetching data for a job which does not exist in done and intermediate directories but the date part in done - // directory exists. - try { - data = fetcher.fetchData(new AnalyticJob().setAppId("application_1526555315992_0007").setUser("user"). - setQueueName("default").setStartTime(1526567231900L).setFinishTime(1526567243567L)); - fail("File not found Exception should have been thrown for application_1526555315992_0007."); - } catch (FileNotFoundException e) { - assertTrue(e.getMessage().contains("2018/05/17")); - } catch (Exception e) { - fail("Unexpected exception thrown for application_1526555315992_0007."); - } - - // Fetching data for a job which does not exist in done and intermediate directories. - try { - data = fetcher.fetchData(new AnalyticJob().setAppId("application_1526555315992_0008").setUser("user"). - setQueueName("default").setStartTime(1525567230004L).setFinishTime(1525567243567L)); - fail("File not found Exception should have been thrown for application_1526555315992_0008."); - } catch (FileNotFoundException e) { - assertTrue(e.getMessage().contains("2018/05/06")); - } catch (Exception e) { - fail("Unexpected exception thrown for application_1526555315992_0008."); - } - } catch (Exception e) { - fail("Fetch data failed."); - } - } - - @Test - public void testGetTaskData() { - FetcherConfiguration fetcherConf = new FetcherConfiguration(document9.getDocumentElement()); - - try { - MapReduceFSFetcherHadoop2 fetcher = new MapReduceFSFetcherHadoop2( - fetcherConf.getFetchersConfigurationData().get(0)); - String jobId = "job_14000_001"; - List infoList = new ArrayList(); - infoList.add(new MockTaskInfo(1, true)); - infoList.add(new MockTaskInfo(2, false)); - - MapReduceTaskData[] taskList = fetcher.getTaskData(jobId, infoList); - Assert.assertNotNull("taskList should not be null.", taskList); - int succeededTaskCount = 0; - for (MapReduceTaskData task : taskList) { - Assert.assertNotNull("Null pointer in taskList.", task); - if(task.getState().equals("SUCCEEDED")) { - succeededTaskCount++; - } - } - Assert.assertEquals("Should have total two tasks.", 2, taskList.length); - Assert.assertEquals("Should have only one succeeded task.", 1, succeededTaskCount); - } catch (IOException e) { - Assert.assertNull("Failed to initialize FileSystem.", e); - } - } - - class MockTaskInfo extends JobHistoryParser.TaskInfo { - TaskID taskId; - TaskType taskType; - boolean succeeded; - Counters counters; - long startTime, finishTime; - TaskAttemptID failedDueToAttemptId; - TaskAttemptID successfulAttemptId; - Map attemptsMap; - - public MockTaskInfo(int id, boolean succeeded) { - this.taskId = new TaskID("job1", 1, TaskType.MAP, id); - this.taskType = TaskType.MAP; - this.succeeded = succeeded; - this.counters = new Counters(); - this.finishTime = System.currentTimeMillis(); - this.startTime = finishTime - 10000; - this.failedDueToAttemptId = new TaskAttemptID(taskId, 0); - this.successfulAttemptId = new TaskAttemptID(taskId, 1); - this.attemptsMap = new HashMap(); - this.attemptsMap.put(failedDueToAttemptId, new JobHistoryParser.TaskAttemptInfo()); - this.attemptsMap.put(successfulAttemptId, new JobHistoryParser.TaskAttemptInfo()); - } - - public TaskID getTaskId() { return taskId; } - public long getStartTime() { return startTime; } - public long getFinishTime() { return finishTime; } - public Counters getCounters() { return counters; } - public TaskType getTaskType() { return taskType; } - public String getTaskStatus() { return succeeded ? "SUCCEEDED" : "FAILED"; } - public TaskAttemptID getSuccessfulAttemptId() { return successfulAttemptId; } - public TaskAttemptID getFailedDueToAttemptId() { return failedDueToAttemptId; } - public Map getAllTaskAttempts() { - return attemptsMap; - } - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java b/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java deleted file mode 100644 index 8ce84bec1..000000000 --- a/test/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFetcherHadoop2Test.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.fetchers; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.linkedin.drelephant.util.ThreadContextMR2; -import org.junit.Assert; -import org.junit.Test; - - -public class MapReduceFetcherHadoop2Test { - - @Test - public void testDiagnosticMatcher() { - Matcher matcher = ThreadContextMR2.getDiagnosticMatcher("Task task_1443068695259_9143_m_000475 failed 1 time"); - Assert.assertEquals(".*[\\s\\u00A0]+(task_[0-9]+_[0-9]+_[m|r]_[0-9]+)[\\s\\u00A0]+.*", matcher.pattern().toString()); - Assert.assertEquals(true, matcher.matches()); - Assert.assertEquals(1, matcher.groupCount()); - Assert.assertEquals("task_1443068695259_9143_m_000475", matcher.group(1)); - } - -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/DistributedCacheLimitHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/DistributedCacheLimitHeuristicTest.java deleted file mode 100644 index b655a68d3..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/DistributedCacheLimitHeuristicTest.java +++ /dev/null @@ -1,137 +0,0 @@ -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - - -/** - * Tests for the DistributedCacheLimitHeuristic class. - */ -public class DistributedCacheLimitHeuristicTest { - private static Map paramMap = new HashMap(); - private static Properties jobConf = new Properties(); - private static final String cacheFileList = - "/path/to/firstCacheFile,/path/to/secondCacheFile,/path/to/thirdCacheFile"; - private static final String archiveCacheFileList = - "/path/to/firstArchiveCacheFile,/path/to/secondArchiveCacheFile,/path/to/thirdArchiveCacheFile"; - - private static Heuristic _heuristic = new DistributedCacheLimitHeuristic( - new HeuristicConfigurationData("test.heuristic", "test.class", "test.view", new ApplicationType("mapreduce"), - paramMap)); - - @Before - public void setup() { - paramMap.put("distributed.cache.file.size.limit", "500000000"); - jobConf.setProperty("mapreduce.job.cache.files", cacheFileList); - jobConf.setProperty("mapreduce.job.cache.archives", archiveCacheFileList); - } - - /** - * All cache file sizes are within the limit. - */ - @Test - public void testHeuristicResult() { - jobConf.setProperty("mapreduce.job.cache.files.filesizes", "100,200,300"); - jobConf.setProperty("mapreduce.job.cache.archives.filesizes", "400,500,600"); - - MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.NONE); - } - - /** - * File size not found for all the files in cache. - */ - @Test - public void testHeuristicResultCacheFilesAndSizeLengthMismatch() { - jobConf.setProperty("mapreduce.job.cache.files.filesizes", "100,200"); - MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.MODERATE); - } - - /** - * File size not found for all the files in archive cache. - */ - @Test - public void testHeuristicResultArchiveCacheFilesAndSizeLengthMismatch() { - jobConf.setProperty("mapreduce.job.cache.files.filesizes", "100,200,300"); - jobConf.setProperty("mapreduce.job.cache.archives.filesizes", "400,500"); - MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.MODERATE); - } - - /** - * File size limit exceeded for file in cache. - */ - @Test - public void testHeuristicResultCacheFileLimitViolated() { - jobConf.setProperty("mapreduce.job.cache.files.filesizes", "100,200,600000000"); - jobConf.setProperty("mapreduce.job.cache.archives.filesizes", "400,500,600"); - - MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.CRITICAL); - } - - /** - * File size limit exceeded for file in archive cache. - */ - @Test - public void testHeuristicResultArchiveCacheFileLimitViolated() { - jobConf.setProperty("mapreduce.job.cache.files.filesizes", "100,200,300"); - jobConf.setProperty("mapreduce.job.cache.archives.filesizes", "400,500,600000000"); - - MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.CRITICAL); - } - - /** - * Either of the caches are not used by the application. - */ - @Test - public void testHeuristicResultNoDistributedCacheFiles() { - jobConf.remove("mapreduce.job.cache.files"); - jobConf.remove("mapreduce.job.cache.archives"); - MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - assertTrue("Failed to match on expected severity", result == null); - } - - /** - * Cache files are not used by the application. - */ - @Test - public void testHeuristicResultWithEmptyCacheFiles() { - jobConf.remove("mapreduce.job.cache.files"); - jobConf.setProperty("mapreduce.job.cache.archives.filesizes", "400,500,600"); - MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.NONE); - } - - /** - * Archive cache not used by the application. - */ - @Test - public void testHeuristicResultWithEmptyArchiveCacheFiles() { - jobConf.remove("mapreduce.job.cache.archives"); - jobConf.setProperty("mapreduce.job.cache.files.filesizes", "100,200,300"); - MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.NONE); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/JobQueueLimitHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/JobQueueLimitHeuristicTest.java deleted file mode 100644 index abd40f9cd..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/JobQueueLimitHeuristicTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import org.junit.Test; - -import junit.framework.TestCase; - -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; - - -public class JobQueueLimitHeuristicTest extends TestCase { - private static Map paramMap = new HashMap(); - private static Heuristic _heuristic = new JobQueueLimitHeuristic(new HeuristicConfigurationData("test_heuristic", "test_class", - "test_view", new ApplicationType("test_apptype"), paramMap)); - - private static final int NUM_TASKS = 100; - - @Test - public void testRuntimeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob((long) (14.5 * 60 * 1000), "default")); - } - - public void testRuntimeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(14 * 60 * 1000, "default")); - } - - public void testRuntimeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob((long) (13.5 * 60 * 1000), "default")); - } - - public void testRuntimeLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(13 * 60 * 1000, "default")); - } - - public void testRuntimeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(12 * 60 * 1000, "default")); - } - - public void testNonDefaultRuntimeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(15 * 60 * 1000, "non-default")); - } - - private Severity analyzeJob(long runtimeMs, String queueName) throws IOException { - MapReduceCounterData dummyCounter = new MapReduceCounterData(); - MapReduceTaskData[] mappers = new MapReduceTaskData[(2 * NUM_TASKS / 3) + 1]; - MapReduceTaskData[] reducers = new MapReduceTaskData[(NUM_TASKS / 3) + 1]; - Properties jobConf = new Properties(); - jobConf.put("mapred.job.queue.name", queueName); - int i = 0; - for (; i < 2 * NUM_TASKS / 3; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, dummyCounter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - for (i = 0; i < NUM_TASKS / 3; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, dummyCounter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - MapReduceApplicationData data = - new MapReduceApplicationData().setCounters(dummyCounter).setReducerData(reducers).setMapperData(mappers) - .setJobConf(jobConf); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperGCHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/MapperGCHeuristicTest.java deleted file mode 100644 index a03bb13b0..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperGCHeuristicTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; - - -public class MapperGCHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperGCHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static int NUMTASKS = 100; - - public void testGCCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1000000, 50000, 2000)); - } - - public void testGCSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1000000, 50000, 1500)); - } - - public void testGCModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1000000, 50000, 1000)); - } - - public void testGCNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1000000, 50000, 300)); - } - - public void testShortTasksNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(100000, 50000, 2000)); - } - - - private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] mappers = new MapReduceTaskData[NUMTASKS + 1]; - - MapReduceCounterData counter = new MapReduceCounterData(); - counter.set(MapReduceCounterData.CounterName.CPU_MILLISECONDS, cpuMs); - counter.set(MapReduceCounterData.CounterName.GC_MILLISECONDS, gcMs); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[]{runtimeMs, 0 , 0, 0, 0}, counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperMemoryHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/MapperMemoryHeuristicTest.java deleted file mode 100644 index 0de180ee6..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperMemoryHeuristicTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import org.apache.commons.io.FileUtils; - -import junit.framework.TestCase; - - -public class MapperMemoryHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperMemoryHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private int NUMTASKS = 100; - - public void testLargeContainerSizeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(2048, 8192)); - } - - public void testLargeContainerSizeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(3072, 8192)); - } - - public void testLargeContainerSizeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(4096, 8192)); - } - - public void testLargeContainerSizeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(6144, 8192)); - } - - // If the task use default container size, it should not be flagged - public void testDefaultContainerNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(256, 2048)); - } - - public void testDefaultContainerNoneMore() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1024, 2048)); - } - - private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] mappers = new MapReduceTaskData[NUMTASKS + 1]; - - MapReduceCounterData counter = new MapReduceCounterData(); - counter.set(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB* FileUtils.ONE_MB); - - Properties p = new Properties(); - p.setProperty(MapperMemoryHeuristic.MAPPER_MEMORY_CONF, Long.toString(containerMemMB)); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers); - data.setJobConf(p); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSkewHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSkewHeuristicTest.java deleted file mode 100644 index 3a26d31c6..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSkewHeuristicTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HDFSContext; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; - -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; - - -public class MapperSkewHeuristicTest extends TestCase { - - private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64; //1MB - private static final long UNITSIZETIME = 1000000; //1000sec - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperSkewHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(200, 200, 1 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(200, 200, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(200, 200, 20 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(200, 200, 30 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 50 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSmallFiles() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 1 * UNITSIZE, 5 * UNITSIZE)); - } - - public void testSmallTasks() throws IOException { - assertEquals(Severity.NONE, analyzeJob(5, 5, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testCriticalTime() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJobTime(200, 200, 1 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSevereTime() throws IOException { - assertEquals(Severity.SEVERE, analyzeJobTime(200, 200, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testModerateTime() throws IOException { - assertEquals(Severity.MODERATE, analyzeJobTime(200, 200, 20 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testLowTime() throws IOException { - assertEquals(Severity.LOW, analyzeJobTime(200, 200, 30 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testNoneTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(200, 200, 50 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSmallTasksTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(5, 5, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) - throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] mappers = new MapReduceTaskData[numSmallTasks + numLargeTasks + 1]; - - MapReduceCounterData smallCounter = new MapReduceCounterData(); - smallCounter.set(MapReduceCounterData.CounterName.HDFS_BYTES_READ, smallInputSize); - - MapReduceCounterData largeCounter = new MapReduceCounterData(); - largeCounter.set(MapReduceCounterData.CounterName.S3A_BYTES_READ, largeInputSize); - - int i = 0; - for (; i < numSmallTasks; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], smallCounter); - } - for (; i < numSmallTasks + numLargeTasks; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], largeCounter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } - - private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) - throws IOException { - MapReduceTaskData[] mappers = new MapReduceTaskData[numSmallTasks + numLongTasks + 1]; - - int i = 0; - for (; i < numSmallTasks; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTotalTimeMs(smallTimeTaken, true); - } - for (; i < numSmallTasks + numLongTasks; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTotalTimeMs(longTimeTaken, true); - } - // Non-sampled task, which does not contain time data - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - MapReduceApplicationData data = new MapReduceApplicationData().setMapperData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSpeedHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSpeedHeuristicTest.java deleted file mode 100644 index a3246e297..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSpeedHeuristicTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HDFSContext; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import com.linkedin.drelephant.math.Statistics; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; - -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; - - -public class MapperSpeedHeuristicTest extends TestCase { - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperSpeedHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static final long MB_IN_BYTES = FileUtils.ONE_MB; - private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS; - private static final int NUMTASKS = 100; - - public void testCritical() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.CRITICAL, analyzeJob(runtime, 1 * speed_factor)); - } - - public void testSevere() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.SEVERE, analyzeJob(runtime, 4 * speed_factor)); - } - - public void testModerate() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.MODERATE, analyzeJob(runtime, 13 * speed_factor)); - } - - public void testLow() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.LOW, analyzeJob(runtime, 50 * speed_factor)); - } - - public void testNone() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.NONE, analyzeJob(runtime, 51 * speed_factor)); - } - - public void testShortTask() throws IOException { - long runtime = 2 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.NONE, analyzeJob(runtime, 1 * speed_factor)); - } - - private Severity analyzeJob(long runtimeMs, long readBytes) throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] mappers = new MapReduceTaskData[NUMTASKS + 1]; - - MapReduceCounterData counter = new MapReduceCounterData(); - counter.set(MapReduceCounterData.CounterName.HDFS_BYTES_READ, readBytes / 4); - counter.set(MapReduceCounterData.CounterName.S3_BYTES_READ, readBytes / 4); - counter.set(MapReduceCounterData.CounterName.S3A_BYTES_READ, readBytes / 4); - counter.set(MapReduceCounterData.CounterName.S3N_BYTES_READ, readBytes / 4); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0 ,0, 0}, counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSpillHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSpillHeuristicTest.java deleted file mode 100644 index 1652c9610..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperSpillHeuristicTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; - - -public class MapperSpillHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperSpillHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - // Spill ratio 3.0, 1000 tasks - assertEquals(Severity.CRITICAL, analyzeJob(3000, 1000, 1000)); - } - - public void testSevere() throws IOException { - // Spill ratio 2.5, 1000 tasks - assertEquals(Severity.SEVERE, analyzeJob(2500, 1000, 1000)); - } - - public void testModerate() throws IOException { - // Spill ratio 2.3, 1000 tasks - assertEquals(Severity.MODERATE, analyzeJob(2300, 1000, 1000)); - } - - public void testLow() throws IOException { - // Spill ratio 2.1, 1000 tasks - assertEquals(Severity.LOW, analyzeJob(2100, 1000, 1000)); - } - - public void testNone() throws IOException { - // Spill ratio 1.0, 1000 tasks - assertEquals(Severity.NONE, analyzeJob(1000, 1000, 1000)); - } - - public void testSmallNumTasks() throws IOException { - // Spill ratio 3.0, should be critical, but number of task is small(10), final result is NONE - assertEquals(Severity.NONE, analyzeJob(3000, 1000, 10)); - } - - private Severity analyzeJob(long spilledRecords, long mapRecords, int numTasks) throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] mappers = new MapReduceTaskData[numTasks + 1]; - - MapReduceCounterData counter = new MapReduceCounterData(); - counter.set(MapReduceCounterData.CounterName.SPILLED_RECORDS, spilledRecords); - counter.set(MapReduceCounterData.CounterName.MAP_OUTPUT_RECORDS, mapRecords); - - int i = 0; - for (; i < numTasks; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperTimeHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/MapperTimeHeuristicTest.java deleted file mode 100644 index 8ba353f41..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/MapperTimeHeuristicTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; - - -public class MapperTimeHeuristicTest extends TestCase { - - private static final long DUMMY_INPUT_SIZE = 0; - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperTimeHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - // Test batch 1: Large runtime. Heuristic is not affected by various number of tasks */ - - public void testLongRuntimeTasksCritical() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(10, 120 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksCriticalMore() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(1000, 120 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksSevere() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.SEVERE, analyzeJob(10, 60 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksSevereMore() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.SEVERE, analyzeJob(1000, 60 * Statistics.MINUTE_IN_MS)); - } - - // Test batch 2: Short runtime and various number of tasks - - public void testShortRuntimeTasksCritical() throws IOException { - // Should increase split size and decrease number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(1000, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksSevere() throws IOException { - // Should increase split size and decrease number of tasks - assertEquals(Severity.SEVERE, analyzeJob(500, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(101, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(50, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksNone() throws IOException { - // Small file with small number of tasks and short runtime. This should be the common case. - assertEquals(Severity.NONE, analyzeJob(5, 1 * Statistics.MINUTE_IN_MS)); - } - - private Severity analyzeJob(int numTasks, long runtime) throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] mappers = new MapReduceTaskData[numTasks + 1]; - - MapReduceCounterData taskCounter = new MapReduceCounterData(); - taskCounter.set(MapReduceCounterData.CounterName.HDFS_BYTES_READ, DUMMY_INPUT_SIZE / 4); - taskCounter.set(MapReduceCounterData.CounterName.S3_BYTES_READ, DUMMY_INPUT_SIZE / 4); - taskCounter.set(MapReduceCounterData.CounterName.S3A_BYTES_READ, DUMMY_INPUT_SIZE / 4); - taskCounter.set(MapReduceCounterData.CounterName.S3N_BYTES_READ, DUMMY_INPUT_SIZE / 4); - - int i = 0; - for (; i < numTasks; i++) { - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[] { runtime, 0, 0, 0, 0 }, taskCounter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerGCHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerGCHeuristicTest.java deleted file mode 100644 index 83859225d..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerGCHeuristicTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; - - -public class ReducerGCHeuristicTest extends TestCase { - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerGCHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static int NUMTASKS = 100; - - public void testGCCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1000000, 50000, 2000)); - } - - public void testGCSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1000000, 50000, 1500)); - } - - public void testGCModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1000000, 50000, 1000)); - } - - public void testGCNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1000000, 50000, 300)); - } - - public void testShortTasksNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(100000, 50000, 2000)); - } - - - private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1]; - - MapReduceCounterData counter = new MapReduceCounterData(); - counter.set(MapReduceCounterData.CounterName.CPU_MILLISECONDS, cpuMs); - counter.set(MapReduceCounterData.CounterName.GC_MILLISECONDS, gcMs); - - int i = 0; - for (; i < NUMTASKS; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerMemoryHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerMemoryHeuristicTest.java deleted file mode 100644 index 9627c7e9a..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerMemoryHeuristicTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import org.apache.commons.io.FileUtils; - -import junit.framework.TestCase; - - -public class ReducerMemoryHeuristicTest extends TestCase { - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerMemoryHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private int NUMTASKS = 100; - - public void testLargeContainerSizeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(2048, 8192)); - } - - public void testLargeContainerSizeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(3072, 8192)); - } - - public void testLargeContainerSizeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(4096, 8192)); - } - - public void testLargeContainerSizeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(6144, 8192)); - } - - // If the task use default container size, it should not be flagged - public void testDefaultContainerNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(256, 2048)); - } - - public void testDefaultContainerNoneMore() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1024, 2048)); - } - - private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1]; - - MapReduceCounterData counter = new MapReduceCounterData(); - counter.set(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB* FileUtils.ONE_MB); - - Properties p = new Properties(); - p.setProperty(ReducerMemoryHeuristic.REDUCER_MEMORY_CONF, Long.toString(containerMemMB)); - - int i = 0; - for (; i < NUMTASKS; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], counter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers); - data.setJobConf(p); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerSkewHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerSkewHeuristicTest.java deleted file mode 100644 index 450e0becb..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerSkewHeuristicTest.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HDFSContext; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; - -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; - -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; - - -public class ReducerSkewHeuristicTest extends TestCase { - private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64; //1mb - private static final long UNITSIZETIME = 1000000; //1000sec - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerSkewHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(200, 200, 1 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(200, 200, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(200, 200, 20 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(200, 200, 30 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 50 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSmallFiles() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 1 * UNITSIZE, 5 * UNITSIZE)); - } - - public void testSmallTasks() throws IOException { - assertEquals(Severity.NONE, analyzeJob(5, 5, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testCriticalTime() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJobTime(200, 200, 1 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSevereTime() throws IOException { - assertEquals(Severity.SEVERE, analyzeJobTime(200, 200, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testModerateTime() throws IOException { - assertEquals(Severity.MODERATE, analyzeJobTime(200, 200, 20 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testLowTime() throws IOException { - assertEquals(Severity.LOW, analyzeJobTime(200, 200, 30 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testNoneTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(200, 200, 50 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSmallTasksTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(5, 5, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) - throws IOException { - MapReduceCounterData jobCounter = new MapReduceCounterData(); - MapReduceTaskData[] reducers = new MapReduceTaskData[numSmallTasks + numLargeTasks + 1]; - - MapReduceCounterData smallCounter = new MapReduceCounterData(); - smallCounter.set(MapReduceCounterData.CounterName.REDUCE_SHUFFLE_BYTES, smallInputSize); - - MapReduceCounterData largeCounter = new MapReduceCounterData(); - largeCounter.set(MapReduceCounterData.CounterName.REDUCE_SHUFFLE_BYTES, largeInputSize); - - int i = 0; - for (; i < numSmallTasks; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], smallCounter); - } - for (; i < numSmallTasks + numLargeTasks; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], largeCounter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } - - private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) - throws IOException { - MapReduceTaskData[] reducers = new MapReduceTaskData[numSmallTasks + numLongTasks + 1]; - - int i = 0; - for (; i < numSmallTasks; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTotalTimeMs(smallTimeTaken, true); - } - for (; i < numSmallTasks + numLongTasks; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTotalTimeMs(longTimeTaken, true); - } - // Non-sampled task, which does not contain time data - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - MapReduceApplicationData data = new MapReduceApplicationData().setReducerData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerTimeHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerTimeHeuristicTest.java deleted file mode 100644 index 5e953819d..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/ReducerTimeHeuristicTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; - -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import com.linkedin.drelephant.math.Statistics; - -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; - - -public class ReducerTimeHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerTimeHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS;; - - public void testShortRunetimeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1 * MINUTE_IN_MS, 1000)); - } - - public void testShortRunetimeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1 * MINUTE_IN_MS, 500)); - } - - public void testShortRunetimeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1 * MINUTE_IN_MS, 101)); - } - - public void testShortRunetimeLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(1 * MINUTE_IN_MS, 50)); - } - - public void testShortRunetimeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1 * MINUTE_IN_MS, 2)); - } - - public void testLongRunetimeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 10)); - } - - // Long runtime severity is not affected by number of tasks - public void testLongRunetimeCriticalMore() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 1000)); - } - - public void testLongRunetimeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(60 * MINUTE_IN_MS, 10)); - } - - public void testLongRunetimeSevereMore() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(60 * MINUTE_IN_MS, 1000)); - } - - private Severity analyzeJob(long runtimeMs, int numTasks) throws IOException { - MapReduceCounterData dummyCounter = new MapReduceCounterData(); - MapReduceTaskData[] reducers = new MapReduceTaskData[numTasks + 1]; - - int i = 0; - for (; i < numTasks; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, dummyCounter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(dummyCounter).setReducerData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} diff --git a/test/com/linkedin/drelephant/mapreduce/heuristics/ShuffleSortHeuristicTest.java b/test/com/linkedin/drelephant/mapreduce/heuristics/ShuffleSortHeuristicTest.java deleted file mode 100644 index 834148195..000000000 --- a/test/com/linkedin/drelephant/mapreduce/heuristics/ShuffleSortHeuristicTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.mapreduce.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import java.io.IOException; - -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.mapreduce.data.MapReduceCounterData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceTaskData; -import com.linkedin.drelephant.math.Statistics; - -import java.util.HashMap; -import java.util.Map; -import junit.framework.TestCase; - - -public class ShuffleSortHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ShuffleSortHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static final int NUMTASKS = 100; - private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS;; - - public void testLongShuffleCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(30 * MINUTE_IN_MS, 0, 5 * MINUTE_IN_MS)); - } - - public void testLongShuffleSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(30 * MINUTE_IN_MS, 0, 10 * MINUTE_IN_MS)); - } - - public void testLongShuffleModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(30 * MINUTE_IN_MS, 0, 20 * MINUTE_IN_MS)); - } - - public void testLongShuffleLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(30 * MINUTE_IN_MS, 0, 40 * MINUTE_IN_MS)); - } - - public void testLongShuffleNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(30 * MINUTE_IN_MS, 0, 80 * MINUTE_IN_MS)); - } - - public void testLongSortCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(0, 30 * MINUTE_IN_MS, 5 * MINUTE_IN_MS)); - } - - public void testLongSortSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(0, 30 * MINUTE_IN_MS, 10 * MINUTE_IN_MS)); - } - - public void testLongSortModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(0, 30 * MINUTE_IN_MS, 20 * MINUTE_IN_MS)); - } - - public void testLongSortLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(0, 30 * MINUTE_IN_MS, 40 * MINUTE_IN_MS)); - } - - public void testLongSortNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(0, 30 * MINUTE_IN_MS, 80 * MINUTE_IN_MS)); - } - - public void testShortShuffle() throws IOException { - assertEquals(Severity.NONE, analyzeJob(MINUTE_IN_MS / 2, 0, MINUTE_IN_MS / 2)); - } - - public void testShortSort() throws IOException { - assertEquals(Severity.NONE, analyzeJob(0, MINUTE_IN_MS / 2, MINUTE_IN_MS / 2)); - } - - private Severity analyzeJob(long shuffleTimeMs, long sortTimeMs, long reduceTimeMs) throws IOException { - MapReduceCounterData dummyCounter = new MapReduceCounterData(); - MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1]; - - int i = 0; - for (; i < NUMTASKS; i++) { - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter( - new long[] { shuffleTimeMs + sortTimeMs + reduceTimeMs, shuffleTimeMs, sortTimeMs, 0, 0}, dummyCounter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new MapReduceTaskData("task-id-"+i, "task-attempt-id-"+i); - MapReduceApplicationData data = new MapReduceApplicationData().setCounters(dummyCounter).setReducerData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } - -} diff --git a/test/com/linkedin/drelephant/math/StatisticsTest.java b/test/com/linkedin/drelephant/math/StatisticsTest.java deleted file mode 100644 index 9c28bf56b..000000000 --- a/test/com/linkedin/drelephant/math/StatisticsTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.math; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import static org.junit.Assert.assertEquals; - - -public class StatisticsTest { - - @Test - public void testAverage1() { - assertEquals(6, Statistics.average(new long[]{2, 4, 6, 8, 10})); - assertEquals(0, Statistics.average(new long[] {})); - } - - @Test - public void testAverage2() { - ArrayList list1 = new ArrayList(); - list1.add(2l); - list1.add(4l); - list1.add(6l); - list1.add(8l); - list1.add(10l); - assertEquals(6, Statistics.average(list1)); - - ArrayList list2 = new ArrayList(); - assertEquals(0, Statistics.average(list2)); - } - - @Rule - public ExpectedException expectedEx = ExpectedException.none(); - - @Test - public void testMedian1() { - ArrayList list1 = new ArrayList(); - expectedEx.expect(IllegalArgumentException.class); - expectedEx.expectMessage("Median of an empty list is not defined."); - Statistics.median(list1); - } - - @Test - public void testMedian2() { - ArrayList list2 = new ArrayList(); - list2.add(2l); - list2.add(4l); - list2.add(6l); - list2.add(8l); - assertEquals(5, Statistics.median(list2)); - - list2.add(15l); - assertEquals(6, Statistics.median(list2)); - } - - @Test - public void testDescribeFactor() { - assertEquals("", Statistics.describeFactor(0, 0, "test")); - assertEquals("(5.00test)", Statistics.describeFactor(10, 2, "test")); - } - - @Test - public void testReadableTimespan() { - assertEquals("0 sec", Statistics.readableTimespan(0)); - assertEquals("1 sec", Statistics.readableTimespan(1000)); - assertEquals("1 min", Statistics.readableTimespan(60000)); - assertEquals("1 hr", Statistics.readableTimespan(3600000)); - } - - @Test - public void testPercentile() { - List finishTimes = new ArrayList(); - for(int i=1;i<=10;i++) { - finishTimes.add((long)i*10); - } - assertEquals(100, Statistics.percentile(finishTimes,100)); - assertEquals(50, Statistics.percentile(finishTimes,50)); - assertEquals(0, Statistics.percentile(finishTimes,0)); - assertEquals(10, Statistics.percentile(finishTimes,10)); - assertEquals(10, Statistics.percentile(finishTimes,4)); - - List oddLengthValues = new ArrayList(); - oddLengthValues.add(1L); - oddLengthValues.add(2L); - oddLengthValues.add(3L); - oddLengthValues.add(4L); - oddLengthValues.add(5L); - assertEquals(3L, Statistics.percentile(oddLengthValues, 50)); - - List finishTimeSingle = new ArrayList(); - finishTimeSingle.add(10L); - assertEquals(10,Statistics.percentile(finishTimeSingle,100)); - assertEquals(0, Statistics.percentile(finishTimeSingle,0)); - assertEquals(10,Statistics.percentile(finishTimeSingle, 10)); - assertEquals(10,Statistics.percentile(finishTimeSingle, 50)); - } -} diff --git a/test/com/linkedin/drelephant/priorityexecutor/PriorityBasedThreadPoolExecutorTest.java b/test/com/linkedin/drelephant/priorityexecutor/PriorityBasedThreadPoolExecutorTest.java deleted file mode 100644 index 081a2cdd2..000000000 --- a/test/com/linkedin/drelephant/priorityexecutor/PriorityBasedThreadPoolExecutorTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.priorityexecutor; - -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import org.junit.Assert; -import org.junit.Test; - - -/** - * Test Priority based thread pool executor. - */ -public class PriorityBasedThreadPoolExecutorTest { - - @Test (timeout = 120000) - public void testPriorityBasedExecutor() throws Exception { - List priorityList = Collections.synchronizedList(new ArrayList()); - ThreadFactory factory = new ThreadFactoryBuilder().setNameFormat("priority-exec-thread-%d").build(); - ThreadPoolExecutor threadPoolExecutor = new PriorityBasedThreadPoolExecutor(2, 2, 0L, - TimeUnit.MILLISECONDS, factory); - Future future1 = threadPoolExecutor.submit( - RunnableWithPriority.get(new DummyRunnable(priorityList, Priority.LOW), Priority.LOW)); - Future future2 = threadPoolExecutor.submit( - RunnableWithPriority.get(new DummyRunnable(priorityList, Priority.LOW), Priority.LOW)); - Future future3 = threadPoolExecutor.submit( - RunnableWithPriority.get(new DummyRunnable(priorityList, Priority.LOW), Priority.LOW)); - Future future4 = threadPoolExecutor.submit( - RunnableWithPriority.get(new DummyRunnable(priorityList, Priority.LOW), Priority.LOW)); - Future future5 = threadPoolExecutor.submit( - RunnableWithPriority.get(new DummyRunnable(priorityList, Priority.NORMAL), Priority.NORMAL)); - Future future6 = threadPoolExecutor.submit( - RunnableWithPriority.get(new DummyRunnable(priorityList, Priority.NORMAL), Priority.NORMAL)); - Future future7 = threadPoolExecutor.submit( - RunnableWithPriority.get(new DummyRunnable(priorityList, Priority.HIGH), Priority.HIGH)); - Future future8 = threadPoolExecutor.submit( - RunnableWithPriority.get(new DummyRunnable(priorityList, Priority.HIGH), Priority.HIGH)); - List expectedPriorityList = Lists.newArrayList(Priority.LOW, Priority.LOW); - while (priorityList.size() < 2) { - Thread.sleep(100); - } - Assert.assertEquals(expectedPriorityList, priorityList); - future1.cancel(true); - future2.cancel(true); - expectedPriorityList.addAll(Lists.newArrayList(Priority.HIGH, Priority.HIGH)); - while (priorityList.size() < 4) { - Thread.sleep(100); - } - Assert.assertEquals(expectedPriorityList, priorityList); - future7.cancel(true); - future8.cancel(true); - expectedPriorityList.addAll(Lists.newArrayList(Priority.NORMAL, Priority.NORMAL)); - while (priorityList.size() < 6) { - Thread.sleep(100); - } - Assert.assertEquals(expectedPriorityList, priorityList); - future5.cancel(true); - future6.cancel(true); - expectedPriorityList.addAll(Lists.newArrayList(Priority.LOW, Priority.LOW)); - while (priorityList.size() < 8) { - Thread.sleep(100); - } - Assert.assertEquals(expectedPriorityList, priorityList); - future3.cancel(true); - future4.cancel(true); - threadPoolExecutor.shutdownNow(); - } - - private static class DummyRunnable implements Runnable { - private List _priorityList; - private Priority _priority; - private DummyRunnable(List priorityList, Priority priority) { - _priorityList = priorityList; - _priority = priority; - } - - @Override - public void run() { - _priorityList.add(_priority); - try { - Thread.sleep(2000); - } catch (InterruptedException e) {} - } - } -} diff --git a/test/com/linkedin/drelephant/schedulers/AirflowSchedulerTest.java b/test/com/linkedin/drelephant/schedulers/AirflowSchedulerTest.java deleted file mode 100644 index 780e730e4..000000000 --- a/test/com/linkedin/drelephant/schedulers/AirflowSchedulerTest.java +++ /dev/null @@ -1,114 +0,0 @@ -package com.linkedin.drelephant.schedulers; - -import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import org.junit.Test; - -import static com.linkedin.drelephant.schedulers.AirflowScheduler.AIRFLOW_DAG_ID; -import static com.linkedin.drelephant.schedulers.AirflowScheduler.AIRFLOW_DAG_RUN_EXECUTION_DATE; -import static com.linkedin.drelephant.schedulers.AirflowScheduler.AIRFLOW_TASK_ID; -import static com.linkedin.drelephant.schedulers.AirflowScheduler.AIRFLOW_TASK_INSTANCE_EXECUTION_DATE; - -import static org.junit.Assert.assertEquals; - - -public class AirflowSchedulerTest { - - @Test - public void testAirflowLoadInfoWithCompleteConf() { - - AirflowScheduler airScheduler = new AirflowScheduler("id", getAirflowProperties(), getSchedulerConfData()); - - assertEquals("http://localhost:1717/admin/airflow/graph?dag_id=dag_id", airScheduler.getFlowDefUrl()); - assertEquals("dag_id", airScheduler.getFlowDefId()); - assertEquals("http://localhost:1717/admin/airflow/graph?dag_id=dag_id&execution_date=dag_run_execution_date", airScheduler.getFlowExecUrl()); - assertEquals("dag_id/dag_run_execution_date", airScheduler.getFlowExecId()); - - assertEquals("http://localhost:1717/admin/airflow/code?dag_id=dag_id&task_id=task_id", airScheduler.getJobDefUrl()); - assertEquals("dag_id/task_id", airScheduler.getJobDefId()); - assertEquals("http://localhost:1717/admin/airflow/log?dag_id=dag_id&task_id=task_id&execution_date=task_instance_execution_date", airScheduler.getJobExecUrl()); - assertEquals("dag_id/dag_run_execution_date/task_id/task_instance_execution_date", airScheduler.getJobExecId()); - - assertEquals("task_id", airScheduler.getJobName()); - assertEquals(0, airScheduler.getWorkflowDepth()); - assertEquals("airflow", airScheduler.getSchedulerName()); - - } - - @Test - public void testAirflowLoadInfoWithMissingProperty() { - - AirflowScheduler airScheduler = new AirflowScheduler("id", getPropertiesAndRemove(AIRFLOW_TASK_ID), getSchedulerConfData()); - - assertEquals("http://localhost:1717/admin/airflow/graph?dag_id=dag_id", airScheduler.getFlowDefUrl()); - assertEquals("dag_id", airScheduler.getFlowDefId()); - assertEquals("http://localhost:1717/admin/airflow/graph?dag_id=dag_id&execution_date=dag_run_execution_date", airScheduler.getFlowExecUrl()); - assertEquals("dag_id/dag_run_execution_date", airScheduler.getFlowExecId()); - - assertEquals(null, airScheduler.getJobDefUrl()); - assertEquals(null, airScheduler.getJobDefId()); - assertEquals(null, airScheduler.getJobExecUrl()); - assertEquals(null, airScheduler.getJobExecId()); - - assertEquals(null, airScheduler.getJobName()); - assertEquals(0, airScheduler.getWorkflowDepth()); - assertEquals("airflow", airScheduler.getSchedulerName()); - } - - @Test - public void testAirflowLoadInfoWithNullProperty() { - - AirflowScheduler airScheduler = new AirflowScheduler("id", null, getSchedulerConfData()); - - assertEquals(null, airScheduler.getFlowDefUrl()); - assertEquals(null, airScheduler.getFlowDefId()); - assertEquals(null, airScheduler.getFlowExecId()); - assertEquals(null, airScheduler.getFlowExecUrl()); - - assertEquals(null, airScheduler.getJobDefId()); - assertEquals(null, airScheduler.getJobDefUrl()); - assertEquals(null, airScheduler.getJobExecId()); - assertEquals(null, airScheduler.getJobExecUrl()); - - assertEquals(null, airScheduler.getJobName()); - assertEquals(0, airScheduler.getWorkflowDepth()); - assertEquals("airflow", airScheduler.getSchedulerName()); - } - - @Test - public void testAirflowLoadsNameFromConfData() { - - AirflowScheduler airScheduler = new AirflowScheduler("id", null, getSchedulerConfData("othername")); - assertEquals("othername", airScheduler.getSchedulerName()); - - } - - private static Properties getAirflowProperties() { - Properties properties = new Properties(); - properties.put(AIRFLOW_DAG_ID, "dag_id"); - properties.put(AIRFLOW_DAG_RUN_EXECUTION_DATE, "dag_run_execution_date"); - properties.put(AIRFLOW_TASK_ID, "task_id"); - properties.put(AIRFLOW_TASK_INSTANCE_EXECUTION_DATE, "task_instance_execution_date"); - - return properties; - } - - private static Properties getPropertiesAndRemove(String key) { - Properties properties = getAirflowProperties(); - properties.remove(key); - return properties; - } - - private static SchedulerConfigurationData getSchedulerConfData() { - return getSchedulerConfData("airflow"); - } - - private static SchedulerConfigurationData getSchedulerConfData(String name) { - Map paramMap = new HashMap(); - paramMap.put("airflowbaseurl", "http://localhost:1717"); - return new SchedulerConfigurationData(name, null, paramMap); - } -} diff --git a/test/com/linkedin/drelephant/schedulers/AzkabanSchedulerTest.java b/test/com/linkedin/drelephant/schedulers/AzkabanSchedulerTest.java deleted file mode 100644 index 8ef888f1f..000000000 --- a/test/com/linkedin/drelephant/schedulers/AzkabanSchedulerTest.java +++ /dev/null @@ -1,110 +0,0 @@ -package com.linkedin.drelephant.schedulers; - -import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData; - -import java.util.Properties; -import org.junit.Test; - -import static com.linkedin.drelephant.schedulers.AzkabanScheduler.AZKABAN_JOB_URL; -import static com.linkedin.drelephant.schedulers.AzkabanScheduler.AZKABAN_ATTEMPT_URL; -import static com.linkedin.drelephant.schedulers.AzkabanScheduler.AZKABAN_EXECUTION_URL; -import static com.linkedin.drelephant.schedulers.AzkabanScheduler.AZKABAN_WORKFLOW_URL; -import static com.linkedin.drelephant.schedulers.AzkabanScheduler.AZKABAN_JOB_NAME; -import static org.junit.Assert.assertEquals; - - -public class AzkabanSchedulerTest { - - @Test - public void testAzkabanLoadInfoWithCompleteConf() { - - AzkabanScheduler azkScheduler = new AzkabanScheduler("id", getAzkabanProperties(), getSchedulerConfData()); - - assertEquals("https://host:9000/manager?project=project-name&flow=flow-name", azkScheduler.getFlowDefUrl()); - assertEquals("https://host:9000/manager?project=project-name&flow=flow-name", azkScheduler.getFlowDefId()); - assertEquals("https://host:9000/executor?execid=123456", azkScheduler.getFlowExecId()); - assertEquals("https://host:9000/executor?execid=123456", azkScheduler.getFlowExecUrl()); - - assertEquals("https://host:9000/manager?project=project-name&flow=flow-name&job=job-name", azkScheduler.getJobDefId()); - assertEquals("https://host:9000/manager?project=project-name&flow=flow-name&job=job-name", azkScheduler.getJobDefUrl()); - assertEquals("https://host:9000/executor?execid=123456&job=job-name&attempt=0", azkScheduler.getJobExecId()); - assertEquals("https://host:9000/executor?execid=123456&job=job-name&attempt=0", azkScheduler.getJobExecUrl()); - - assertEquals("job-name", azkScheduler.getJobName()); - assertEquals(0, azkScheduler.getWorkflowDepth()); - assertEquals("azkaban", azkScheduler.getSchedulerName()); - } - - @Test - public void testAzkabanLoadInfoWithMissingProperty() { - - AzkabanScheduler azkScheduler = new AzkabanScheduler("id", getPropertiesAndRemove(AZKABAN_JOB_URL), getSchedulerConfData()); - - assertEquals("https://host:9000/manager?project=project-name&flow=flow-name", azkScheduler.getFlowDefUrl()); - assertEquals("https://host:9000/manager?project=project-name&flow=flow-name", azkScheduler.getFlowDefId()); - assertEquals("https://host:9000/executor?execid=123456", azkScheduler.getFlowExecId()); - assertEquals("https://host:9000/executor?execid=123456", azkScheduler.getFlowExecUrl()); - - assertEquals(null, azkScheduler.getJobDefId()); - assertEquals(null, azkScheduler.getJobDefUrl()); - assertEquals("https://host:9000/executor?execid=123456&job=job-name&attempt=0", azkScheduler.getJobExecId()); - assertEquals("https://host:9000/executor?execid=123456&job=job-name&attempt=0", azkScheduler.getJobExecUrl()); - - assertEquals("job-name", azkScheduler.getJobName()); - assertEquals(0, azkScheduler.getWorkflowDepth()); - assertEquals("azkaban", azkScheduler.getSchedulerName()); - } - - @Test - public void testAzkabanLoadInfoWithNullProperty() { - - AzkabanScheduler azkScheduler = new AzkabanScheduler("id", null, getSchedulerConfData()); - - assertEquals(null, azkScheduler.getFlowDefUrl()); - assertEquals(null, azkScheduler.getFlowDefId()); - assertEquals(null, azkScheduler.getFlowExecId()); - assertEquals(null, azkScheduler.getFlowExecUrl()); - - assertEquals(null, azkScheduler.getJobDefId()); - assertEquals(null, azkScheduler.getJobDefUrl()); - assertEquals(null, azkScheduler.getJobExecId()); - assertEquals(null, azkScheduler.getJobExecUrl()); - - assertEquals(null, azkScheduler.getJobName()); - assertEquals(0, azkScheduler.getWorkflowDepth()); - assertEquals("azkaban", azkScheduler.getSchedulerName()); - } - - @Test - public void testAzkabanLoadsNameFromConfData() { - - AzkabanScheduler azkScheduler = new AzkabanScheduler("id", null, getSchedulerConfData("othername")); - assertEquals("othername", azkScheduler.getSchedulerName()); - - } - - private static Properties getAzkabanProperties() { - Properties properties = new Properties(); - properties.put(AZKABAN_JOB_URL, "https://host:9000/manager?project=project-name&flow=flow-name&job=job-name"); - properties.put(AZKABAN_ATTEMPT_URL, "https://host:9000/executor?execid=123456&job=job-name&attempt=0"); - properties.put(AZKABAN_WORKFLOW_URL, "https://host:9000/manager?project=project-name&flow=flow-name"); - properties.put(AZKABAN_EXECUTION_URL, "https://host:9000/executor?execid=123456"); - properties.put(AZKABAN_JOB_NAME, "job-name"); - - return properties; - } - - private static Properties getPropertiesAndRemove(String key) { - Properties properties = getAzkabanProperties(); - properties.remove(key); - return properties; - } - - private static SchedulerConfigurationData getSchedulerConfData() { - return getSchedulerConfData("azkaban"); - } - - private static SchedulerConfigurationData getSchedulerConfData(String name) { - return new SchedulerConfigurationData(name, null, null); - } -} diff --git a/test/com/linkedin/drelephant/schedulers/OozieSchedulerTest.java b/test/com/linkedin/drelephant/schedulers/OozieSchedulerTest.java deleted file mode 100644 index e2c800287..000000000 --- a/test/com/linkedin/drelephant/schedulers/OozieSchedulerTest.java +++ /dev/null @@ -1,319 +0,0 @@ -package com.linkedin.drelephant.schedulers; - -import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData; -import org.apache.oozie.client.*; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.runners.MockitoJUnitRunner; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.*; - -@RunWith(MockitoJUnitRunner.class) -public class OozieSchedulerTest { - private static final String parentJobInfo = "0143705-160828184536493-oozie-oozi-W"; - private static final String jobAppName = "some-workflow-name"; - private static final String parentJobAppName = jobAppName + "parent"; - private static final String jobInfo = "0004167-160629080632562-oozie-oozi-W"; - private static final String actionName = "some-action"; - private static final String actionInfo = jobInfo + "@" + actionName; - private static final String oozieUrl = "http://localhost:11000/oozie?job="; - private static final String coordinatorJobInfo = "0163255-160828184536493-oozie-oozie-C"; - private static final String coordinatorActionInfo = coordinatorJobInfo + "@1537"; - private static final String coordinatorName = "some-coordinator"; - private static final String applicationUrl = "http://localhost:8088/proxy/application_1478790851061_4847/"; - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - @Mock - private OozieClient oozieClient; - - @Mock - private WorkflowJob workflowJob; - - @Mock - private OozieClient manualCommittedJobClient; - - @Mock - private OozieClient scheduledJobClient; - - @Mock - private WorkflowAction manualChildAction; - - @Mock - private WorkflowJob manualChildJob; - - @Mock - private WorkflowJob manualParentJob; - - @Mock - private WorkflowAction scheduledChildAction; - - @Mock - private WorkflowJob scheduledChildJob; - - @Mock - private CoordinatorJob oozieCoordinatorJobInfo; - - private static Properties getNonOozieProperties() { - return new Properties(); - } - - private static Properties getOozieProperties() { - Properties properties = new Properties(); - - properties.put("oozie.action.id", actionInfo); - properties.put("oozie.job.id", jobInfo); - - return properties; - } - - private static SchedulerConfigurationData makeConfig(String name, Map params) { - return new SchedulerConfigurationData(name, OozieScheduler.class.getName(), params); - } - - private static Map getDefaultSchedulerParams() { - Map paramMap = new HashMap(); - - paramMap.put("oozie_api_url", "http://oozie.api/"); - paramMap.put("oozie_job_url_template", "http://oozie/search?workflow=%s"); - paramMap.put("oozie_job_exec_url_template", "http://oozie/workflows/%s"); - paramMap.put("oozie_workflow_url_template", "http://oozie/search?workflow=%s"); - paramMap.put("oozie_workflow_exec_url_template", "http://oozie/workflows/%s"); - - return paramMap; - } - - private static Map getSchedulerConfigWithout(String... keys) { - Map params = getDefaultSchedulerParams(); - - for (String key : keys) { - params.remove(key); - } - - return params; - } - - @Before - public void setUp() throws OozieClientException { - when(workflowJob.getAppName()).thenReturn(jobAppName); - when(workflowJob.getId()).thenReturn(jobInfo); - when(oozieClient.getJobInfo(eq(jobInfo))).thenReturn(workflowJob); - - //Manual committed job - when(manualChildAction.getConsoleUrl()).thenReturn("-"); - when(manualCommittedJobClient.getWorkflowActionInfo(actionInfo)).thenReturn(manualChildAction); - - when(manualChildJob.getId()).thenReturn(jobInfo); - when(manualChildJob.getAppName()).thenReturn(jobAppName); - when(manualChildJob.getParentId()).thenReturn(parentJobInfo); - when(manualChildJob.getConsoleUrl()).thenReturn(oozieUrl + jobInfo); - when(manualCommittedJobClient.getJobInfo(jobInfo)).thenReturn(manualChildJob); - - when(manualParentJob.getId()).thenReturn(parentJobInfo); - when(manualParentJob.getAppName()).thenReturn(parentJobAppName); - when(manualParentJob.getParentId()).thenReturn(null); - when(manualParentJob.getConsoleUrl()).thenReturn(oozieUrl + parentJobInfo); - when(manualCommittedJobClient.getJobInfo(parentJobInfo)).thenReturn(manualParentJob); - - //Oozie coordinated job - when(scheduledChildAction.getConsoleUrl()).thenReturn(applicationUrl); - when(scheduledJobClient.getWorkflowActionInfo(actionInfo)).thenReturn(scheduledChildAction); - - when(scheduledChildJob.getId()).thenReturn(jobInfo); - when(scheduledChildJob.getAppName()).thenReturn(jobAppName); - when(scheduledChildJob.getParentId()).thenReturn(coordinatorActionInfo); - when(scheduledChildJob.getConsoleUrl()).thenReturn(oozieUrl + jobInfo); - when(scheduledJobClient.getJobInfo(jobInfo)).thenReturn(scheduledChildJob); - - when(oozieCoordinatorJobInfo.getConsoleUrl()).thenReturn(null); - when(oozieCoordinatorJobInfo.getAppName()).thenReturn(coordinatorName); - when(scheduledJobClient.getCoordJobInfo(coordinatorJobInfo)).thenReturn(oozieCoordinatorJobInfo); - } - - /* - Job Reference ID: Oozie Job ID - Job Execution ID: Oozie Job ID - Flow Reference ID: Super Parent Oozie job ID - Flow Execution ID: Super Parent Oozie job ID - */ - @Test - public void testManualCommittedJob() throws Exception { - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", new HashMap()); - OozieScheduler scheduler = new OozieScheduler("id", getOozieProperties(), schedulerConfig, manualCommittedJobClient); - - assertEquals(jobInfo, scheduler.getJobDefId()); - assertEquals(jobInfo, scheduler.getJobExecId()); - assertEquals(parentJobInfo, scheduler.getFlowDefId()); - assertEquals(parentJobInfo, scheduler.getFlowExecId()); - assertEquals(oozieUrl + jobInfo, scheduler.getJobDefUrl()); - assertEquals(oozieUrl + jobInfo, scheduler.getJobExecUrl()); - assertEquals(oozieUrl + parentJobInfo, scheduler.getFlowDefUrl()); - assertEquals(oozieUrl + parentJobInfo, scheduler.getFlowExecUrl()); - assertEquals(1, scheduler.getWorkflowDepth()); - assertEquals(jobInfo, scheduler.getJobName()); - assertEquals("oozie", scheduler.getSchedulerName()); - } - - /* - Job Reference ID: Job AppName-ActionName - Job Execution ID: Oozie Job ID - Flow Reference ID: Super Parent Job AppName - Flow Execution ID: Super Parent Oozie job ID - */ - @Test - public void testManualCommittedJobAppNameUnique() throws Exception { - Map params = new HashMap(); - params.put("oozie_app_name_uniqueness", "true"); - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", params); - OozieScheduler scheduler = new OozieScheduler("id", getOozieProperties(), schedulerConfig, manualCommittedJobClient); - - assertEquals(oozieUrl + jobInfo, scheduler.getJobDefUrl()); - assertEquals(oozieUrl + jobInfo, scheduler.getJobExecUrl()); - assertEquals(oozieUrl + parentJobInfo, scheduler.getFlowDefUrl()); - assertEquals(oozieUrl + parentJobInfo, scheduler.getFlowExecUrl()); - assertEquals(1, scheduler.getWorkflowDepth()); - assertEquals(jobAppName + "-" + actionName, scheduler.getJobName()); - assertEquals("oozie", scheduler.getSchedulerName()); - } - - /* - Job Reference ID: C_ID-ActionName-Depth - Job Execution ID: Oozie Job ID - Flow Reference ID: Coordinator ID = C_ID - Flow Execution ID: Coordinator Action ID = C_ID@1 - */ - @Test - public void testOozieScheduledJob() throws Exception { - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", new HashMap()); - OozieScheduler scheduler = new OozieScheduler("id", getOozieProperties(), schedulerConfig, scheduledJobClient); - - assertEquals(coordinatorJobInfo + "-" + actionName + "-0", scheduler.getJobDefId()); - assertEquals(jobInfo, scheduler.getJobExecId()); - assertEquals(coordinatorJobInfo, scheduler.getFlowDefId()); - assertEquals(coordinatorActionInfo, scheduler.getFlowExecId()); - assertEquals(oozieUrl + jobInfo, scheduler.getJobDefUrl()); - assertEquals(oozieUrl + jobInfo, scheduler.getJobExecUrl()); - assertEquals(coordinatorJobInfo, scheduler.getFlowDefUrl()); - assertEquals(coordinatorActionInfo, scheduler.getFlowExecUrl()); - assertEquals(0, scheduler.getWorkflowDepth()); - assertEquals(coordinatorJobInfo + "-" + actionName + "-0", scheduler.getJobName()); - assertEquals("oozie", scheduler.getSchedulerName()); - } - - /* - Job Reference ID: Job AppName-ActionName - Job Execution ID: Oozie Job ID - Flow Reference ID: Coordinator Job name - Flow Execution ID: Coordinator Action ID = C_ID@1 - */ - @Test - public void tesOozieScheduledJobAppNameUnique() throws Exception { - HashMap params = new HashMap(); - params.put("oozie_app_name_uniqueness", "true"); - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", params); - OozieScheduler scheduler = new OozieScheduler("id", getOozieProperties(), schedulerConfig, scheduledJobClient); - - assertEquals(jobAppName + "-" + actionName, scheduler.getJobDefId()); - assertEquals(jobInfo, scheduler.getJobExecId()); - assertEquals(coordinatorName, scheduler.getFlowDefId()); - assertEquals(coordinatorActionInfo, scheduler.getFlowExecId()); - assertEquals(oozieUrl + jobInfo, scheduler.getJobDefUrl()); - assertEquals(oozieUrl + jobInfo, scheduler.getJobExecUrl()); - assertEquals(coordinatorJobInfo, scheduler.getFlowDefUrl()); - assertEquals(coordinatorActionInfo, scheduler.getFlowExecUrl()); - assertEquals(0, scheduler.getWorkflowDepth()); - assertEquals(jobAppName + "-" + actionName, scheduler.getJobName()); - assertEquals("oozie", scheduler.getSchedulerName()); - } - - @Test - public void testUserGivenTemplateArePreferredUrl() throws Exception { - Map params = getDefaultSchedulerParams(); - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", params); - OozieScheduler scheduler = new OozieScheduler("id", getOozieProperties(), schedulerConfig, manualCommittedJobClient); - - assertEquals("http://oozie/search?workflow=" + jobInfo, scheduler.getJobDefUrl()); - assertEquals("http://oozie/workflows/" + jobInfo, scheduler.getJobExecUrl()); - assertEquals("http://oozie/search?workflow=" + parentJobInfo, scheduler.getFlowDefUrl()); - assertEquals("http://oozie/workflows/" + parentJobInfo, scheduler.getFlowExecUrl()); - } - - @Test - public void testDepthCalculation() throws Exception { - when(workflowJob.getParentId()).thenReturn(jobInfo, jobInfo, jobInfo, null); - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", new HashMap()); - OozieScheduler scheduler = new OozieScheduler("id", getOozieProperties(), schedulerConfig, oozieClient); - - assertEquals(1, scheduler.getWorkflowDepth()); - - } - - @Test - public void testOozieLoadInfoWithOozieClientException() throws Exception { - thrown.expect(RuntimeException.class); - thrown.expectMessage("Failed fetching Oozie workflow " + jobInfo + " info"); - - doThrow(new OozieClientException("500 Internal server error", "BOOM")).when(oozieClient).getJobInfo(anyString()); - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", getDefaultSchedulerParams()); - new OozieScheduler("id", getOozieProperties(), schedulerConfig, oozieClient); - } - - @Test - public void testOozieLoadInfoWithMissingProperty() { - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", getDefaultSchedulerParams()); - OozieScheduler scheduler = new OozieScheduler("id", getNonOozieProperties(), schedulerConfig); - - assertEquals(null, scheduler.getFlowDefUrl()); - assertEquals(null, scheduler.getFlowDefId()); - assertEquals(null, scheduler.getFlowExecUrl()); - assertEquals(null, scheduler.getFlowExecId()); - - assertEquals(null, scheduler.getJobDefUrl()); - assertEquals(null, scheduler.getJobDefId()); - assertEquals(null, scheduler.getJobExecUrl()); - assertEquals(null, scheduler.getJobExecId()); - - assertEquals(null, scheduler.getJobName()); - assertEquals(0, scheduler.getWorkflowDepth()); - assertEquals("oozie", scheduler.getSchedulerName()); - } - - @Test - public void testOozieLoadInfoWithNullProperty() { - SchedulerConfigurationData schedulerConfig = makeConfig("oozie", getDefaultSchedulerParams()); - OozieScheduler scheduler = new OozieScheduler("id", null, schedulerConfig); - - assertEquals(null, scheduler.getFlowDefUrl()); - assertEquals(null, scheduler.getFlowDefId()); - assertEquals(null, scheduler.getFlowExecId()); - assertEquals(null, scheduler.getFlowExecUrl()); - - assertEquals(null, scheduler.getJobDefId()); - assertEquals(null, scheduler.getJobDefUrl()); - assertEquals(null, scheduler.getJobExecId()); - assertEquals(null, scheduler.getJobExecUrl()); - - assertEquals(null, scheduler.getJobName()); - assertEquals(0, scheduler.getWorkflowDepth()); - assertEquals("oozie", scheduler.getSchedulerName()); - } - - @Test - public void testOozieLoadsNameFromConfData() { - SchedulerConfigurationData schedulerConfig = makeConfig("othername", getDefaultSchedulerParams()); - OozieScheduler scheduler = new OozieScheduler("id", null, schedulerConfig); - assertEquals("othername", scheduler.getSchedulerName()); - } -} diff --git a/test/com/linkedin/drelephant/schedulers/PinballSchedulerTest.java b/test/com/linkedin/drelephant/schedulers/PinballSchedulerTest.java deleted file mode 100644 index 5c4154a88..000000000 --- a/test/com/linkedin/drelephant/schedulers/PinballSchedulerTest.java +++ /dev/null @@ -1,106 +0,0 @@ -package com.linkedin.drelephant.schedulers; - -import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData; - -import java.util.Properties; -import org.junit.Test; - -import static com.linkedin.drelephant.schedulers.PinballScheduler.PINBALL_WORKFLOW; -import static com.linkedin.drelephant.schedulers.PinballScheduler.PINBALL_INSTANCE; -import static com.linkedin.drelephant.schedulers.PinballScheduler.PINBALL_JOB; -import static com.linkedin.drelephant.schedulers.PinballScheduler.PINBALL_EXECUTION; -import static com.linkedin.drelephant.schedulers.PinballScheduler.PINBALL_BASE_URL; - -import static org.junit.Assert.assertEquals; - - -public class PinballSchedulerTest { - - @Test - public void testPinballLoadInfoWithCompleteConf() { - PinballScheduler pinballScheduler = new PinballScheduler("id", getPinballProperties(), getSchedulerConfData()); - - assertEquals("http://localhost:8080/instances/?workflow=workflow_name", pinballScheduler.getFlowDefUrl()); - assertEquals("workflow_name", pinballScheduler.getFlowDefId()); - assertEquals("http://localhost:8080/jobs/?workflow=workflow_name&instance=workflow_instance", pinballScheduler.getFlowExecUrl()); - assertEquals("workflow_name/workflow_instance", pinballScheduler.getFlowExecId()); - - assertEquals("http://localhost:8080/executions/?workflow=workflow_name&instance=workflow_instance&job=job_name", pinballScheduler.getJobDefUrl()); - assertEquals("workflow_name/job_name", pinballScheduler.getJobDefId()); - assertEquals("http://localhost:8080/execution/?workflow=workflow_name&instance=workflow_instance&job=job_name&execution=job_execution", pinballScheduler.getJobExecUrl()); - assertEquals("workflow_name/workflow_instance/job_name/job_execution", pinballScheduler.getJobExecId()); - - assertEquals("job_name", pinballScheduler.getJobName()); - assertEquals(0, pinballScheduler.getWorkflowDepth()); - assertEquals("pinball", pinballScheduler.getSchedulerName()); - } - - @Test - public void testPinballLoadInfoWithMissingProperty() { - PinballScheduler pinballScheduler = new PinballScheduler("id", getPropertiesAndRemove(PINBALL_JOB), getSchedulerConfData()); - - assertEquals("http://localhost:8080/instances/?workflow=workflow_name", pinballScheduler.getFlowDefUrl()); - assertEquals("workflow_name", pinballScheduler.getFlowDefId()); - assertEquals("http://localhost:8080/jobs/?workflow=workflow_name&instance=workflow_instance", pinballScheduler.getFlowExecUrl()); - assertEquals("workflow_name/workflow_instance", pinballScheduler.getFlowExecId()); - - assertEquals(null, pinballScheduler.getJobDefUrl()); - assertEquals(null, pinballScheduler.getJobDefId()); - assertEquals(null, pinballScheduler.getJobExecUrl()); - assertEquals(null, pinballScheduler.getJobExecId()); - - assertEquals(null, pinballScheduler.getJobName()); - assertEquals(0, pinballScheduler.getWorkflowDepth()); - assertEquals("pinball", pinballScheduler.getSchedulerName()); - } - - @Test - public void testPinballLoadInfoWithNullProperty() { - PinballScheduler pinballScheduler = new PinballScheduler("id", null, getSchedulerConfData()); - - assertEquals(null, pinballScheduler.getFlowDefUrl()); - assertEquals(null, pinballScheduler.getFlowDefId()); - assertEquals(null, pinballScheduler.getFlowExecId()); - assertEquals(null, pinballScheduler.getFlowExecUrl()); - - assertEquals(null, pinballScheduler.getJobDefId()); - assertEquals(null, pinballScheduler.getJobDefUrl()); - assertEquals(null, pinballScheduler.getJobExecId()); - assertEquals(null, pinballScheduler.getJobExecUrl()); - - assertEquals(null, pinballScheduler.getJobName()); - assertEquals(0, pinballScheduler.getWorkflowDepth()); - assertEquals("pinball", pinballScheduler.getSchedulerName()); - } - - @Test - public void testPinballLoadsNameFromConfData() { - PinballScheduler pinballScheduler = new PinballScheduler("id", null, getSchedulerConfData("othername")); - assertEquals("othername", pinballScheduler.getSchedulerName()); - } - - private static Properties getPinballProperties() { - Properties properties = new Properties(); - properties.put(PINBALL_WORKFLOW, "workflow_name"); - properties.put(PINBALL_INSTANCE, "workflow_instance"); - properties.put(PINBALL_JOB, "job_name"); - properties.put(PINBALL_EXECUTION, "job_execution"); - properties.put(PINBALL_BASE_URL, "http://localhost:8080"); - - return properties; - } - - private static Properties getPropertiesAndRemove(String key) { - Properties properties = getPinballProperties(); - properties.remove(key); - return properties; - } - - private static SchedulerConfigurationData getSchedulerConfData() { - return getSchedulerConfData("pinball"); - } - - private static SchedulerConfigurationData getSchedulerConfData(String name) { - return new SchedulerConfigurationData(name, null, null); - } -} diff --git a/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala b/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala deleted file mode 100644 index 77e3e1d29..000000000 --- a/test/com/linkedin/drelephant/spark/SparkMetricsAggregatorTest.scala +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark - -import java.util.Date - -import scala.collection.JavaConverters - -import com.linkedin.drelephant.analysis.ApplicationType -import com.linkedin.drelephant.configurations.aggregator.AggregatorConfigurationData -import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationAttemptInfoImpl, ApplicationInfoImpl, ExecutorSummaryImpl} -import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate -import org.scalatest.{FunSpec, Matchers} - -class SparkMetricsAggregatorTest extends FunSpec with Matchers { - import SparkMetricsAggregatorTest._ - - describe("SparkMetricsAggregator") { - val aggregatorConfigurationData = newFakeAggregatorConfigurationData( - Map("allocated_memory_waste_buffer_percentage" -> "0.5") - ) - - val appId = "application_1" - - val applicationInfo = { - val applicationAttemptInfo = { - val now = System.currentTimeMillis - val duration = 8000000L - newFakeApplicationAttemptInfo(Some("1"), startTime = new Date(now - duration), endTime = new Date(now)) - } - new ApplicationInfoImpl(appId, name = "app", Seq(applicationAttemptInfo)) - } - - val executorSummaries = Seq( - newFakeExecutorSummary(id = "1", totalDuration = 1000000L), - newFakeExecutorSummary(id = "2", totalDuration = 3000000L) - ) - val restDerivedData = { - SparkRestDerivedData( - applicationInfo, - jobDatas = Seq.empty, - stageDatas = Seq.empty, - executorSummaries = executorSummaries - ) - } - - describe("when it has log-derived data") { - val logDerivedData = { - val environmentUpdate = newFakeSparkListenerEnvironmentUpdate( - Map( - "spark.serializer" -> "org.apache.spark.serializer.KryoSerializer", - "spark.storage.memoryFraction" -> "0.3", - "spark.driver.memory" -> "2G", - "spark.executor.instances" -> "2", - "spark.executor.memory" -> "4g", - "spark.shuffle.memoryFraction" -> "0.5" - ) - ) - SparkLogDerivedData(environmentUpdate) - } - - val data = SparkApplicationData(appId, restDerivedData, Some(logDerivedData)) - - val aggregator = new SparkMetricsAggregator(aggregatorConfigurationData) - aggregator.aggregate(data) - - val result = aggregator.getResult - - it("calculates resources used") { - val totalExecutorMemoryMb = 2 * 4096 - val applicationDurationSeconds = 8000 - val executorMemoryMb = 4096 - val totalExecutorTaskTimeSeconds = 1000 + 3000 - result.getResourceUsed should be(totalExecutorMemoryMb * applicationDurationSeconds) - } - - it("calculates resources wasted") { - val totalExecutorMemoryMb = 2 * 4096 - val applicationDurationSeconds = 8000 - val resourceAllocated = totalExecutorMemoryMb * applicationDurationSeconds; - - val executorMemoryMb = 4096 - val totalExecutorTaskTimeSeconds = 1000 + 3000 - val resourceUsed = executorMemoryMb * totalExecutorTaskTimeSeconds; - - - result.getResourceWasted should be(resourceAllocated - resourceUsed * 1.5) - } - - it("doesn't calculate total delay") { - result.getTotalDelay should be(0L) - } - it("sets resourceused as 0 when duration is negative") { - //make the duration negative - val applicationInfo = { - val applicationAttemptInfo = { - val now = System.currentTimeMillis - val duration = -8000000L - newFakeApplicationAttemptInfo(Some("1"), startTime = new Date(now - duration), endTime = new Date(now)) - } - new ApplicationInfoImpl(appId, name = "app", Seq(applicationAttemptInfo)) - } - val restDerivedData = SparkRestDerivedData( - applicationInfo, - jobDatas = Seq.empty, - stageDatas = Seq.empty, - executorSummaries = executorSummaries - ) - - val data = SparkApplicationData(appId, restDerivedData, Some(logDerivedData)) - - val aggregator = new SparkMetricsAggregator(aggregatorConfigurationData) - aggregator.aggregate(data) - - val result = aggregator.getResult - result.getResourceUsed should be(0L) - } - } - - describe("when it doesn't have log-derived data") { - val data = SparkApplicationData(appId, restDerivedData, logDerivedData = None) - - val aggregator = new SparkMetricsAggregator(aggregatorConfigurationData) - aggregator.aggregate(data) - - val result = aggregator.getResult - - it("doesn't calculate resources used") { - result.getResourceUsed should be(0L) - } - - it("doesn't calculate resources wasted") { - result.getResourceWasted should be(0L) - } - - it("doesn't calculate total delay") { - result.getTotalDelay should be(0L) - } - } - } -} - -object SparkMetricsAggregatorTest { - import JavaConverters._ - - def newFakeAggregatorConfigurationData(params: Map[String, String] = Map.empty): AggregatorConfigurationData = - new AggregatorConfigurationData("org.apache.spark.SparkMetricsAggregator", new ApplicationType("SPARK"), params.asJava) - - def newFakeSparkListenerEnvironmentUpdate(appConfigurationProperties: Map[String, String]): SparkListenerEnvironmentUpdate = - SparkListenerEnvironmentUpdate(Map("Spark Properties" -> appConfigurationProperties.toSeq)) - - def newFakeApplicationAttemptInfo( - attemptId: Option[String], - startTime: Date, - endTime: Date - ): ApplicationAttemptInfoImpl = new ApplicationAttemptInfoImpl( - attemptId, - startTime, - endTime, - sparkUser = "foo", - completed = true - ) - - def newFakeExecutorSummary( - id: String, - totalDuration: Long - ): ExecutorSummaryImpl = new ExecutorSummaryImpl( - id, - hostPort = "", - rddBlocks = 0, - memoryUsed = 0, - diskUsed = 0, - activeTasks = 0, - failedTasks = 0, - completedTasks = 0, - totalTasks = 0, - totalDuration, - totalInputBytes = 0, - totalShuffleRead = 0, - totalShuffleWrite = 0, - maxMemory = 0, - totalGCTime = 0, - executorLogs = Map.empty - ) -} diff --git a/test/com/linkedin/drelephant/spark/data/SparkApplicationDataTest.scala b/test/com/linkedin/drelephant/spark/data/SparkApplicationDataTest.scala deleted file mode 100644 index e6ec6d51d..000000000 --- a/test/com/linkedin/drelephant/spark/data/SparkApplicationDataTest.scala +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.data - -import java.util.Date - -import scala.collection.JavaConverters - -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationAttemptInfoImpl, ApplicationInfoImpl} -import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate -import org.scalatest.{FunSpec, Matchers} - -class SparkApplicationDataTest extends FunSpec with Matchers { - import SparkApplicationDataTest._ - import JavaConverters._ - - describe("SparkApplicationData") { - val appId = "application_1" - val attemptId = Some("1") - - val applicationAttemptInfo = { - val now = System.currentTimeMillis - val duration = 8000000L - newFakeApplicationAttemptInfo(attemptId, startTime = new Date(now - duration), endTime = new Date(now)) - } - - val restDerivedData = SparkRestDerivedData( - new ApplicationInfoImpl(appId, "app", Seq(applicationAttemptInfo)), - jobDatas = Seq.empty, - stageDatas = Seq.empty, - executorSummaries = Seq.empty - ) - - val configurationProperties = Map( - "spark.serializer" -> "org.apache.spark.serializer.KryoSerializer", - "spark.storage.memoryFraction" -> "0.3", - "spark.driver.memory" -> "2G", - "spark.executor.instances" -> "900", - "spark.executor.memory" -> "1g", - "spark.shuffle.memoryFraction" -> "0.5" - ) - - val logDerivedData = SparkLogDerivedData( - SparkListenerEnvironmentUpdate(Map("Spark Properties" -> configurationProperties.toSeq)) - ) - - describe(".getConf") { - it("returns the Spark properties") { - val data = SparkApplicationData(appId, restDerivedData, Some(logDerivedData)) - data.getConf.asScala should contain theSameElementsAs(configurationProperties) - } - } - } -} - -object SparkApplicationDataTest { - def newFakeApplicationAttemptInfo( - attemptId: Option[String], - startTime: Date, - endTime: Date - ): ApplicationAttemptInfoImpl = new ApplicationAttemptInfoImpl( - attemptId, - startTime, - endTime, - sparkUser = "foo", - completed = true - ) -} diff --git a/test/com/linkedin/drelephant/spark/fetchers/SparkFetcherTest.scala b/test/com/linkedin/drelephant/spark/fetchers/SparkFetcherTest.scala deleted file mode 100644 index 9843c3575..000000000 --- a/test/com/linkedin/drelephant/spark/fetchers/SparkFetcherTest.scala +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.fetchers - -import java.nio.file.Files -import java.util.Date -import java.util.concurrent.TimeoutException - -import scala.concurrent.{ExecutionContext, Future} -import com.linkedin.drelephant.analysis.{AnalyticJob, ApplicationType} -import com.linkedin.drelephant.configurations.fetcher.FetcherConfigurationData -import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} -import com.linkedin.drelephant.spark.fetchers.SparkFetcher.EventLogSource -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationAttemptInfoImpl, ApplicationInfoImpl} -import com.linkedin.drelephant.util.{HadoopUtils, SparkUtils} -import org.apache.log4j.Logger -import org.apache.spark.SparkConf -import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate -import org.mockito.Mockito -import org.scalatest.{FunSpec, Matchers} -import org.scalatest.mockito.MockitoSugar - -class SparkFetcherTest extends FunSpec with Matchers with MockitoSugar { - import SparkFetcherTest._ - - describe("SparkFetcher") { - import ExecutionContext.Implicits.global - - val fetcherConfigurationData = newFakeFetcherConfigurationData() - - val appId = "application_1" - - val t2 = System.currentTimeMillis - val t1 = t2 - 1 - val duration = 8000000L - - val restApplicationInfo: Seq[ApplicationInfoImpl] = Seq( - new ApplicationInfoImpl( - appId, - "app", - Seq( - newFakeApplicationAttemptInfo(Some("2"), startTime = new Date(t2 - duration), endTime = new Date(t2)), - newFakeApplicationAttemptInfo(Some("1"), startTime = new Date(t1 - duration), endTime = new Date(t1)) - ) - ) - ) - - val restDerivedData = SparkRestDerivedData( - restApplicationInfo.head, - jobDatas = Seq.empty, - stageDatas = Seq.empty, - executorSummaries = Seq.empty - ) - - val logDerivedData = SparkLogDerivedData(SparkListenerEnvironmentUpdate(Map.empty)) - - val analyticJob = new AnalyticJob().setAppId(appId) - - it("returns data") { - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf = new SparkConf() - override lazy val sparkRestClient = newFakeSparkRestClient(appId, Future(restDerivedData)) - override lazy val sparkLogClient = newFakeSparkLogClient(appId, Some("2"), Future(logDerivedData)) - } - val data = sparkFetcher.fetchData(analyticJob) - data.appId should be(appId) - } - - it("throws an exception if the REST client fails") { - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf = new SparkConf() - override lazy val sparkRestClient = newFakeSparkRestClient(appId, Future { throw new Exception() }) - override lazy val sparkLogClient = newFakeSparkLogClient(appId, Some("2"), Future(logDerivedData)) - } - - an[Exception] should be thrownBy { sparkFetcher.fetchData(analyticJob) } - } - - it("throws an exception if the log client fails") { - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "true") - override lazy val sparkRestClient = newFakeSparkRestClient(appId, Future(restDerivedData)) - override lazy val sparkLogClient = newFakeSparkLogClient(appId, Some("2"), Future { throw new Exception() }) - } - - an[Exception] should be thrownBy { sparkFetcher.fetchData(analyticJob) } - } - - it("returns SparkApplicationData when use_rest_for_eventlogs and should_process_logs_locally both are true") { - val fetcherConfigurationData = newFakeFetcherConfigurationData( - Map("use_rest_for_eventlogs" -> "true", "should_process_logs_locally" -> "true")) - val mockSparkRestClient = Mockito.mock(classOf[SparkRestClient]) - val mockSparkApplicationData = Mockito.mock(classOf[SparkApplicationData]) - Mockito.when(mockSparkApplicationData.getAppId()).thenReturn(appId) - val analyticJob = new AnalyticJob().setAppId(appId) - Mockito.when(mockSparkRestClient.fetchEventLogAndParse(analyticJob)).thenReturn(mockSparkApplicationData) - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "true") - override lazy val sparkRestClient = mockSparkRestClient - } - - sparkFetcher.fetchData(analyticJob).getAppId() should be(appId) - } - - it("throws an exception when use_rest_for_eventlogs and should_process_logs_locally both are true, and fetchSparkApplicationData throws an excpetion") { - val fetcherConfigurationData = newFakeFetcherConfigurationData( - Map("use_rest_for_eventlogs" -> "true", "should_process_logs_locally" -> "true")) - val mockSparkRestClient = Mockito.mock(classOf[SparkRestClient]) - val analyticJob = new AnalyticJob().setAppId(appId) - Mockito.when(mockSparkRestClient.fetchEventLogAndParse(analyticJob)).thenThrow(new RuntimeException()) - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "true") - override lazy val sparkRestClient = mockSparkRestClient - } - - an[TimeoutException] should be thrownBy { sparkFetcher.fetchData(analyticJob) } - } - - it("gets its SparkConf when SPARK_CONF_DIR is set") { - val tempDir = Files.createTempDirectory(null) - - val testResourceIn = getClass.getClassLoader.getResourceAsStream("spark-defaults.conf") - val testResourceFile = tempDir.resolve("spark-defaults.conf") - Files.copy(testResourceIn, testResourceFile) - - val fetcherConfigurationData = newFakeFetcherConfigurationData() - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkUtils = new SparkUtils() { - override lazy val logger = mock[Logger] - override lazy val hadoopUtils = mock[HadoopUtils] - override lazy val defaultEnv = Map("SPARK_CONF_DIR" -> tempDir.toString) - } - } - val sparkConf = sparkFetcher.sparkConf - - testResourceIn.close() - Files.delete(testResourceFile) - Files.delete(tempDir) - - sparkConf.get("spark.yarn.historyServer.address") should be("jh1.grid.example.com:18080") - sparkConf.get("spark.eventLog.enabled") should be("true") - sparkConf.get("spark.eventLog.compress") should be("true") - sparkConf.get("spark.eventLog.dir") should be("hdfs://nn1.grid.example.com:9000/logs/spark") - } - - it("gets its SparkConf when SPARK_HOME is set") { - val tempDir = Files.createTempDirectory(null) - val tempConfDir = Files.createDirectory(tempDir.resolve("conf")) - - val testResourceIn = getClass.getClassLoader.getResourceAsStream("spark-defaults.conf") - val testResourceFile = tempConfDir.resolve("spark-defaults.conf") - Files.copy(testResourceIn, testResourceFile) - - val fetcherConfigurationData = newFakeFetcherConfigurationData() - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkUtils = new SparkUtils() { - override lazy val logger = mock[Logger] - override lazy val hadoopUtils = mock[HadoopUtils] - override lazy val defaultEnv = Map("SPARK_HOME" -> tempDir.toString) - } - } - val sparkConf = sparkFetcher.sparkConf - - testResourceIn.close() - Files.delete(testResourceFile) - Files.delete(tempConfDir) - Files.delete(tempDir) - - sparkConf.get("spark.yarn.historyServer.address") should be("jh1.grid.example.com:18080") - sparkConf.get("spark.eventLog.enabled") should be("true") - sparkConf.get("spark.eventLog.compress") should be("true") - sparkConf.get("spark.eventLog.dir") should be("hdfs://nn1.grid.example.com:9000/logs/spark") - } - - it("throws an exception if neither SPARK_CONF_DIR nor SPARK_HOME are set") { - val fetcherConfigurationData = newFakeFetcherConfigurationData() - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkUtils = new SparkUtils() { - override lazy val logger = mock[Logger] - override lazy val hadoopUtils = mock[HadoopUtils] - override lazy val defaultEnv = Map.empty[String, String] - } - } - - an[IllegalStateException] should be thrownBy { sparkFetcher.sparkConf } - } - - it("eventlog source defaults to WebHDFS") { - val fetcherConfigurationData = newFakeFetcherConfigurationData() - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "true") - } - - sparkFetcher.eventLogSource should be(EventLogSource.WebHdfs) - } - - it("eventlog source is WebHDFS if use_rest_for_eventlogs is false") { - val fetcherConfigurationData = newFakeFetcherConfigurationData( - Map("use_rest_for_eventlogs" -> "false")) - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "true") - } - - sparkFetcher.eventLogSource should be(EventLogSource.WebHdfs) - } - - it("eventlog source is REST if use_rest_for_eventlogs is true") { - val fetcherConfigurationData = newFakeFetcherConfigurationData( - Map("use_rest_for_eventlogs" -> "true")) - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "true") - } - - sparkFetcher.eventLogSource should be(EventLogSource.Rest) - } - - it("eventlog fetching is disabled when spark.eventLog is false") { - val fetcherConfigurationData = newFakeFetcherConfigurationData() - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "false") - } - - sparkFetcher.eventLogSource should be(EventLogSource.None) - } - - it("processing log locally via rest is disabled when should_process_logs_locally is false") { - val fetcherConfigurationData = newFakeFetcherConfigurationData( - Map("use_rest_for_eventlogs" -> "true", "should_process_logs_locally" -> "false")) - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "true") - } - - sparkFetcher.eventLogSource should be(EventLogSource.Rest) - sparkFetcher.shouldProcessLogsLocally should be(false) - } - - it("processing log locally via rest is disabled when use_rest_for_eventlogs is false") { - val fetcherConfigurationData = newFakeFetcherConfigurationData( - Map("use_rest_for_eventlogs" -> "false", "should_process_logs_locally" -> "true")) - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "true") - } - - sparkFetcher.shouldProcessLogsLocally should be(false) - } - - it("processing log locally via rest is disabled when spark.eventLog is false") { - val fetcherConfigurationData = newFakeFetcherConfigurationData( - Map("use_rest_for_eventlogs" -> "true", "should_process_logs_locally" -> "true")) - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf: SparkConf = new SparkConf() - .set(SparkFetcher.SPARK_EVENT_LOG_ENABLED_KEY, "false") - } - - sparkFetcher.shouldProcessLogsLocally should be(false) - } - - it("fetch jobs for backfill") { - val sparkFetcher = new SparkFetcher(fetcherConfigurationData) { - override lazy val sparkConf = new SparkConf() - override lazy val sparkRestClient = newFakeSparkRestClient(appId, 100, 200, restApplicationInfo) - override lazy val sparkLogClient = newFakeSparkLogClient(appId, Some("2"), Future(logDerivedData)) - } - val result = sparkFetcher.fetchJobsForBackfill(100, 200); - result.size() should be(1) - result.get(0).getAppId should be(restApplicationInfo.head.id) - val lastAttempt = restApplicationInfo.head.attempts.maxBy { - _.endTime - } - result.get(0).getFinishTime should be(lastAttempt.endTime.getTime) - result.get(0).getUser should be(lastAttempt.sparkUser) - } - } -} - -object SparkFetcherTest { - import scala.collection.JavaConverters._ - - def newFakeFetcherConfigurationData(paramMap: Map[String, String] = Map.empty): FetcherConfigurationData = - new FetcherConfigurationData(classOf[SparkFetcher].getName, new ApplicationType("SPARK"), paramMap.asJava) - - def newFakeApplicationAttemptInfo( - attemptId: Option[String], - startTime: Date, - endTime: Date - ): ApplicationAttemptInfoImpl = new ApplicationAttemptInfoImpl( - attemptId, - startTime, - endTime, - sparkUser = "foo", - completed = true - ) - - def newFakeSparkRestClient( - appId: String, - restDerivedData: Future[SparkRestDerivedData] - )( - implicit ec: ExecutionContext - ): SparkRestClient = { - val sparkRestClient = Mockito.mock(classOf[SparkRestClient]) - Mockito.when(sparkRestClient.fetchData(appId)).thenReturn(restDerivedData) - sparkRestClient - } - - def newFakeSparkRestClient( - appId: String, - startTime: Long, - endTime: Long, - restApplicationInfo: Seq[ApplicationInfoImpl] - ) : SparkRestClient = { - val sparkRestClient = Mockito.mock(classOf[SparkRestClient]) - Mockito.when(sparkRestClient.fetchCompletedApplicationsData(startTime, endTime)).thenReturn(restApplicationInfo) - sparkRestClient - } - - def newFakeSparkLogClient( - appId: String, - attemptId: Option[String], - logDerivedData: Future[SparkLogDerivedData] - )( - implicit ec: ExecutionContext - ): SparkLogClient = { - val sparkLogClient = Mockito.mock(classOf[SparkLogClient]) - Mockito.when(sparkLogClient.fetchData(appId, attemptId)).thenReturn(logDerivedData) - sparkLogClient - } -} diff --git a/test/com/linkedin/drelephant/spark/fetchers/SparkLogClientTest.scala b/test/com/linkedin/drelephant/spark/fetchers/SparkLogClientTest.scala deleted file mode 100644 index 994af486f..000000000 --- a/test/com/linkedin/drelephant/spark/fetchers/SparkLogClientTest.scala +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.fetchers - -import java.io.{ByteArrayOutputStream, InputStream} -import java.net.URI - -import scala.concurrent.ExecutionContext - -import com.linkedin.drelephant.util.{SparkUtils, SparkUtilsTest} -import org.apache.commons.io.IOUtils -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.SparkConf -import org.scalatest.{AsyncFunSpec, Matchers} -import org.scalatest.mockito.MockitoSugar -import org.xerial.snappy.SnappyOutputStream - -class SparkLogClientTest extends AsyncFunSpec with Matchers with MockitoSugar { - describe("SparkLogClient") { - it("returns log-derived data") { - val hadoopConfiguration = new Configuration(false) - - val sparkConf = - new SparkConf() - .set("spark.eventLog.dir", "webhdfs://nn1.grid.example.com:50070/logs/spark") - .set("spark.eventLog.compress", "true") - .set("spark.io.compression.codec", "snappy") - - val appId = "application_1" - val attemptId = Some("1") - - val eventLogBytes = { - val bout = new ByteArrayOutputStream() - for { - in <- resource.managed(getClass.getClassLoader.getResourceAsStream("spark_event_logs/event_log_2")) - out <- resource.managed(new SnappyOutputStream(bout)) - } { - IOUtils.copy(in, out) - } - bout.toByteArray - } - - val sparkLogClient = new SparkLogClient(hadoopConfiguration, sparkConf, None) { - override lazy val sparkUtils = SparkUtilsTest.newFakeSparkUtilsForEventLog( - new URI("webhdfs://nn1.grid.example.com:50070"), - new Path("/logs/spark"), - new Path("application_1_1.snappy"), - eventLogBytes - ) - - override protected def doAsPrivilegedAction[T](action: () => T): T = action() - } - - sparkLogClient.fetchData(appId, attemptId).map { logDerivedData => - val expectedProperties = Map( - "spark.serializer" -> "org.apache.spark.serializer.KryoSerializer", - "spark.storage.memoryFraction" -> "0.3", - "spark.driver.memory" -> "2G", - "spark.executor.instances" -> "900", - "spark.executor.memory" -> "1g", - "spark.shuffle.memoryFraction" -> "0.5" - ) - val actualProperties = logDerivedData.appConfigurationProperties - actualProperties should be(expectedProperties) - } - } - } -} diff --git a/test/com/linkedin/drelephant/spark/fetchers/SparkRestClientTest.scala b/test/com/linkedin/drelephant/spark/fetchers/SparkRestClientTest.scala deleted file mode 100644 index 8ad9732ba..000000000 --- a/test/com/linkedin/drelephant/spark/fetchers/SparkRestClientTest.scala +++ /dev/null @@ -1,524 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.fetchers - -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream} -import java.text.SimpleDateFormat -import java.util.zip.{ZipEntry, ZipInputStream, ZipOutputStream} -import java.util.{Calendar, Date, SimpleTimeZone} - -import javax.ws.rs.client.WebTarget -import com.linkedin.drelephant.spark.fetchers.statusapiv1.StageStatus - -import scala.concurrent.ExecutionContext -import scala.util.Try -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationAttemptInfoImpl, ApplicationInfoImpl, ExecutorSummaryImpl, JobDataImpl, StageDataImpl} -import javax.ws.rs.{GET, Path, PathParam, Produces} -import javax.ws.rs.core.{Application, MediaType, Response} -import javax.ws.rs.ext.ContextResolver -import com.google.common.io.Resources -import com.linkedin.drelephant.analysis.AnalyticJob -import com.ning.compress.lzf.LZFEncoder -import org.apache.spark.{JobExecutionStatus, SparkConf} -import org.glassfish.jersey.client.ClientConfig -import org.glassfish.jersey.server.ResourceConfig -import org.glassfish.jersey.test.{JerseyTest, TestProperties} -import org.scalatest.{AsyncFunSpec, Matchers} -import org.scalatest.compatible.Assertion - -class SparkRestClientTest extends AsyncFunSpec with Matchers { - import SparkRestClientTest._ - - describe("SparkRestClient") { - it("returns the desired data from the Spark REST API for cluster mode application") { - import ExecutionContext.Implicits.global - val fakeJerseyServer = new FakeJerseyServer() { - override def configure(): Application = super.configure() match { - case resourceConfig: ResourceConfig => - resourceConfig - .register(classOf[FetchClusterModeDataFixtures.ApiResource]) - .register(classOf[FetchClusterModeDataFixtures.ApplicationResource]) - .register(classOf[FetchClusterModeDataFixtures.JobsResource]) - .register(classOf[FetchClusterModeDataFixtures.StagesResource]) - .register(classOf[FetchClusterModeDataFixtures.ExecutorsResource]) - .register(classOf[FetchClusterModeDataFixtures.LogsResource]) - case config => config - } - } - fakeJerseyServer.setUp() - - val historyServerUri = fakeJerseyServer.target.getUri - - val sparkConf = new SparkConf().set("spark.yarn.historyServer.address", s"${historyServerUri.getHost}:${historyServerUri.getPort}") - val sparkRestClient = new SparkRestClient(sparkConf) - - sparkRestClient.fetchData(FetchClusterModeDataFixtures.APP_ID) map { restDerivedData => - restDerivedData.applicationInfo.id should be(FetchClusterModeDataFixtures.APP_ID) - restDerivedData.applicationInfo.name should be(FetchClusterModeDataFixtures.APP_NAME) - restDerivedData.jobDatas should not be (None) - restDerivedData.stageDatas should not be (None) - restDerivedData.executorSummaries should not be (None) - restDerivedData.logDerivedData should be(None) - } flatMap { - case assertion: Try[Assertion] => assertion - case _ => - sparkRestClient.fetchData(FetchClusterModeDataFixtures.APP_ID, fetchLogs = true) - .map { _.logDerivedData.get.appConfigurationProperties should be(EXPECTED_PROPERTIES_FROM_LOG_1) } - } andThen { case assertion: Try[Assertion] => - fakeJerseyServer.tearDown() - assertion - } - } - - it("returns the desired SparkApplicationData using Spark REST API based eventlog for cluster mode application") { - import ExecutionContext.Implicits.global - val fakeJerseyServer = new FakeJerseyServer() { - override def configure(): Application = super.configure() match { - case resourceConfig: ResourceConfig => - resourceConfig - .register(classOf[FetchClusterModeDataFixtures.ApiResource]) - .register(classOf[FetchClusterModeDataFixtures.LogsResource]) - case config => config - } - } - fakeJerseyServer.setUp() - - val historyServerUri = fakeJerseyServer.target.getUri - - val sparkConf = new SparkConf().set("spark.yarn.historyServer.address", s"${historyServerUri.getHost}:${historyServerUri.getPort}") - val sparkRestClient = new SparkRestClient(sparkConf) - val job = new AnalyticJob().setAppId(FetchClusterModeDataFixtures.APP_ID) - val sparkApplicationData = sparkRestClient.fetchEventLogAndParse(job) - - sparkApplicationData.applicationInfo.id should be("application_1457600942802_0093") - sparkApplicationData.applicationInfo.name should be("PythonPi") - sparkApplicationData.jobDatas.size should be (1) - sparkApplicationData.stageDatas.size should be (1) - sparkApplicationData.executorSummaries.size should be(3) - sparkApplicationData.appConfigurationProperties.size should be(6) - sparkApplicationData.jobDatas(0).jobId should be(0) - sparkApplicationData.jobDatas(0).numCompletedTasks should be(10) - sparkApplicationData.jobDatas(0).numCompletedStages should be(1) - sparkApplicationData.jobDatas(0).status should be(JobExecutionStatus.SUCCEEDED) - sparkApplicationData.stageDatas(0).stageId should be(0) - sparkApplicationData.stageDatas(0).status should be(StageStatus.COMPLETE) - sparkApplicationData.stageDatas(0).numCompleteTasks should be(10) - sparkApplicationData.stageDatas(0).executorRunTime should be(2470) - sparkApplicationData.stageDatas(0).name should be("reduce at pi.py:39") - sparkApplicationData.executorSummaries(0).id should be("1") - sparkApplicationData.executorSummaries(1).id should be("2") - sparkApplicationData.executorSummaries(2).id should be("driver") - sparkApplicationData.executorSummaries(0).hostPort should be(".hello.com:38464") - sparkApplicationData.executorSummaries(1).hostPort should be(".hello.com:36478") - sparkApplicationData.executorSummaries(2).hostPort should be("10.20.0.71:58838") - sparkApplicationData.executorSummaries(0).maxMemory should be(2223023063L) - sparkApplicationData.executorSummaries(1).maxMemory should be(2223023063L) - sparkApplicationData.executorSummaries(2).maxMemory should be(1111794647L) - sparkApplicationData.executorSummaries(0).totalTasks should be(5) - sparkApplicationData.executorSummaries(1).totalTasks should be(5) - sparkApplicationData.executorSummaries(2).totalTasks should be(0) - sparkApplicationData.appConfigurationProperties should be(EXPECTED_PROPERTIES_FROM_LOG_1) - } - - it("throws RunTimeException when eventlog name ends with .inprogress") { - import ExecutionContext.Implicits.global - val fakeJerseyServer = new FakeJerseyServer() { - override def configure(): Application = super.configure() match { - case resourceConfig: ResourceConfig => - resourceConfig - .register(classOf[FetchClusterModeDataFixtures.ApiResource]) - .register(classOf[FetchClusterModeDataFixtures.LogsResource]) - case config => config - } - } - fakeJerseyServer.setUp() - - val historyServerUri = fakeJerseyServer.target.getUri - - val sparkConf = new SparkConf().set("spark.yarn.historyServer.address", s"${historyServerUri.getHost}:${historyServerUri.getPort}") - val sparkRestClient = new SparkRestClient(sparkConf) { - override def getApplicationLogs(logTarget: WebTarget): ZipInputStream = { - new ZipInputStream(newFakeLog(FetchClusterModeDataFixtures.APP_ID, None, ".inprogress")) - } - } - - val job = new AnalyticJob().setAppId(FetchClusterModeDataFixtures.APP_ID) - val thrown = the[RuntimeException] thrownBy(sparkRestClient.fetchEventLogAndParse(job)) - thrown.getMessage should be (s"Application for the log application_1.lzf.inprogress has not finished yet.") - } - - it("returns the desired data from the Spark REST API for client mode application") { - import ExecutionContext.Implicits.global - val fakeJerseyServer = new FakeJerseyServer() { - override def configure(): Application = super.configure() match { - case resourceConfig: ResourceConfig => - resourceConfig - .register(classOf[FetchClientModeDataFixtures.ApiResource]) - .register(classOf[FetchClientModeDataFixtures.ApplicationResource]) - .register(classOf[FetchClientModeDataFixtures.JobsResource]) - .register(classOf[FetchClientModeDataFixtures.StagesResource]) - .register(classOf[FetchClientModeDataFixtures.ExecutorsResource]) - .register(classOf[FetchClientModeDataFixtures.LogsResource]) - case config => config - } - } - - fakeJerseyServer.setUp() - - val historyServerUri = fakeJerseyServer.target.getUri - - val sparkConf = new SparkConf().set("spark.yarn.historyServer.address", s"${historyServerUri.getHost}:${historyServerUri.getPort}") - val sparkRestClient = new SparkRestClient(sparkConf) - - sparkRestClient.fetchData(FetchClusterModeDataFixtures.APP_ID) map { restDerivedData => - restDerivedData.applicationInfo.id should be(FetchClusterModeDataFixtures.APP_ID) - restDerivedData.applicationInfo.name should be(FetchClusterModeDataFixtures.APP_NAME) - restDerivedData.jobDatas should not be(None) - restDerivedData.stageDatas should not be(None) - restDerivedData.executorSummaries should not be(None) - restDerivedData.logDerivedData should be(None) - } flatMap { - case assertion: Try[Assertion] => assertion - case _ => - sparkRestClient.fetchData(FetchClientModeDataFixtures.APP_ID, fetchLogs = true) - .map { _.logDerivedData.get.appConfigurationProperties should be(EXPECTED_PROPERTIES_FROM_LOG_1) } - } andThen { case assertion: Try[Assertion] => - fakeJerseyServer.tearDown() - assertion - } - } - - it("returns the desired data from the Spark REST API for cluster mode application when http in jobhistory address") { - import ExecutionContext.Implicits.global - val fakeJerseyServer = new FakeJerseyServer() { - override def configure(): Application = super.configure() match { - case resourceConfig: ResourceConfig => - resourceConfig - .register(classOf[FetchClusterModeDataFixtures.ApiResource]) - .register(classOf[FetchClusterModeDataFixtures.ApplicationResource]) - .register(classOf[FetchClusterModeDataFixtures.JobsResource]) - .register(classOf[FetchClusterModeDataFixtures.StagesResource]) - .register(classOf[FetchClusterModeDataFixtures.ExecutorsResource]) - case config => config - } - } - - fakeJerseyServer.setUp() - - val historyServerUri = fakeJerseyServer.target.getUri - - val sparkConf = new SparkConf().set("spark.yarn.historyServer.address", s"http://${historyServerUri.getHost}:${historyServerUri.getPort}") - val sparkRestClient = new SparkRestClient(sparkConf) - - sparkRestClient.fetchData(FetchClusterModeDataFixtures.APP_ID) map { restDerivedData => - restDerivedData.applicationInfo.id should be(FetchClusterModeDataFixtures.APP_ID) - restDerivedData.applicationInfo.name should be(FetchClusterModeDataFixtures.APP_NAME) - restDerivedData.jobDatas should not be(None) - restDerivedData.stageDatas should not be(None) - restDerivedData.executorSummaries should not be(None) - } andThen { case assertion: Try[Assertion] => - fakeJerseyServer.tearDown() - assertion - } - } - - it("throws an exception if spark.yarn.historyServer.address is missing") { - an[IllegalArgumentException] should be thrownBy(new SparkRestClient(new SparkConf())) - } - - it("handles unrecognized fields gracefully when parsing") { - val objectMapper = SparkRestClient.SparkRestObjectMapper - val json = s"""{ - "startTime" : "2016-09-12T19:30:18.101GMT", - "endTime" : "1969-12-31T23:59:59.999GMT", - "sparkUser" : "foo", - "completed" : false, - "unrecognized" : "bar" - }""" - - val applicationAttemptInfo = objectMapper.readValue[ApplicationAttemptInfoImpl](json) - applicationAttemptInfo.sparkUser should be("foo") - } - - it("returns the desired list of applications using Spark REST API") { - import ExecutionContext.Implicits.global - val fakeJerseyServer = new FakeJerseyServer() { - override def configure(): Application = super.configure() match { - case resourceConfig: ResourceConfig => - resourceConfig - .register(classOf[FetchClusterModeDataFixtures.ApiResource]) - .register(classOf[FetchClusterModeDataFixtures.ApplicationsResource]) - case config => config - } - } - fakeJerseyServer.setUp() - - val historyServerUri = fakeJerseyServer.target.getUri - - val sparkConf = new SparkConf().set( - "spark.yarn.historyServer.address", s"${historyServerUri.getHost}:${historyServerUri.getPort}") - val sparkRestClient = new SparkRestClient(sparkConf) - val startTs = System.currentTimeMillis - 200000 - val endTs = System.currentTimeMillis + 120000 - val sparkApplications = sparkRestClient.fetchCompletedApplicationsData(startTs, endTs) - - sparkApplications.size should be(1) - sparkApplications.head.id should be(FetchClusterModeDataFixtures.APP_ID) - sparkApplications.head.name should be(FetchClusterModeDataFixtures.APP_NAME) - sparkApplications.head.attempts.size should be(2) - } - } -} - -object SparkRestClientTest { - class FakeJerseyServer extends JerseyTest { - override def configure(): Application = { - forceSet(TestProperties.CONTAINER_PORT, "0") - enable(TestProperties.LOG_TRAFFIC) - enable(TestProperties.DUMP_ENTITY) - - new ResourceConfig() - .register(classOf[FakeJerseyObjectMapperProvider]) - } - - override def configureClient(clientConfig: ClientConfig): Unit = { - clientConfig.register(classOf[FakeJerseyObjectMapperProvider]) - } - } - - class FakeJerseyObjectMapperProvider extends ContextResolver[ObjectMapper] { - lazy val objectMapper = { - val objectMapper = new ObjectMapper() - objectMapper.registerModule(DefaultScalaModule) - objectMapper.setDateFormat(dateFormat) - objectMapper - } - - lazy val dateFormat = { - val iso8601 = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'GMT'") - val cal = Calendar.getInstance(new SimpleTimeZone(0, "GMT")) - iso8601.setCalendar(cal) - iso8601 - } - - override def getContext(cls: Class[_]): ObjectMapper = objectMapper - } - - object FetchClusterModeDataFixtures { - val APP_ID = "application_1" - val APP_NAME = "app" - - @Path("/api/v1") - class ApiResource { - @Path("applications") - def getApplications(): ApplicationsResource = new ApplicationsResource() - - @Path("applications/{appId}") - def getApplication(): ApplicationResource = new ApplicationResource() - - @Path("applications/{appId}/{attemptId}/jobs") - def getJobs(): JobsResource = new JobsResource() - - @Path("applications/{appId}/{attemptId}/stages") - def getStages(): StagesResource = new StagesResource() - - @Path("applications/{appId}/{attemptId}/executors") - def getExecutors(): ExecutorsResource = new ExecutorsResource() - - @Path("applications/{appId}/{attemptId}/logs") - def getLogs(): LogsResource = new LogsResource() - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class ApplicationResource { - @GET - def getApplication(@PathParam("appId") appId: String): ApplicationInfoImpl = { - val t2 = System.currentTimeMillis - val t1 = t2 - 1 - val duration = 8000000L - new ApplicationInfoImpl( - APP_ID, - APP_NAME, - Seq( - newFakeApplicationAttemptInfo(Some("2"), startTime = new Date(t2 - duration), endTime = new Date(t2)), - newFakeApplicationAttemptInfo(Some("1"), startTime = new Date(t1 - duration), endTime = new Date(t1)) - ) - ) - } - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class ApplicationsResource { - @GET - def getApplications(): Seq[ApplicationInfoImpl] = { - val t2 = System.currentTimeMillis - val t1 = t2 + 1 - val duration = 8000000L - Seq( - new ApplicationInfoImpl( - APP_ID, - APP_NAME, - Seq( - newFakeApplicationAttemptInfo(Some("2"), startTime = new Date(t2 - duration), endTime = new Date(t2)), - newFakeApplicationAttemptInfo(Some("1"), startTime = new Date(t1 - duration), endTime = new Date(t1)) - ) - ) - ) - } - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class JobsResource { - @GET - def getJobs(@PathParam("appId") appId: String, @PathParam("attemptId") attemptId: String): Seq[JobDataImpl] = - if (attemptId == "2") Seq.empty else throw new Exception() - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class StagesResource { - @GET - def getStages(@PathParam("appId") appId: String, @PathParam("attemptId") attemptId: String): Seq[StageDataImpl] = - if (attemptId == "2") Seq.empty else throw new Exception() - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class ExecutorsResource { - @GET - def getExecutors(@PathParam("appId") appId: String, @PathParam("attemptId") attemptId: String): Seq[ExecutorSummaryImpl] = - if (attemptId == "2") Seq.empty else throw new Exception() - } - - @Produces(Array(MediaType.APPLICATION_OCTET_STREAM)) - class LogsResource { - @GET - def getLogs(@PathParam("appId") appId: String, @PathParam("attemptId") attemptId: String): Response = { - if (attemptId == "2") { - Response.ok(newFakeLog(appId, Some(attemptId))).build() - } else throw new Exception() - } - } - } - - object FetchClientModeDataFixtures { - val APP_ID = "application_1" - val APP_NAME = "app" - - @Path("/api/v1") - class ApiResource { - @Path("applications/{appId}") - def getApplication(): ApplicationResource = new ApplicationResource() - - @Path("applications/{appId}/jobs") - def getJobs(): JobsResource = new JobsResource() - - @Path("applications/{appId}/stages") - def getStages(): StagesResource = new StagesResource() - - @Path("applications/{appId}/executors") - def getExecutors(): ExecutorsResource = new ExecutorsResource() - - @Path("applications/{appId}/logs") - def getLogs(): LogsResource = new LogsResource() - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class ApplicationResource { - @GET - def getApplication(@PathParam("appId") appId: String): ApplicationInfoImpl = { - val t2 = System.currentTimeMillis - val t1 = t2 - 1 - val duration = 8000000L - new ApplicationInfoImpl( - APP_ID, - APP_NAME, - Seq( - newFakeApplicationAttemptInfo(None, startTime = new Date(t2 - duration), endTime = new Date(t2)), - newFakeApplicationAttemptInfo(None, startTime = new Date(t1 - duration), endTime = new Date(t1)) - ) - ) - } - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class JobsResource { - @GET - def getJobs(@PathParam("appId") appId: String): Seq[JobDataImpl] = - Seq.empty - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class StagesResource { - @GET - def getStages(@PathParam("appId") appId: String): Seq[StageDataImpl] = - Seq.empty - } - - @Produces(Array(MediaType.APPLICATION_JSON)) - class ExecutorsResource { - @GET - def getExecutors(@PathParam("appId") appId: String): Seq[ExecutorSummaryImpl] = - Seq.empty - } - - @Produces(Array(MediaType.APPLICATION_OCTET_STREAM)) - class LogsResource { - @GET - def getLogs(@PathParam("appId") appId: String): Response = { - Response.ok(newFakeLog(appId, None)).build() - } - } - } - - def newFakeApplicationAttemptInfo( - attemptId: Option[String], - startTime: Date, - endTime: Date - ): ApplicationAttemptInfoImpl = new ApplicationAttemptInfoImpl( - attemptId, - startTime, - endTime, - sparkUser = "foo", - completed = true - ) - - private val EVENT_LOG_1 = Resources.toByteArray( - Resources.getResource("spark_event_logs/event_log_1")) - - private val EXPECTED_PROPERTIES_FROM_LOG_1 = Map( - "spark.serializer" -> "org.apache.spark.serializer.KryoSerializer", - "spark.storage.memoryFraction" -> "0.3", - "spark.driver.memory" -> "2G", - "spark.executor.instances" -> "900", - "spark.executor.memory" -> "1g", - "spark.shuffle.memoryFraction" -> "0.5" - ) - - def newFakeLog(appId: String, attemptId: Option[String], inProgress: String = ""): InputStream = { - val os = new ByteArrayOutputStream() - val zos = new ZipOutputStream(os) - val name = attemptId.map(id => s"${appId}_$id").getOrElse(appId) + ".lzf" + inProgress - zos.putNextEntry(new ZipEntry(name)) - // LZFEncoder instead of Snappy, because of xerial/snappy-java#76. - zos.write(LZFEncoder.encode(EVENT_LOG_1)) - zos.closeEntry() - zos.close() - - new ByteArrayInputStream(os.toByteArray) - } -} diff --git a/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala deleted file mode 100644 index f61d741e2..000000000 --- a/test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.heuristics - -import com.linkedin.drelephant.spark.data.SparkRestDerivedData -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationAttemptInfoImpl, ApplicationInfoImpl} -import scala.collection.JavaConverters - -import com.linkedin.drelephant.analysis.{ApplicationType, Severity} -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData -import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData} -import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate -import org.scalatest.{FunSpec, Matchers} -import java.util.Date - - -class ConfigurationHeuristicTest extends FunSpec with Matchers { - import ConfigurationHeuristicTest._ - - describe("ConfigurationHeuristic") { - val heuristicConfigurationData = newFakeHeuristicConfigurationData( - Map( - "serializer_if_non_null_recommendation" -> "org.apache.spark.serializer.KryoSerializer", - "shuffle_manager_if_non_null_recommendation" -> "sort" - ) - ) - - val configurationHeuristic = new ConfigurationHeuristic(heuristicConfigurationData) - - describe("apply with NO Severity") { - val configurationProperties = Map( - "spark.serializer" -> "org.apache.spark.serializer.KryoSerializer", - "spark.storage.memoryFraction" -> "0.3", - "spark.driver.memory" -> "2G", - "spark.executor.instances" -> "900", - "spark.executor.memory" -> "1g", - "spark.shuffle.memoryFraction" -> "0.5", - "spark.shuffle.service.enabled" -> "true", - "spark.dynamicAllocation.enabled" -> "true", - "spark.yarn.secondary.jars" -> "something without star", - "spark.yarn.driver.memoryOverhead" -> "500" - ) - - val data = newFakeSparkApplicationData(configurationProperties) - val heuristicResult = configurationHeuristic.apply(data) - val heuristicResultDetails = heuristicResult.getHeuristicResultDetails - - it("returns the size of result details") { - heuristicResultDetails.size() should be(9) - } - - it("returns the severity") { - heuristicResult.getSeverity should be(Severity.NONE) - } - - it("returns the driver memory") { - val details = heuristicResultDetails.get(0) - details.getName should include("spark.driver.memory") - details.getValue should be("2 GB") - } - - it("returns the executor memory") { - val details = heuristicResultDetails.get(1) - details.getName should include("spark.executor.memory") - details.getValue should be("1 GB") - } - - it("returns the executor instances") { - val details = heuristicResultDetails.get(2) - details.getName should include("spark.executor.instances") - details.getValue should be("900") - } - - it("returns the executor cores") { - val details = heuristicResultDetails.get(3) - details.getName should include("spark.executor.cores") - details.getValue should include("default") - } - - it("returns the application duration") { - val details = heuristicResultDetails.get(4) - details.getName should include("spark.application.duration") - details.getValue should include("10") - } - - it("returns the dynamic allocation flag") { - val details = heuristicResultDetails.get(5) - details.getName should include("spark.dynamicAllocation.enabled") - details.getValue should be("true") - } - - it("returns the driver cores") { - val details = heuristicResultDetails.get(6) - details.getName should include("spark.driver.cores") - details.getValue should include("default") - } - - it("returns the driver overhead memory") { - val details = heuristicResultDetails.get(7) - details.getName should include("spark.yarn.driver.memoryOverhead") - details.getValue should include("500 MB") - } - } - - describe("apply with Severity") { - val configurationProperties = Map( - "spark.serializer" -> "dummySerializer", - "spark.shuffle.service.enabled" -> "false", - "spark.dynamicAllocation.enabled" -> "true" - ) - - val data = newFakeSparkApplicationData(configurationProperties) - val heuristicResult = configurationHeuristic.apply(data) - val heuristicResultDetails = heuristicResult.getHeuristicResultDetails - - it("returns the size of result details") { - heuristicResultDetails.size() should be(11) - } - - it("returns the severity") { - heuristicResult.getSeverity should be(Severity.SEVERE) - } - - it("returns the dynamic allocation flag") { - val details = heuristicResultDetails.get(5) - details.getName should include("spark.dynamicAllocation.enabled") - details.getValue should be("true") - } - - it("returns the serializer") { - val details = heuristicResultDetails.get(9) - details.getName should include("spark.serializer") - details.getValue should be("dummySerializer") - details.getDetails should be("KyroSerializer is Not Enabled.") - } - - it("returns the shuffle service flag") { - val details = heuristicResultDetails.get(10) - details.getName should include("spark.shuffle.service.enabled") - details.getValue should be("false") - details.getDetails should be("Spark shuffle service is not enabled.") - } - } - - describe(".Evaluator") { - import ConfigurationHeuristic.Evaluator - - def newEvaluatorWithConfigurationProperties(configurationProperties: Map[String, String]): Evaluator = { - new Evaluator(configurationHeuristic, newFakeSparkApplicationData(configurationProperties)) - } - - it("has the driver memory bytes when they're present") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.driver.memory" -> "2G")) - evaluator.driverMemoryBytes should be(Some(2L * 1024 * 1024 * 1024)) - } - - it("has no driver memory bytes when they're absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) - evaluator.driverMemoryBytes should be(None) - } - - it("has the executor memory bytes when they're present") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.executor.memory" -> "1g")) - evaluator.executorMemoryBytes should be(Some(1L * 1024 * 1024 * 1024)) - } - - it("has no executor memory bytes when they're absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) - evaluator.executorMemoryBytes should be(None) - } - - it("has the executor instances when they're present") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.executor.instances" -> "900")) - evaluator.executorInstances should be(Some(900)) - } - - it("has no executor instances when they're absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) - evaluator.executorInstances should be(None) - } - - it("has the executor cores when they're present") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.executor.cores" -> "2")) - evaluator.executorCores should be(Some(2)) - } - - it("has the driver cores when they're present") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.driver.cores" -> "3")) - evaluator.driverCores should be(Some(3)) - } - - it("has no executor cores when they're absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) - evaluator.executorCores should be(None) - } - - it("has no driver cores when they're absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) - evaluator.driverCores should be(None) - } - - it("has the serializer when it's present") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer")) - evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer")) - } - - it("jars severity when NONE") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.yarn.secondary.jars" -> "somethingWithoutStar")) - evaluator.jarsSeverity should be(Severity.NONE) - } - - it("jars severity when CRITICAL") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.yarn.secondary.jars" -> "somethingWith*.jar")) - evaluator.jarsSeverity should be(Severity.CRITICAL) - } - - it("has no serializer, dynamic allocation flag, and shuffle flag when they are absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map.empty) - evaluator.serializer should be(None) - evaluator.isDynamicAllocationEnabled should be(Some(false)) - evaluator.isShuffleServiceEnabled should be(Some(false)) - evaluator.serializerSeverity should be(Severity.MODERATE) - evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.MODERATE) - evaluator.severity should be(Severity.MODERATE) - } - - it("has no dynamic allocation flag and shuffle flag, and serializer setting matches our recommendation") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer")) - evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer")) - evaluator.isDynamicAllocationEnabled should be(Some(false)) - evaluator.isShuffleServiceEnabled should be(Some(false)) - evaluator.serializerSeverity should be(Severity.NONE) - evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.MODERATE) - evaluator.severity should be(Severity.MODERATE) - } - - it("has no dynamic allocation flag and shuffle flag, and serializer setting doesn't match our recommendation and is non-null") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.FooSerializer")) - evaluator.serializer should be(Some("org.apache.spark.serializer.FooSerializer")) - evaluator.isDynamicAllocationEnabled should be(Some(false)) - evaluator.isShuffleServiceEnabled should be(Some(false)) - evaluator.serializerSeverity should be(Severity.MODERATE) - evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.MODERATE) - evaluator.severity should be(Severity.MODERATE) - } - - it("true dynamic allocation flag and shuffle flag, and serializer setting matches our recommendation") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer", - "spark.shuffle.service.enabled" -> "true", "spark.dynamicAllocation.enabled" -> "true")) - evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer")) - evaluator.isDynamicAllocationEnabled should be(Some(true)) - evaluator.isShuffleServiceEnabled should be(Some(true)) - evaluator.serializerSeverity should be(Severity.NONE) - evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.NONE) - evaluator.severity should be(Severity.NONE) - } - - it("true dynamic allocation flag and shuffle flag, and serializer setting is absent") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.shuffle.service.enabled" -> "true", - "spark.dynamicAllocation.enabled" -> "true")) - evaluator.serializer should be(None) - evaluator.isDynamicAllocationEnabled should be(Some(true)) - evaluator.isShuffleServiceEnabled should be(Some(true)) - evaluator.serializerSeverity should be(Severity.MODERATE) - evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.NONE) - evaluator.severity should be(Severity.MODERATE) - } - - it("true dynamic allocation flag and false shuffle flag, and serializer setting matches our recommendation") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer", - "spark.shuffle.service.enabled" -> "false", "spark.dynamicAllocation.enabled" -> "true")) - evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer")) - evaluator.isDynamicAllocationEnabled should be(Some(true)) - evaluator.isShuffleServiceEnabled should be(Some(false)) - evaluator.serializerSeverity should be(Severity.NONE) - evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.SEVERE) - evaluator.severityConfThresholds should be(Severity.NONE) - evaluator.severity should be(Severity.SEVERE) - } - - it("false dynamic allocation flag and shuffle flag, and serializer setting matches our recommendation") { - val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer", - "spark.shuffle.service.enabled" -> "false", "spark.dynamicAllocation.enabled" -> "false")) - evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer")) - evaluator.isDynamicAllocationEnabled should be(Some(false)) - evaluator.isShuffleServiceEnabled should be(Some(false)) - evaluator.serializerSeverity should be(Severity.NONE) - evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.MODERATE) - evaluator.severity should be(Severity.MODERATE) - } - } - } -} - -object ConfigurationHeuristicTest { - import JavaConverters._ - - def newFakeHeuristicConfigurationData(params: Map[String, String] = Map.empty): HeuristicConfigurationData = - new HeuristicConfigurationData("heuristic", "class", "view", new ApplicationType("type"), params.asJava) - - def newFakeSparkApplicationData(appConfigurationProperties: Map[String, String]): SparkApplicationData = { - val logDerivedData = SparkLogDerivedData( - SparkListenerEnvironmentUpdate(Map("Spark Properties" -> appConfigurationProperties.toSeq)) - ) - - val appId = "application_1" - val startDate = new Date() - val endDate = new Date(startDate.getTime() + 10000) - val applicationAttempt = new ApplicationAttemptInfoImpl(Option("attempt1"),startDate, endDate, "sparkUser") - val applicationAttempts = Seq(applicationAttempt) - - val restDerivedData = SparkRestDerivedData( - new ApplicationInfoImpl(appId, name = "app", applicationAttempts), - jobDatas = Seq.empty, - stageDatas = Seq.empty, - executorSummaries = Seq.empty - ) - - SparkApplicationData(appId, restDerivedData, Some(logDerivedData)) - } -} diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala deleted file mode 100644 index bd1ba9864..000000000 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristicTest.scala +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.heuristics - -import scala.collection.JavaConverters -import com.linkedin.drelephant.analysis.{ApplicationType, Severity} -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData -import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkRestDerivedData} -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl} -import org.scalatest.{FunSpec, Matchers} - -import scala.concurrent.duration.Duration - - -class ExecutorGcHeuristicTest extends FunSpec with Matchers { - import ExecutorGcHeuristicTest._ - - describe("ExecutorGcHeuristic") { - val heuristicConfigurationData = newFakeHeuristicConfigurationData( - Map( - "max_to_median_ratio_severity_thresholds" -> "1.414,2,4,16", - "ignore_max_bytes_less_than_threshold" -> "4000000", - "ignore_max_millis_less_than_threshold" -> "4000001", - ExecutorGcHeuristic.GC_SEVERITY_D_THRESHOLDS_KEY -> "0.5,0.3,0.2,0.1" - ) - ) - val executorGcHeuristic = new ExecutorGcHeuristic(heuristicConfigurationData) - - val executorSummaries = Seq( - newFakeExecutorSummary( - id = "1", - totalGCTime = Duration("2min").toMillis, - totalDuration = Duration("15min").toMillis - ), - newFakeExecutorSummary( - id = "2", - totalGCTime = Duration("6min").toMillis, - totalDuration = Duration("14min").toMillis - ), - newFakeExecutorSummary( - id = "3", - totalGCTime = Duration("4min").toMillis, - totalDuration = Duration("20min").toMillis - ), - newFakeExecutorSummary( - id = "4", - totalGCTime = Duration("8min").toMillis, - totalDuration = Duration("30min").toMillis - ) - ) - - describe(".apply") { - val data1 = newFakeSparkApplicationData(executorSummaries) - val heuristicResult = executorGcHeuristic.apply(data1) - val heuristicResultDetails = heuristicResult.getHeuristicResultDetails - - it("returns the severity") { - heuristicResult.getSeverity should be(Severity.CRITICAL) - } - - it("returns the JVM GC time to Executor Run time duration") { - val details = heuristicResultDetails.get(0) - details.getName should include("GC time to Executor Run time ratio") - details.getValue should include("0.2531") - } - - it("returns the total GC time") { - val details = heuristicResultDetails.get(1) - details.getName should include("Total GC time") - details.getValue should be("1200000") - } - - it("returns the executor's run time") { - val details = heuristicResultDetails.get(2) - details.getName should include("Total Executor Runtime") - details.getValue should be("4740000") - } - - it("return Gc ratio low") { - val details = heuristicResultDetails.get(4) - details.getName should include("Gc ratio low") - } - } - } -} - -object ExecutorGcHeuristicTest { - import JavaConverters._ - - def newFakeHeuristicConfigurationData(params: Map[String, String] = Map.empty): HeuristicConfigurationData = - new HeuristicConfigurationData("heuristic", "class", "view", new ApplicationType("type"), params.asJava) - - def newFakeExecutorSummary( - id: String, - totalGCTime: Long, - totalDuration: Long - ): ExecutorSummaryImpl = new ExecutorSummaryImpl( - id, - hostPort = "", - rddBlocks = 0, - memoryUsed=0, - diskUsed = 0, - activeTasks = 0, - failedTasks = 0, - completedTasks = 0, - totalTasks = 0, - totalDuration, - totalInputBytes=0, - totalShuffleRead=0, - totalShuffleWrite= 0, - maxMemory= 0, - totalGCTime, - executorLogs = Map.empty - ) - - def newFakeSparkApplicationData( - executorSummaries: Seq[ExecutorSummaryImpl] - ): SparkApplicationData = { - val appId = "application_1" - - val restDerivedData = SparkRestDerivedData( - new ApplicationInfoImpl(appId, name = "app", Seq.empty), - jobDatas = Seq.empty, - stageDatas = Seq.empty, - executorSummaries = executorSummaries - ) - SparkApplicationData(appId, restDerivedData, None) - } -} diff --git a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala deleted file mode 100644 index 7dbeea921..000000000 --- a/test/com/linkedin/drelephant/spark/heuristics/ExecutorsHeuristicTest.scala +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.heuristics - -import scala.collection.JavaConverters - -import com.linkedin.drelephant.analysis.{ApplicationType, Severity, SeverityThresholds} -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData -import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, ExecutorSummaryImpl} -import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate -import org.scalatest.{FunSpec, Matchers} - - -class ExecutorsHeuristicTest extends FunSpec with Matchers { - import ExecutorsHeuristicTest._ - - describe("ExecutorsHeuristic") { - val heuristicConfigurationData = newFakeHeuristicConfigurationData( - Map( - "max_to_median_ratio_severity_thresholds" -> "1.414,2,4,16", - "ignore_max_bytes_less_than_threshold" -> "4000000", - "ignore_max_millis_less_than_threshold" -> "4000001" - ) - ) - val executorsHeuristic = new ExecutorsHeuristic(heuristicConfigurationData) - - val maxMemory = 5000000L - - val executorSummaries = Seq( - newFakeExecutorSummary( - id = "1", - memoryUsed = 1000000L, - totalDuration = 1000001L, - totalInputBytes = 1000002L, - totalShuffleRead = 1000003L, - totalShuffleWrite = 1000004L, - maxMemory - ), - newFakeExecutorSummary( - id = "2", - memoryUsed = 2000000L, - totalDuration = 2000001L, - totalInputBytes = 2000002L, - totalShuffleRead = 2000003L, - totalShuffleWrite = 2000004L, - maxMemory - ), - newFakeExecutorSummary( - id = "3", - memoryUsed = 3000000L, - totalDuration = 3000001L, - totalInputBytes = 3000002L, - totalShuffleRead = 3000003L, - totalShuffleWrite = 3000004L, - maxMemory - ), - newFakeExecutorSummary( - id = "4", - memoryUsed = 4000000L, - totalDuration = 4000001L, - totalInputBytes = 4000002L, - totalShuffleRead = 4000003L, - totalShuffleWrite = 4000004L, - maxMemory - ) - ) - - describe(".apply") { - val data = newFakeSparkApplicationData(executorSummaries) - val heuristicResult = executorsHeuristic.apply(data) - val heuristicResultDetails = heuristicResult.getHeuristicResultDetails - - it("returns the severity") { - heuristicResult.getSeverity should be(Severity.LOW) - } - - it("returns the total storage memory allocated") { - val details = heuristicResultDetails.get(0) - details.getName should include("storage memory allocated") - details.getValue should be("19.07 MB") - } - - it("returns the total storage memory used") { - val details = heuristicResultDetails.get(1) - details.getName should include("storage memory used") - details.getValue should be("9.54 MB") - } - - it("returns the storage memory utilization rate") { - val details = heuristicResultDetails.get(2) - details.getName should include("storage memory utilization rate") - details.getValue should be("0.500") - } - - it("returns the distribution of storage memory used among executors") { - val details = heuristicResultDetails.get(3) - details.getName should include("storage memory used") - details.getValue should include regex("976.56 KB.*976.56 KB.*2.38 MB.*2.86 MB.*3.81 MB") - } - - it("returns the distribution of task time among executors") { - val details = heuristicResultDetails.get(4) - details.getName should include("task time") - details.getValue should include regex("16 min 40 sec.*16 min 40 sec.*41 min 40 sec.*50 min.*1 hr 6 min 40 sec") - } - - it("returns the total sum of task time among executors") { - val details = heuristicResultDetails.get(5) - details.getName should include("task time sum") - details.getValue should include regex("10000") - } - - it("returns the distribution of input bytes among executors") { - val details = heuristicResultDetails.get(6) - details.getName should include("input bytes") - details.getValue should include regex("976.56 KB.*976.56 KB.*2.38 MB.*2.86 MB.*3.81 MB") - } - - it("returns the distribution of shuffle read bytes among executors") { - val details = heuristicResultDetails.get(7) - details.getName should include("shuffle read bytes") - details.getValue should include regex("976.57 KB.*976.57 KB.*2.38 MB.*2.86 MB.*3.81 MB") - } - - it("returns the distribution of shuffle write bytes among executors") { - val details = heuristicResultDetails.get(8) - details.getName should include("shuffle write bytes") - details.getValue should include regex("976.57 KB.*976.57 KB.*2.38 MB.*2.86 MB.*3.81 MB") - } - } - - describe(".Evaluator") { - import ExecutorsHeuristic.Evaluator - import ExecutorsHeuristic.Distribution - - val data = newFakeSparkApplicationData(executorSummaries) - val evaluator = new Evaluator(executorsHeuristic, data) - - it("has the total storage memory allocated") { - evaluator.totalStorageMemoryAllocated should be(20000000L) - } - - it("has the total storage memory used") { - evaluator.totalStorageMemoryUsed should be(10000000L) - } - - it("has the storage memory utilization rate") { - evaluator.storageMemoryUtilizationRate should be(0.5D) - } - - it("has the distribution of storage memory used among executors") { - evaluator.storageMemoryUsedDistribution should be( - Distribution(1000000L, 1000000L, 2500000L, 3000000L, 4000000L) - ) - } - - it("has the distribution of task time among executors") { - evaluator.taskTimeDistribution should be( - Distribution(1000001L, 1000001L, 2500001L, 3000001L, 4000001L) - ) - } - - it("has the distribution of input bytes among executors") { - evaluator.inputBytesDistribution should be( - Distribution(1000002L, 1000002L, 2500002L, 3000002L, 4000002L) - ) - } - - it("has the distribution of shuffle read among executors") { - evaluator.shuffleReadBytesDistribution should be( - Distribution(1000003L, 1000003L, 2500003L, 3000003L, 4000003L) - ) - } - - it("has the distribution of shuffle write among executors") { - evaluator.shuffleWriteBytesDistribution should be( - Distribution(1000004L, 1000004L, 2500004L, 3000004L, 4000004L) - ) - } - - it("computes the overall severity") { - evaluator.severity should be(Severity.LOW) - } - - it("computes the severity of a given distribution, when the max is large enough") { - val distribution = Distribution(min = 0L, p25 = 1000L, median = 1000L, p75 = 1000L, max = 16000L) - evaluator.severityOfDistribution(distribution, ignoreMaxLessThanThreshold = 16000L) should be(Severity.CRITICAL) - } - - it("computes the severity of a given distribution, when the max is not large enough") { - val distribution = Distribution(min = 0L, p25 = 1000L, median = 1000L, p75 = 1000L, max = 16000L) - evaluator.severityOfDistribution(distribution, ignoreMaxLessThanThreshold = 16001L) should be(Severity.NONE) - } - - it("computes the severity of a given distribution, when the median is zero and the max is large enough") { - val distribution = Distribution(min = 0L, p25 = 0L, median = 0L, p75 = 0L, max = 16000L) - evaluator.severityOfDistribution(distribution, ignoreMaxLessThanThreshold = 16000L) should be(Severity.CRITICAL) - } - - it("computes the severity of a given distribution, when the median is zero and the max is not large enough") { - val distribution = Distribution(min = 0L, p25 = 0L, median = 0L, p75 = 0L, max = 16000L) - evaluator.severityOfDistribution(distribution, ignoreMaxLessThanThreshold = 16001L) should be(Severity.NONE) - } - } - } -} - -object ExecutorsHeuristicTest { - import JavaConverters._ - - def newFakeHeuristicConfigurationData(params: Map[String, String] = Map.empty): HeuristicConfigurationData = - new HeuristicConfigurationData("heuristic", "class", "view", new ApplicationType("type"), params.asJava) - - def newFakeExecutorSummary( - id: String, - memoryUsed: Long, - totalDuration: Long, - totalInputBytes: Long, - totalShuffleRead: Long, - totalShuffleWrite: Long, - maxMemory: Long - ): ExecutorSummaryImpl = new ExecutorSummaryImpl( - id, - hostPort = "", - rddBlocks = 0, - memoryUsed, - diskUsed = 0, - activeTasks = 0, - failedTasks = 0, - completedTasks = 0, - totalTasks = 0, - totalDuration, - totalInputBytes, - totalShuffleRead, - totalShuffleWrite, - maxMemory, - totalGCTime = 0, - executorLogs = Map.empty - ) - - def newFakeSparkApplicationData(executorSummaries: Seq[ExecutorSummaryImpl]): SparkApplicationData = { - val appId = "application_1" - - val restDerivedData = SparkRestDerivedData( - new ApplicationInfoImpl(appId, name = "app", Seq.empty), - jobDatas = Seq.empty, - stageDatas = Seq.empty, - executorSummaries = executorSummaries - ) - - SparkApplicationData(appId, restDerivedData, logDerivedData = None) - } -} diff --git a/test/com/linkedin/drelephant/spark/heuristics/JobsHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/JobsHeuristicTest.scala deleted file mode 100644 index 240f80d7d..000000000 --- a/test/com/linkedin/drelephant/spark/heuristics/JobsHeuristicTest.scala +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.heuristics - -import scala.collection.JavaConverters - -import com.linkedin.drelephant.analysis.{ApplicationType, Severity} -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData -import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, JobDataImpl} -import org.apache.spark.JobExecutionStatus -import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate -import org.scalatest.{FunSpec, Matchers} - - -class JobsHeuristicTest extends FunSpec with Matchers { - import JobsHeuristicTest._ - - describe("JobsHeuristic") { - val heuristicConfigurationData = newFakeHeuristicConfigurationData( - Map( - "job_failure_rate_severity_thresholds" -> "0.2,0.4,0.6,0.8", - "job_task_failure_rate_severity_thresholds" -> "0.2,0.4,0.6,0.8" - ) - ) - val jobsHeuristic = new JobsHeuristic(heuristicConfigurationData) - val jobDatas = Seq( - newFakeJobData(0, "foo", JobExecutionStatus.SUCCEEDED, numCompleteTasks = 10, numFailedTasks = 0), - newFakeJobData(1, "bar", JobExecutionStatus.SUCCEEDED, numCompleteTasks = 8, numFailedTasks = 2), - newFakeJobData(2, "baz", JobExecutionStatus.SUCCEEDED, numCompleteTasks = 6, numFailedTasks = 4), - newFakeJobData(3, "aaa", JobExecutionStatus.FAILED, numCompleteTasks = 4, numFailedTasks = 6), - newFakeJobData(4, "zzz", JobExecutionStatus.FAILED, numCompleteTasks = 2, numFailedTasks = 8) - ) - - describe(".apply") { - val data = newFakeSparkApplicationData(jobDatas) - val heuristicResult = jobsHeuristic.apply(data) - val heuristicResultDetails = heuristicResult.getHeuristicResultDetails - - it("returns the severity") { - heuristicResult.getSeverity should be(Severity.CRITICAL) - } - - it("returns the number of completed jobs") { - heuristicResultDetails.get(0).getValue should be("3") - } - - it("returns the number of failed jobs") { - heuristicResultDetails.get(1).getValue should be("2") - } - - it("returns the list of failed jobs") { - heuristicResultDetails.get(2).getValue should be( - s"""|job 3, aaa - |job 4, zzz""".stripMargin - ) - } - - it("returns the job failure rate") { - heuristicResultDetails.get(3).getValue should be("0.400") - } - - it("returns the list of jobs with high task failure rates") { - heuristicResultDetails.get(4).getValue should be( - s"""|job 3, aaa (task failure rate: 0.600) - |job 4, zzz (task failure rate: 0.800)""".stripMargin - ) - } - } - - describe(".Evaluator") { - import JobsHeuristic.Evaluator - - val data = newFakeSparkApplicationData(jobDatas) - val evaluator = new Evaluator(jobsHeuristic, data) - - it("has the number of completed jobs") { - evaluator.numCompletedJobs should be(3) - } - - it("has the number of failed jobs") { - evaluator.numFailedJobs should be(2) - } - - it("has the list of failed jobs") { - val jobIds = evaluator.failedJobs.map { _.jobId } - jobIds should contain theSameElementsInOrderAs(Seq(3, 4)) - } - - it("has the job failure rate") { - evaluator.jobFailureRate should be(Some(0.4D)) - } - - it("has the list of jobs with high task failure rates") { - val jobIdsAndTaskFailureRates = - evaluator.jobsWithHighTaskFailureRates.map { case (jobData, taskFailureRate) => (jobData.jobId, taskFailureRate) } - jobIdsAndTaskFailureRates should contain theSameElementsInOrderAs(Seq((3, 0.6D), (4, 0.8D))) - } - - it("computes the overall severity") { - evaluator.severity should be(Severity.CRITICAL) - } - } - } -} - -object JobsHeuristicTest { - import JavaConverters._ - - def newFakeHeuristicConfigurationData(params: Map[String, String] = Map.empty): HeuristicConfigurationData = - new HeuristicConfigurationData("heuristic", "class", "view", new ApplicationType("type"), params.asJava) - - def newFakeJobData( - jobId: Int, - name: String, - status: JobExecutionStatus, - numCompleteTasks: Int, - numFailedTasks: Int - ): JobDataImpl = new JobDataImpl( - jobId, - name, - description = None, - submissionTime = None, - completionTime = None, - stageIds = Seq.empty, - jobGroup = None, - status, - numTasks = numCompleteTasks + numFailedTasks, - numActiveTasks = 0, - numCompleteTasks, - numSkippedTasks = 0, - numFailedTasks, - numActiveStages = 0, - numCompletedStages = 0, - numSkippedStages = 0, - numFailedStages = 0 - ) - - def newFakeSparkApplicationData(jobDatas: Seq[JobDataImpl]): SparkApplicationData = { - val appId = "application_1" - - val restDerivedData = SparkRestDerivedData( - new ApplicationInfoImpl(appId, name = "app", Seq.empty), - jobDatas, - stageDatas = Seq.empty, - executorSummaries = Seq.empty - ) - - SparkApplicationData(appId, restDerivedData, logDerivedData = None) - } -} diff --git a/test/com/linkedin/drelephant/spark/heuristics/StagesHeuristicTest.scala b/test/com/linkedin/drelephant/spark/heuristics/StagesHeuristicTest.scala deleted file mode 100644 index 7e1d8acb1..000000000 --- a/test/com/linkedin/drelephant/spark/heuristics/StagesHeuristicTest.scala +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.heuristics - -import scala.collection.JavaConverters -import scala.concurrent.duration.Duration - -import com.linkedin.drelephant.analysis.{ApplicationType, Severity} -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData -import com.linkedin.drelephant.spark.data.{SparkApplicationData, SparkLogDerivedData, SparkRestDerivedData} -import com.linkedin.drelephant.spark.fetchers.statusapiv1.{ApplicationInfoImpl, JobDataImpl, StageDataImpl} -import com.linkedin.drelephant.spark.fetchers.statusapiv1.StageStatus -import org.apache.spark.scheduler.SparkListenerEnvironmentUpdate -import org.scalatest.{FunSpec, Matchers} - -class StagesHeuristicTest extends FunSpec with Matchers { - import StagesHeuristicTest._ - - describe("StagesHeuristic") { - val heuristicConfigurationData = newFakeHeuristicConfigurationData( - Map( - "stage_failure_rate_severity_thresholds" -> "0.2,0.4,0.6,0.8", - "stage_task_failure_rate_severity_thresholds" -> "0.2,0.4,0.6,0.8", - "stage_runtime_minutes_severity_thresholds" -> "15,30,45,60" - ) - ) - val stagesHeuristic = new StagesHeuristic(heuristicConfigurationData) - val stageDatas = Seq( - newFakeStageData(StageStatus.COMPLETE, 0, numCompleteTasks = 10, numFailedTasks = 0, executorRunTime = Duration("2min").toMillis, "foo"), - newFakeStageData(StageStatus.COMPLETE, 1, numCompleteTasks = 8, numFailedTasks = 2, executorRunTime = Duration("2min").toMillis, "bar"), - newFakeStageData(StageStatus.COMPLETE, 2, numCompleteTasks = 6, numFailedTasks = 4, executorRunTime = Duration("2min").toMillis, "baz"), - newFakeStageData(StageStatus.FAILED, 3, numCompleteTasks = 4, numFailedTasks = 6, executorRunTime = Duration("2min").toMillis, "aaa"), - newFakeStageData(StageStatus.FAILED, 4, numCompleteTasks = 2, numFailedTasks = 8, executorRunTime = Duration("2min").toMillis, "zzz"), - newFakeStageData(StageStatus.COMPLETE, 5, numCompleteTasks = 10, numFailedTasks = 0, executorRunTime = Duration("0min").toMillis, "bbb"), - newFakeStageData(StageStatus.COMPLETE, 6, numCompleteTasks = 10, numFailedTasks = 0, executorRunTime = Duration("30min").toMillis, "ccc"), - newFakeStageData(StageStatus.COMPLETE, 7, numCompleteTasks = 10, numFailedTasks = 0, executorRunTime = Duration("60min").toMillis, "ddd"), - newFakeStageData(StageStatus.COMPLETE, 8, numCompleteTasks = 10, numFailedTasks = 0, executorRunTime = Duration("90min").toMillis, "eee"), - newFakeStageData(StageStatus.COMPLETE, 9, numCompleteTasks = 10, numFailedTasks = 0, executorRunTime = Duration("120min").toMillis, "fff") - ) - - val appConfigurationProperties = Map("spark.executor.instances" -> "2") - - describe(".apply") { - val data = newFakeSparkApplicationData(stageDatas, appConfigurationProperties) - val heuristicResult = stagesHeuristic.apply(data) - val heuristicResultDetails = heuristicResult.getHeuristicResultDetails - - it("returns the severity") { - heuristicResult.getSeverity should be(Severity.CRITICAL) - } - - it("returns the number of completed stages") { - heuristicResultDetails.get(0).getValue should be("8") - } - - it("returns the number of failed stages") { - heuristicResultDetails.get(1).getValue should be("2") - } - - it("returns the stage failure rate") { - heuristicResultDetails.get(2).getValue should be("0.200") - } - - it("returns the list of stages with high task failure rates") { - heuristicResultDetails.get(3).getValue should be( - s"""|stage 3, attempt 0 (task failure rate: 0.600) - |stage 4, attempt 0 (task failure rate: 0.800)""".stripMargin - ) - } - - it("returns the list of stages with long runtimes") { - heuristicResultDetails.get(4).getValue should be( - s"""|stage 8, attempt 0 (runtime: 45 min) - |stage 9, attempt 0 (runtime: 1 hr)""".stripMargin - ) - } - } - - describe(".Evaluator") { - import StagesHeuristic.Evaluator - - val data = newFakeSparkApplicationData(stageDatas, appConfigurationProperties) - val evaluator = new Evaluator(stagesHeuristic, data) - - it("has the number of completed stages") { - evaluator.numCompletedStages should be(8) - } - - it("has the number of failed stages") { - evaluator.numFailedStages should be(2) - } - - it("has the stage failure rate") { - evaluator.stageFailureRate should be(Some(0.2D)) - } - - it("has the list of stages with high task failure rates") { - val stageIdsAndTaskFailureRates = - evaluator.stagesWithHighTaskFailureRates.map { case (stageData, taskFailureRate) => (stageData.stageId, taskFailureRate) } - stageIdsAndTaskFailureRates should contain theSameElementsInOrderAs(Seq((3, 0.6D), (4, 0.8D))) - } - - it("has the list of stages with long average executor runtimes") { - val stageIdsAndRuntimes = - evaluator.stagesWithLongAverageExecutorRuntimes.map { case (stageData, runtime) => (stageData.stageId, runtime) } - stageIdsAndRuntimes should contain theSameElementsInOrderAs( - Seq((8, Duration("45min").toMillis), (9, Duration("60min").toMillis)) - ) - } - - it("computes the overall severity") { - evaluator.severity should be(Severity.CRITICAL) - } - } - } -} - -object StagesHeuristicTest { - import JavaConverters._ - - def newFakeHeuristicConfigurationData(params: Map[String, String] = Map.empty): HeuristicConfigurationData = - new HeuristicConfigurationData("heuristic", "class", "view", new ApplicationType("type"), params.asJava) - - def newFakeStageData( - status: StageStatus, - stageId: Int, - numCompleteTasks: Int, - numFailedTasks: Int, - executorRunTime: Long, - name: String - ): StageDataImpl = new StageDataImpl( - status, - stageId, - attemptId = 0, - numActiveTasks = numCompleteTasks + numFailedTasks, - numCompleteTasks, - numFailedTasks, - executorRunTime, - inputBytes = 0, - inputRecords = 0, - outputBytes = 0, - outputRecords = 0, - shuffleReadBytes = 0, - shuffleReadRecords = 0, - shuffleWriteBytes = 0, - shuffleWriteRecords = 0, - memoryBytesSpilled = 0, - diskBytesSpilled = 0, - name, - details = "", - schedulingPool = "", - accumulatorUpdates = Seq.empty, - tasks = None, - executorSummary = None - ) - - def newFakeSparkApplicationData( - stageDatas: Seq[StageDataImpl], - appConfigurationProperties: Map[String, String] - ): SparkApplicationData = { - val appId = "application_1" - - val restDerivedData = SparkRestDerivedData( - new ApplicationInfoImpl(appId, name = "app", Seq.empty), - jobDatas = Seq.empty, - stageDatas = stageDatas, - executorSummaries = Seq.empty - ) - - val logDerivedData = SparkLogDerivedData( - SparkListenerEnvironmentUpdate(Map("Spark Properties" -> appConfigurationProperties.toSeq)) - ) - - SparkApplicationData(appId, restDerivedData, Some(logDerivedData)) - } -} diff --git a/test/com/linkedin/drelephant/spark/legacydata/LegacyDataConvertersTest.scala b/test/com/linkedin/drelephant/spark/legacydata/LegacyDataConvertersTest.scala deleted file mode 100644 index 43695509d..000000000 --- a/test/com/linkedin/drelephant/spark/legacydata/LegacyDataConvertersTest.scala +++ /dev/null @@ -1,319 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.legacydata - -import java.util.Date - -import org.apache.spark.JobExecutionStatus -import com.linkedin.drelephant.spark.fetchers.statusapiv1.StageStatus -import org.scalatest.{FunSpec, Matchers} - -class LegacyDataConvertersTest extends FunSpec with Matchers { - describe("LegacyDataConverters") { - describe(".convert") { - } - - describe(".extractAppConfigurationProperties") { - it("returns a Map of Spark properties extracted from the given legacy SparkApplicationData") { - val legacyData = new MockSparkApplicationData() { - val environmentData = { - val environmentData = new SparkEnvironmentData() - environmentData.addSparkProperty("a", "b") - environmentData.addSparkProperty("c", "d") - environmentData - } - - override def getEnvironmentData(): SparkEnvironmentData = environmentData - } - - val appConfigurationProperties = LegacyDataConverters.extractAppConfigurationProperties(legacyData) - appConfigurationProperties should contain theSameElementsAs Map("a" -> "b", "c" -> "d") - } - } - - describe(".extractApplicationInfo") { - it("returns an ApplicationInfo extracted from the given legacy SparkApplicationData") { - val legacyData = new MockSparkApplicationData() { - val generalData = { - val generalData = new SparkGeneralData() - generalData.setApplicationId("application_1") - generalData.setApplicationName("app") - generalData.setStartTime(1000L) - generalData.setEndTime(2000L) - generalData.setSparkUser("foo") - generalData - } - - override def getGeneralData(): SparkGeneralData = generalData - } - - val applicationInfo = LegacyDataConverters.extractApplicationInfo(legacyData) - applicationInfo.id should be("application_1") - applicationInfo.name should be("app") - applicationInfo.attempts.size should be(1) - - val applicationAttemptInfo = applicationInfo.attempts.last - applicationAttemptInfo.attemptId should be(Some("1")) - applicationAttemptInfo.startTime should be(new Date(1000L)) - applicationAttemptInfo.endTime should be(new Date(2000L)) - applicationAttemptInfo.sparkUser should be("foo") - applicationAttemptInfo.completed should be(true) - } - } - - describe(".extractJobDatas") { - it("returns JobDatas extracted from the given legacy SparkApplicationData") { - val legacyData = new MockSparkApplicationData() { - val jobProgressData = { - val jobProgressData = new SparkJobProgressData() - - val jobInfo1 = { - val jobInfo = new SparkJobProgressData.JobInfo() - jobInfo.jobId = 1 - - jobInfo.numTasks = 10 - jobInfo.numActiveTasks = 1 - jobInfo.numCompletedTasks = 2 - jobInfo.numSkippedTasks = 3 - jobInfo.numFailedTasks = 4 - - for (i <- 1 to 100) { jobInfo.stageIds.add(i) } - jobInfo.numActiveStages = 10 - for (i <- 1 to 20) { jobInfo.completedStageIndices.add(i) } - jobInfo.numSkippedStages = 30 - jobInfo.numFailedStages = 40 - - jobInfo - } - jobProgressData.addJobInfo(1, jobInfo1) - jobProgressData.addCompletedJob(1) - - val jobInfo2 = { - val jobInfo = new SparkJobProgressData.JobInfo() - jobInfo.jobId = 2 - jobInfo - } - jobProgressData.addJobInfo(2, jobInfo2) - jobProgressData.addFailedJob(2) - - jobProgressData - } - - override def getJobProgressData(): SparkJobProgressData = jobProgressData - } - - val jobDatas = LegacyDataConverters.extractJobDatas(legacyData) - jobDatas.size should be(2) - - val jobData1 = jobDatas(0) - jobData1.jobId should be(1) - jobData1.name should be("1") - jobData1.description should be(None) - jobData1.submissionTime should be(None) - jobData1.completionTime should be(None) - jobData1.stageIds should be((1 to 100).toSeq) - jobData1.jobGroup should be(None) - jobData1.status should be(JobExecutionStatus.SUCCEEDED) - jobData1.numTasks should be(10) - jobData1.numActiveTasks should be(1) - jobData1.numCompletedTasks should be(2) - jobData1.numSkippedTasks should be(3) - jobData1.numFailedTasks should be(4) - jobData1.numActiveStages should be(10) - jobData1.numCompletedStages should be(20) - jobData1.numSkippedStages should be(30) - jobData1.numFailedStages should be(40) - - val jobData2 = jobDatas(1) - jobData2.jobId should be(2) - jobData2.name should be("2") - jobData2.status should be(JobExecutionStatus.FAILED) - } - } - - describe(".extractStageDatas") { - it("returns StageDatas extracted from the given legacy SparkApplicationData") { - val legacyData = new MockSparkApplicationData() { - val jobProgressData = { - val jobProgressData = new SparkJobProgressData() - - val stageInfoS1A1 = { - val stageInfo = new SparkJobProgressData.StageInfo() - - stageInfo.numActiveTasks = 1 - stageInfo.numCompleteTasks = 2 - stageInfo.numFailedTasks = 3 - - stageInfo.executorRunTime = 1000L - - stageInfo.inputBytes = 10000L - stageInfo.outputBytes = 20000L - stageInfo.shuffleReadBytes = 30000L - stageInfo.shuffleWriteBytes = 40000L - stageInfo.memoryBytesSpilled = 50000L - stageInfo.diskBytesSpilled = 60000L - - stageInfo.name = "1,1" - stageInfo.description = "a" - - stageInfo - } - jobProgressData.addStageInfo(1, 1, stageInfoS1A1) - jobProgressData.addCompletedStages(1, 1) - - val stageInfoS1A2 = { - val stageInfo = new SparkJobProgressData.StageInfo() - stageInfo.name = "1,2" - stageInfo - } - jobProgressData.addStageInfo(1, 2, stageInfoS1A2) - jobProgressData.addCompletedStages(1, 2) - - val stageInfoS2A1 = { - val stageInfo = new SparkJobProgressData.StageInfo() - stageInfo.name = "2,1" - stageInfo - } - jobProgressData.addStageInfo(2, 1, stageInfoS2A1) - jobProgressData.addFailedStages(2, 1) - - jobProgressData - } - - override def getJobProgressData(): SparkJobProgressData = jobProgressData - } - - val stageDatas = LegacyDataConverters.extractStageDatas(legacyData) - stageDatas.size should be(3) - - val stageDataS1A1 = stageDatas(0) - stageDataS1A1.status should be(StageStatus.COMPLETE) - stageDataS1A1.stageId should be(1) - stageDataS1A1.attemptId should be(1) - stageDataS1A1.numActiveTasks should be(1) - stageDataS1A1.numCompleteTasks should be(2) - stageDataS1A1.numFailedTasks should be(3) - stageDataS1A1.executorRunTime should be(1000L) - stageDataS1A1.inputBytes should be(10000L) - stageDataS1A1.inputRecords should be(0L) - stageDataS1A1.outputBytes should be(20000L) - stageDataS1A1.outputRecords should be(0L) - stageDataS1A1.shuffleReadBytes should be(30000L) - stageDataS1A1.shuffleReadRecords should be(0L) - stageDataS1A1.shuffleWriteBytes should be(40000L) - stageDataS1A1.shuffleWriteRecords should be(0L) - stageDataS1A1.memoryBytesSpilled should be(50000L) - stageDataS1A1.diskBytesSpilled should be(60000L) - stageDataS1A1.name should be("1,1") - stageDataS1A1.details should be("a") - stageDataS1A1.schedulingPool should be("") - stageDataS1A1.accumulatorUpdates should be(Seq.empty) - stageDataS1A1.tasks should be(None) - stageDataS1A1.executorSummary should be(None) - - val stageDataS1A2 = stageDatas(1) - stageDataS1A2.status should be(StageStatus.COMPLETE) - stageDataS1A2.stageId should be(1) - stageDataS1A2.attemptId should be(2) - stageDataS1A2.name should be("1,2") - - val stageDataS2A1 = stageDatas(2) - stageDataS2A1.status should be(StageStatus.FAILED) - stageDataS2A1.stageId should be(2) - stageDataS2A1.attemptId should be(1) - stageDataS2A1.name should be("2,1") - } - } - - describe(".extractExecutorSummaries") { - it("returns ExecutorSummaries extracted from the given legacy SparkApplicationData") { - val legacyData = new MockSparkApplicationData() { - val executorData = { - val executorData = new SparkExecutorData() - - val executorInfo1 = { - val executorInfo = new SparkExecutorData.ExecutorInfo() - - executorInfo.execId = "1" - executorInfo.hostPort = "9090" - - executorInfo.rddBlocks = 10 - executorInfo.memUsed = 10000L - executorInfo.maxMem = 20000L - executorInfo.diskUsed = 30000L - - executorInfo.activeTasks = 1 - executorInfo.completedTasks = 2 - executorInfo.failedTasks = 3 - executorInfo.totalTasks = 6 - - executorInfo.duration = 1000L - - executorInfo.inputBytes = 100000L - executorInfo.shuffleRead = 200000L - executorInfo.shuffleWrite = 300000L - - executorInfo - } - executorData.setExecutorInfo("1", executorInfo1) - - val executorInfo2 = { - val executorInfo = new SparkExecutorData.ExecutorInfo() - executorInfo.execId = "2" - executorInfo - } - executorData.setExecutorInfo("2", executorInfo2) - - executorData - } - - override def getExecutorData(): SparkExecutorData = executorData - } - - val executorSummaries = LegacyDataConverters.extractExecutorSummaries(legacyData) - executorSummaries.size should be(2) - - val executorSummary1 = executorSummaries(0) - executorSummary1.id should be("1") - executorSummary1.hostPort should be("9090") - executorSummary1.rddBlocks should be(10) - executorSummary1.memoryUsed should be(10000L) - executorSummary1.diskUsed should be(30000L) - executorSummary1.activeTasks should be(1) - executorSummary1.failedTasks should be(3) - executorSummary1.completedTasks should be(2) - executorSummary1.totalTasks should be(6) - executorSummary1.totalDuration should be(1000L) - executorSummary1.totalInputBytes should be(100000L) - executorSummary1.totalShuffleRead should be(200000L) - executorSummary1.totalShuffleWrite should be(300000L) - executorSummary1.maxMemory should be(20000L) - executorSummary1.executorLogs should be(Map.empty) - - val executorSummary2 = executorSummaries(1) - executorSummary2.id should be("2") - } - } - - describe(".") { - } - } -} - -object LegacyDataConvertersTest { - -} diff --git a/test/com/linkedin/drelephant/spark/legacydata/MockSparkApplicationData.java b/test/com/linkedin/drelephant/spark/legacydata/MockSparkApplicationData.java deleted file mode 100644 index 34917f6a4..000000000 --- a/test/com/linkedin/drelephant/spark/legacydata/MockSparkApplicationData.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.spark.legacydata; - -import com.linkedin.drelephant.analysis.ApplicationType; -import java.util.Properties; - - -/** - * This is a pseudo local implementation of SparkApplicationData interface, supposed to be used for test purpose. - */ -public class MockSparkApplicationData implements SparkApplicationData { - private static final ApplicationType APPLICATION_TYPE = new ApplicationType("SPARK"); - - private final SparkGeneralData _sparkGeneralData; - private final SparkEnvironmentData _sparkEnvironmentData; - private final SparkExecutorData _sparkExecutorData; - private final SparkJobProgressData _sparkJobProgressData; - private final SparkStorageData _sparkStorageData; - - public MockSparkApplicationData() { - _sparkGeneralData = new SparkGeneralData(); - _sparkEnvironmentData = new SparkEnvironmentData(); - _sparkExecutorData = new SparkExecutorData(); - _sparkJobProgressData = new SparkJobProgressData(); - _sparkStorageData = new SparkStorageData(); - } - - @Override - public boolean isThrottled() { - return false; - } - - @Override - public SparkGeneralData getGeneralData() { - return _sparkGeneralData; - } - - @Override - public SparkEnvironmentData getEnvironmentData() { - return _sparkEnvironmentData; - } - - @Override - public SparkExecutorData getExecutorData() { - return _sparkExecutorData; - } - - @Override - public SparkJobProgressData getJobProgressData() { - return _sparkJobProgressData; - } - - @Override - public SparkStorageData getStorageData() { - return _sparkStorageData; - } - - @Override - public Properties getConf() { - return getEnvironmentData().getSparkProperties(); - } - - @Override - public String getAppId() { - return getGeneralData().getApplicationId(); - } - - @Override - public ApplicationType getApplicationType() { - return APPLICATION_TYPE; - } - - @Override - public boolean isEmpty() { - return getExecutorData().getExecutors().isEmpty(); - } -} diff --git a/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java b/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java deleted file mode 100644 index 4a42a8617..000000000 --- a/test/com/linkedin/drelephant/tez/TezTaskLevelAggregatedMetricsTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez; - - -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import org.junit.Assert; -import org.junit.Test; - -public class TezTaskLevelAggregatedMetricsTest { - - @Test - public void testZeroTasks() { - TezTaskData taskData[] = {}; - TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(taskData, 0, 0); - Assert.assertEquals(taskMetrics.getDelay(), 0); - Assert.assertEquals(taskMetrics.getResourceUsed(), 0); - Assert.assertEquals(taskMetrics.getResourceWasted(), 0); - } - - @Test - public void testNullTaskArray() { - TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(null, 0, 0); - Assert.assertEquals(taskMetrics.getDelay(), 0); - Assert.assertEquals(taskMetrics.getResourceUsed(), 0); - Assert.assertEquals(taskMetrics.getResourceWasted(), 0); - } - - @Test - public void testTaskLevelData() { - TezTaskData taskData[] = new TezTaskData[3]; - TezCounterData counterData = new TezCounterData(); - counterData.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, 655577088L); - counterData.set(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES, 3051589632L); - long time[] = {0,0,0,1464218501117L, 1464218534148L}; - taskData[0] = new TezTaskData("task", "id"); - taskData[0].setTimeAndCounter(time,counterData); - taskData[1] = new TezTaskData("task", "id"); - taskData[1].setTimeAndCounter(new long[5],counterData); - // Non-sampled task, which does not contain time and counter data - taskData[2] = new TezTaskData("task", "id"); - TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(taskData, 4096L, 1463218501117L); - Assert.assertEquals(taskMetrics.getDelay(), 1000000000L); - Assert.assertEquals(taskMetrics.getResourceUsed(), 135168L); - Assert.assertEquals(taskMetrics.getResourceWasted(), 66627L); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java b/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java deleted file mode 100644 index bf38f10c3..000000000 --- a/test/com/linkedin/drelephant/tez/fetchers/TezFetcherTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.fetchers; - -import java.util.regex.Matcher; -import org.junit.Assert; -import org.junit.Test; -import com.linkedin.drelephant.util.ThreadContextMR2; - -public class TezFetcherTest { - - @Test - public void testDiagnosticMatcher() { - Matcher matcher = ThreadContextMR2.getDiagnosticMatcher("Task task_1443068695259_9143_m_000475 failed 1 time"); - Assert.assertEquals(".*[\\s\\u00A0]+(task_[0-9]+_[0-9]+_[m|r]_[0-9]+)[\\s\\u00A0]+.*", matcher.pattern().toString()); - Assert.assertEquals(true, matcher.matches()); - Assert.assertEquals(1, matcher.groupCount()); - Assert.assertEquals("task_1443068695259_9143_m_000475", matcher.group(1)); - } - -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java deleted file mode 100644 index 8792e70f4..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperDataSkewHeuristicTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.*; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - - -public class MapperDataSkewHeuristicTest extends TestCase { - - private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64; //1MB - private static final long UNITSIZETIME = 1000000; //1000sec - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperDataSkewHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(200, 200, 1 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(200, 200, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(200, 200, 20 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(200, 200, 30 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 50 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSmallFiles() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 1 * UNITSIZE, 5 * UNITSIZE)); - } - - public void testSmallTasks() throws IOException { - assertEquals(Severity.NONE, analyzeJob(5, 5, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testCriticalTime() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJobTime(200, 200, 1 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSevereTime() throws IOException { - assertEquals(Severity.SEVERE, analyzeJobTime(200, 200, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testModerateTime() throws IOException { - assertEquals(Severity.MODERATE, analyzeJobTime(200, 200, 20 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testLowTime() throws IOException { - assertEquals(Severity.LOW, analyzeJobTime(200, 200, 30 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testNoneTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(200, 200, 50 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSmallTasksTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(5, 5, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) - throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[numSmallTasks + numLargeTasks + 1]; - - TezCounterData smallCounter = new TezCounterData(); - smallCounter.set(TezCounterData.CounterName.HDFS_BYTES_READ, smallInputSize); - - TezCounterData largeCounter = new TezCounterData(); - largeCounter.set(TezCounterData.CounterName.S3A_BYTES_READ, largeInputSize); - - int i = 0; - for (; i < numSmallTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], smallCounter); - } - for (; i < numSmallTasks + numLargeTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], largeCounter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } - - private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) - throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[numSmallTasks + numLongTasks + 1]; - - int i = 0; - for (; i < numSmallTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTotalTimeMs(smallTimeTaken, true); - mappers[i].setCounter(jobCounter); - } - for (; i < numSmallTasks + numLongTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTotalTimeMs(longTimeTaken, true); - mappers[i].setCounter(jobCounter); - } - // Non-sampled task, which does not contain time data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java deleted file mode 100644 index 52cea2035..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperGCHeuristicTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class MapperGCHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - - private static Heuristic _heuristic = new MapperGCHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static int NUMTASKS = 100; - - public void testGCCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1000000, 50000, 2000)); - } - - public void testGCSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1000000, 50000, 1500)); - } - - public void testGCModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1000000, 50000, 1000)); - } - - public void testGCNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1000000, 50000, 300)); - } - - public void testShortTasksNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(100000, 50000, 2000)); - } - - - private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.CPU_MILLISECONDS, cpuMs); - counter.set(TezCounterData.CounterName.GC_TIME_MILLIS, gcMs); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[]{runtimeMs, 0 , 0, 0, 0}, counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } - -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java deleted file mode 100644 index 50f6f1470..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperMemoryHeuristicTest.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class MapperMemoryHeuristicTest extends TestCase{ - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperMemoryHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private int NUMTASKS = 100; - - public void testLargeContainerSizeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(2048, 8192)); - } - - public void testLargeContainerSizeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(3072, 8192)); - } - - public void testLargeContainerSizeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(4096, 8192)); - } - - public void testLargeContainerSizeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(6144, 8192)); - } - - // If the task use default container size, it should not be flagged - // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> CRITICAL - public void testDefaultContainerNone() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(256, 2048)); - } - - // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> MODERATE - public void testDefaultContainerNoneMore() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1024, 2048)); - } - - private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB* FileUtils.ONE_MB); - - Properties p = new Properties(); - p.setProperty(MapperMemoryHeuristic.MAPRED_MAPPER_MEMORY_CONF, Long.toString(containerMemMB)); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTime(new long[5]); - mappers[i].setCounter(counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - data.setConf(p); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java deleted file mode 100644 index 2d21f856a..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperSpeedHeuristicTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class MapperSpeedHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperSpeedHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static final long MB_IN_BYTES = FileUtils.ONE_MB; - private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS; - private static final int NUMTASKS = 100; - - public void testCritical() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.CRITICAL, analyzeJob(runtime, 1 * speed_factor)); - } - - public void testSevere() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.SEVERE, analyzeJob(runtime, 4 * speed_factor)); - } - - public void testModerate() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.MODERATE, analyzeJob(runtime, 13 * speed_factor)); - } - - public void testLow() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.LOW, analyzeJob(runtime, 50 * speed_factor)); - } - - public void testNone() throws IOException { - long runtime = 120 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.NONE, analyzeJob(runtime, 51 * speed_factor)); - } - - public void testShortTask() throws IOException { - long runtime = 2 * MINUTE_IN_MS; - long speed_factor = (runtime * MB_IN_BYTES) / 1000; - assertEquals(Severity.NONE, analyzeJob(runtime, 1 * speed_factor)); - } - - private Severity analyzeJob(long runtimeMs, long readBytes) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.HDFS_BYTES_READ, readBytes / 2); - counter.set(TezCounterData.CounterName.S3A_BYTES_READ, readBytes / 2); - - int i = 0; - for (; i < NUMTASKS; i++) { - mappers[i] = new TezTaskData(counter, new long[] { runtimeMs, 0, 0 ,0, 0}); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java deleted file mode 100644 index 62831a727..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperSpillHeuristicTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - - -public class MapperSpillHeuristicTest extends TestCase{ - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperSpillHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - // Spill ratio 3.0, 1000 tasks - assertEquals(Severity.CRITICAL, analyzeJob(3000, 1000, 1000)); - } - - public void testSevere() throws IOException { - // Spill ratio 2.5, 1000 tasks - assertEquals(Severity.SEVERE, analyzeJob(2500, 1000, 1000)); - } - - public void testModerate() throws IOException { - // Spill ratio 2.3, 1000 tasks - assertEquals(Severity.MODERATE, analyzeJob(2300, 1000, 1000)); - } - - public void testLow() throws IOException { - // Spill ratio 2.1, 1000 tasks - assertEquals(Severity.LOW, analyzeJob(2100, 1000, 1000)); - } - - public void testNone() throws IOException { - // Spill ratio 1.0, 1000 tasks - assertEquals(Severity.NONE, analyzeJob(1000, 1000, 1000)); - } - - public void testSmallNumTasks() throws IOException { - // Spill ratio 3.0, should be critical, but number of task is small(10), final result is NONE - assertEquals(Severity.NONE, analyzeJob(3000, 1000, 10)); - } - - private Severity analyzeJob(long spilledRecords, long mapRecords, int numTasks) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[numTasks + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.SPILLED_RECORDS, spilledRecords); - counter.set(TezCounterData.CounterName.OUTPUT_RECORDS, mapRecords); - - int i = 0; - for (; i < numTasks; i++) { - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - mappers[i].setTimeAndCounter(new long[5], counter); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java deleted file mode 100644 index 39c6ec891..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/MapperTimeHeuristicTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - - -public class MapperTimeHeuristicTest extends TestCase { - - private static final long DUMMY_INPUT_SIZE = 0; - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new MapperTimeHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - // Test batch 1: Large runtime. Heuristic is not affected by various number of tasks */ - - public void testLongRuntimeTasksCritical() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(10, 120 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksCriticalMore() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(1000, 120 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksSevere() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.SEVERE, analyzeJob(10, 60 * Statistics.MINUTE_IN_MS)); - } - - public void testLongRuntimeTasksSevereMore() throws IOException { - // Should decrease split size and increase number of tasks - assertEquals(Severity.SEVERE, analyzeJob(1000, 60 * Statistics.MINUTE_IN_MS)); - } - - // Test batch 2: Short runtime and various number of tasks - - public void testShortRuntimeTasksCritical() throws IOException { - // Should increase split size and decrease number of tasks - assertEquals(Severity.CRITICAL, analyzeJob(1000, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksSevere() throws IOException { - // Should increase split size and decrease number of tasks - assertEquals(Severity.SEVERE, analyzeJob(500, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(101, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(50, 1 * Statistics.MINUTE_IN_MS)); - } - - public void testShortRuntimeTasksNone() throws IOException { - // Small file with small number of tasks and short runtime. This should be the common case. - assertEquals(Severity.NONE, analyzeJob(5, 1 * Statistics.MINUTE_IN_MS)); - } - - private Severity analyzeJob(int numTasks, long runtime) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] mappers = new TezTaskData[numTasks + 1]; - - TezCounterData taskCounter = new TezCounterData(); - taskCounter.set(TezCounterData.CounterName.S3A_BYTES_READ, DUMMY_INPUT_SIZE / 4); - - - int i = 0; - for (; i < numTasks; i++) { - mappers[i] = new TezTaskData(jobCounter,new long[] { runtime, 0, 0, 0, 0 }); - } - // Non-sampled task, which does not contain time and counter data - mappers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } - - -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java deleted file mode 100644 index ff95e862a..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/ReducerDataSkewHeuristicTest.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.*; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class ReducerDataSkewHeuristicTest extends TestCase{ - - private static final long UNITSIZE = HDFSContext.HDFS_BLOCK_SIZE / 64; //1mb - private static final long UNITSIZETIME = 1000000; //1000sec - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerDataSkewHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - public void testCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(200, 200, 1 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(200, 200, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(200, 200, 20 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(200, 200, 30 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 50 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testSmallFiles() throws IOException { - assertEquals(Severity.NONE, analyzeJob(200, 200, 1 * UNITSIZE, 5 * UNITSIZE)); - } - - public void testSmallTasks() throws IOException { - assertEquals(Severity.NONE, analyzeJob(5, 5, 10 * UNITSIZE, 100 * UNITSIZE)); - } - - public void testCriticalTime() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJobTime(200, 200, 1 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSevereTime() throws IOException { - assertEquals(Severity.SEVERE, analyzeJobTime(200, 200, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testModerateTime() throws IOException { - assertEquals(Severity.MODERATE, analyzeJobTime(200, 200, 20 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testLowTime() throws IOException { - assertEquals(Severity.LOW, analyzeJobTime(200, 200, 30 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testNoneTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(200, 200, 50 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - public void testSmallTasksTime() throws IOException { - assertEquals(Severity.NONE, analyzeJobTime(5, 5, 10 * UNITSIZETIME, 100 * UNITSIZETIME)); - } - - private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) - throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[numSmallTasks + numLargeTasks + 1]; - - TezCounterData smallCounter = new TezCounterData(); - smallCounter.set(TezCounterData.CounterName.SHUFFLE_BYTES, smallInputSize); - - TezCounterData largeCounter = new TezCounterData(); - largeCounter.set(TezCounterData.CounterName.SHUFFLE_BYTES, largeInputSize); - - int i = 0; - for (; i < numSmallTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], smallCounter); - } - for (; i < numSmallTasks + numLargeTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], largeCounter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } - - private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) - throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[numSmallTasks + numLongTasks + 1]; - - int i = 0; - for (; i < numSmallTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTotalTimeMs(smallTimeTaken, true); - reducers[i].setCounter(jobCounter); - } - for (; i < numSmallTasks + numLongTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTotalTimeMs(longTimeTaken, true); - reducers[i].setCounter(jobCounter); - } - // Non-sampled task, which does not contain time data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java deleted file mode 100644 index 4a86fbefb..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/ReducerGCHeuristicTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public class ReducerGCHeuristicTest extends TestCase{ - - private static Map paramsMap = new HashMap(); - - private static Heuristic _heuristic = new ReducerGCHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static int NUMTASKS = 100; - - public void testGCCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1000000, 50000, 2000)); - } - - public void testGCSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1000000, 50000, 1500)); - } - - public void testGCModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1000000, 50000, 1000)); - } - - public void testGCNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1000000, 50000, 300)); - } - - public void testShortTasksNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(100000, 50000, 2000)); - } - - - private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.CPU_MILLISECONDS, cpuMs); - counter.set(TezCounterData.CounterName.GC_TIME_MILLIS, gcMs); - - int i = 0; - for (; i < NUMTASKS; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java deleted file mode 100644 index b2074b323..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/ReducerMemoryHeuristicTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; -import org.apache.commons.io.FileUtils; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public class ReducerMemoryHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerMemoryHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private int NUMTASKS = 100; - - public void testLargeContainerSizeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(2048, 8192)); - } - - public void testLargeContainerSizeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(3072, 8192)); - } - - public void testLargeContainerSizeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(4096, 8192)); - } - - public void testLargeContainerSizeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(6144, 8192)); - } - - // If the task use default container size, it should not be flagged - // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> CRITICAL - public void testDefaultContainerNone() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(256, 2048)); - } - - // Not using Default Container param, will calculate severity irrespective of default container size Chaning NONE -> MODERATE - public void testDefaultContainerNoneMore() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1024, 2048)); - } - - private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException { - TezCounterData jobCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[NUMTASKS + 1]; - - TezCounterData counter = new TezCounterData(); - counter.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB* FileUtils.ONE_MB); - - Properties p = new Properties(); - p.setProperty(ReducerMemoryHeuristic.MAPRED_REDUCER_MEMORY_CONF, Long.toString(containerMemMB)); - - int i = 0; - for (; i < NUMTASKS; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTimeAndCounter(new long[5], counter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers); - data.setConf(p); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java b/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java deleted file mode 100644 index ab0abbe38..000000000 --- a/test/com/linkedin/drelephant/tez/heuristics/ReducerTimeHeuristicTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2017 Electronic Arts Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ -package com.linkedin.drelephant.tez.heuristics; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.Heuristic; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.tez.data.TezApplicationData; -import com.linkedin.drelephant.tez.data.TezCounterData; -import com.linkedin.drelephant.tez.data.TezTaskData; -import junit.framework.TestCase; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - - -public class ReducerTimeHeuristicTest extends TestCase { - - private static Map paramsMap = new HashMap(); - private static Heuristic _heuristic = new ReducerTimeHeuristic(new HeuristicConfigurationData("test_heuristic", - "test_class", "test_view", new ApplicationType("test_apptype"), paramsMap)); - - private static final long MINUTE_IN_MS = Statistics.MINUTE_IN_MS;; - - public void testShortRunetimeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(1 * MINUTE_IN_MS, 1000)); - } - - public void testShortRunetimeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(1 * MINUTE_IN_MS, 500)); - } - - public void testShortRunetimeModerate() throws IOException { - assertEquals(Severity.MODERATE, analyzeJob(1 * MINUTE_IN_MS, 101)); - } - - public void testShortRunetimeLow() throws IOException { - assertEquals(Severity.LOW, analyzeJob(1 * MINUTE_IN_MS, 50)); - } - - public void testShortRunetimeNone() throws IOException { - assertEquals(Severity.NONE, analyzeJob(1 * MINUTE_IN_MS, 2)); - } - - public void testLongRunetimeCritical() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 10)); - } - - // Long runtime severity is not affected by number of tasks - public void testLongRunetimeCriticalMore() throws IOException { - assertEquals(Severity.CRITICAL, analyzeJob(120 * MINUTE_IN_MS, 1000)); - } - - public void testLongRunetimeSevere() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(60 * MINUTE_IN_MS, 10)); - } - - public void testLongRunetimeSevereMore() throws IOException { - assertEquals(Severity.SEVERE, analyzeJob(60 * MINUTE_IN_MS, 1000)); - } - - private Severity analyzeJob(long runtimeMs, int numTasks) throws IOException { - TezCounterData dummyCounter = new TezCounterData(); - TezTaskData[] reducers = new TezTaskData[numTasks + 1]; - - int i = 0; - for (; i < numTasks; i++) { - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - reducers[i].setTime(new long[] { runtimeMs, 0, 0, 0, 0 }); - reducers[i].setCounter(dummyCounter); - } - // Non-sampled task, which does not contain time and counter data - reducers[i] = new TezTaskData("task-id-"+i, "task-attempt-id-"+i); - - TezApplicationData data = new TezApplicationData().setCounters(dummyCounter).setReduceTaskData(reducers); - HeuristicResult result = _heuristic.apply(data); - return result.getSeverity(); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tony/TonyMetricsAggregatorTest.java b/test/com/linkedin/drelephant/tony/TonyMetricsAggregatorTest.java deleted file mode 100644 index 2e760e8e9..000000000 --- a/test/com/linkedin/drelephant/tony/TonyMetricsAggregatorTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Copyright 2019 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.linkedin.drelephant.tony; - -import com.google.common.collect.ImmutableList; -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HadoopAggregatedData; -import com.linkedin.drelephant.math.Statistics; -import com.linkedin.drelephant.tony.data.TonyApplicationData; -import com.linkedin.tony.Constants; -import com.linkedin.tony.TonyConfigurationKeys; -import com.linkedin.tony.events.Event; -import com.linkedin.tony.events.EventType; -import com.linkedin.tony.events.Metric; -import com.linkedin.tony.events.TaskFinished; -import com.linkedin.tony.events.TaskStarted; -import com.linkedin.tony.rpc.impl.TaskStatus; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - -import static com.linkedin.drelephant.tony.TonyMetricsAggregator.MEMORY_BUFFER; - - -public class TonyMetricsAggregatorTest { - @Test - public void testMetricsAggregator() { - Configuration conf = new Configuration(false); - conf.set(TonyConfigurationKeys.getResourceKey(Constants.WORKER_JOB_NAME, Constants.MEMORY), "4g"); - conf.setInt(TonyConfigurationKeys.getInstancesKey(Constants.WORKER_JOB_NAME), 2); - conf.set(TonyConfigurationKeys.getResourceKey(Constants.PS_JOB_NAME, Constants.MEMORY), "4g"); - conf.setInt(TonyConfigurationKeys.getInstancesKey(Constants.PS_JOB_NAME), 1); - - List events = new ArrayList<>(); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.WORKER_JOB_NAME, 0, null),0L)); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.WORKER_JOB_NAME, 1, null),0L)); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.PS_JOB_NAME, 0, null),0L)); - events.add(new Event(EventType.TASK_FINISHED, - new TaskFinished(Constants.WORKER_JOB_NAME, 0, TaskStatus.SUCCEEDED.toString(), - ImmutableList.of(new Metric(Constants.MAX_MEMORY_BYTES, (double) FileUtils.ONE_GB))), - 10L * Statistics.SECOND_IN_MS)); - events.add(new Event(EventType.TASK_FINISHED, - new TaskFinished(Constants.WORKER_JOB_NAME, 1, TaskStatus.SUCCEEDED.toString(), - ImmutableList.of(new Metric(Constants.MAX_MEMORY_BYTES, (double) 2 * FileUtils.ONE_GB))), - 20L * Statistics.SECOND_IN_MS)); - events.add(new Event(EventType.TASK_FINISHED, - new TaskFinished(Constants.PS_JOB_NAME, 0, TaskStatus.SUCCEEDED.toString(), - ImmutableList.of(new Metric(Constants.MAX_MEMORY_BYTES, (double) FileUtils.ONE_GB))), - 20L * Statistics.SECOND_IN_MS)); - - long expectedResourcesUsed = 10 * 4 * 1024 + 20 * 4 * 1024 + 20 * 4 * 1024; - long expectedResourcesWasted = 10 * (long) (4 * 1024 - 2 * 1024 * MEMORY_BUFFER) - + 20 * (long) (4 * 1024 - 2 * 1024 * MEMORY_BUFFER) - + 20 * (long) (4 * 1024 - 1 * 1024 * MEMORY_BUFFER); - - ApplicationType appType = new ApplicationType(Constants.APP_TYPE); - TonyApplicationData data = new TonyApplicationData("application_123_456", appType, conf, events); - TonyMetricsAggregator metricsAggregator = new TonyMetricsAggregator(null); - metricsAggregator.aggregate(data); - HadoopAggregatedData result = metricsAggregator.getResult(); - Assert.assertEquals(expectedResourcesUsed, result.getResourceUsed()); - Assert.assertEquals(expectedResourcesWasted, result.getResourceWasted()); - } - - /** - * Verifies that wasted resources is 0 when there are no metrics. - */ - @Test - public void testNullMetrics() { - Configuration conf = new Configuration(false); - conf.set(TonyConfigurationKeys.getResourceKey(Constants.WORKER_JOB_NAME, Constants.MEMORY), "4g"); - conf.setInt(TonyConfigurationKeys.getInstancesKey(Constants.WORKER_JOB_NAME), 2); - conf.set(TonyConfigurationKeys.getResourceKey(Constants.PS_JOB_NAME, Constants.MEMORY), "4g"); - conf.setInt(TonyConfigurationKeys.getInstancesKey(Constants.PS_JOB_NAME), 1); - - List events = new ArrayList<>(); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.WORKER_JOB_NAME, 0, null),0L)); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.WORKER_JOB_NAME, 1, null),0L)); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.PS_JOB_NAME, 0, null),0L)); - events.add(new Event(EventType.TASK_FINISHED, - new TaskFinished(Constants.WORKER_JOB_NAME, 0, TaskStatus.SUCCEEDED.toString(), Collections.emptyList()), - 10L * Statistics.SECOND_IN_MS)); - events.add(new Event(EventType.TASK_FINISHED, - new TaskFinished(Constants.WORKER_JOB_NAME, 1, TaskStatus.SUCCEEDED.toString(), Collections.emptyList()), - 20L * Statistics.SECOND_IN_MS)); - events.add(new Event(EventType.TASK_FINISHED, - new TaskFinished(Constants.PS_JOB_NAME, 0, TaskStatus.SUCCEEDED.toString(), Collections.emptyList()), - 20L * Statistics.SECOND_IN_MS)); - - long expectedResourcesUsed = 10 * 4 * 1024 + 20 * 4 * 1024 + 20 * 4 * 1024; - long expectedResourcesWasted = 0; - - ApplicationType appType = new ApplicationType(Constants.APP_TYPE); - TonyApplicationData data = new TonyApplicationData("application_123_456", appType, conf, events); - TonyMetricsAggregator metricsAggregator = new TonyMetricsAggregator(null); - metricsAggregator.aggregate(data); - HadoopAggregatedData result = metricsAggregator.getResult(); - Assert.assertEquals(expectedResourcesUsed, result.getResourceUsed()); - Assert.assertEquals(expectedResourcesWasted, result.getResourceWasted()); - } - - /** - * Verifies that used and wasted resources are 0 when there are no task finished or application finished events. - */ - @Test - public void testNoEndEvents() { - Configuration conf = new Configuration(false); - conf.set(TonyConfigurationKeys.getResourceKey(Constants.WORKER_JOB_NAME, Constants.MEMORY), "4g"); - conf.setInt(TonyConfigurationKeys.getInstancesKey(Constants.WORKER_JOB_NAME), 2); - conf.set(TonyConfigurationKeys.getResourceKey(Constants.PS_JOB_NAME, Constants.MEMORY), "4g"); - conf.setInt(TonyConfigurationKeys.getInstancesKey(Constants.PS_JOB_NAME), 1); - - List events = new ArrayList<>(); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.WORKER_JOB_NAME, 0, null),0L)); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.WORKER_JOB_NAME, 1, null),0L)); - events.add(new Event(EventType.TASK_STARTED, new TaskStarted(Constants.PS_JOB_NAME, 0, null),0L)); - - long expectedResourcesUsed = 0; - long expectedResourcesWasted = 0; - - ApplicationType appType = new ApplicationType(Constants.APP_TYPE); - TonyApplicationData data = new TonyApplicationData("application_123_456", appType, conf, events); - TonyMetricsAggregator metricsAggregator = new TonyMetricsAggregator(null); - metricsAggregator.aggregate(data); - HadoopAggregatedData result = metricsAggregator.getResult(); - Assert.assertEquals(expectedResourcesUsed, result.getResourceUsed()); - Assert.assertEquals(expectedResourcesWasted, result.getResourceWasted()); - } -} diff --git a/test/com/linkedin/drelephant/tony/fetchers/TonyFetcherTest.java b/test/com/linkedin/drelephant/tony/fetchers/TonyFetcherTest.java deleted file mode 100644 index caf495be5..000000000 --- a/test/com/linkedin/drelephant/tony/fetchers/TonyFetcherTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright 2019 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.linkedin.drelephant.tony.fetchers; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.io.Files; -import com.linkedin.drelephant.analysis.AnalyticJob; -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.configurations.fetcher.FetcherConfigurationData; -import com.linkedin.drelephant.tony.data.TonyApplicationData; -import com.linkedin.drelephant.tony.data.TonyTaskData; -import com.linkedin.tony.Constants; -import com.linkedin.tony.TonyConfigurationKeys; -import com.linkedin.tony.events.Event; -import com.linkedin.tony.events.EventType; -import com.linkedin.tony.events.Metric; -import com.linkedin.tony.events.TaskFinished; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; -import org.apache.avro.file.DataFileWriter; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.specific.SpecificDatumWriter; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - - -public class TonyFetcherTest { - private static final String APPLICATION_ID_1 = "application_123_456"; - private static final String APPLICATION_ID_2 = "application_789_101"; - private static File _intermediateDir; - private static File _finishedDir; - private static String _tonyConfDir; - private static Date _endDate; - - @BeforeClass - public static void setup() throws IOException, ParseException { - setupTestData(); - setupTestTonyConfDir(); - } - - private static void setupTestData() throws IOException, ParseException { - String yearMonthDay = "2019/05/02"; - SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd"); - _endDate = sdf.parse(yearMonthDay); - - File tempDir = Files.createTempDir(); - _intermediateDir = new File(tempDir, "intermediate"); - _finishedDir = new File(tempDir, "finished"); - - createAppHistoryDir(new File(_intermediateDir, APPLICATION_ID_1)); - createAppHistoryDir(new File(_finishedDir, yearMonthDay + Path.SEPARATOR + APPLICATION_ID_2)); - } - - private static void createAppHistoryDir(File appDir) throws IOException { - appDir.mkdirs(); - - // write fake config history file - Configuration conf = new Configuration(false); - conf.set("foo", "bar"); - File configFile = new File(appDir, Constants.TONY_FINAL_XML); - conf.writeXml(new FileOutputStream(configFile)); - - // create fake events - Event event0 = new Event(EventType.TASK_FINISHED, new TaskFinished("worker", 0, "SUCCEEDED", - ImmutableList.of(new Metric("my_metric", 0.0))), System.currentTimeMillis()); - Event event1 = new Event(EventType.TASK_FINISHED, new TaskFinished("worker", 1, "SUCCEEDED", - ImmutableList.of(new Metric("my_metric", 1.0))), System.currentTimeMillis()); - Event event2 = new Event(EventType.TASK_FINISHED, new TaskFinished("ps", 0, "SUCCEEDED", - ImmutableList.of(new Metric("my_metric", 0.0))), System.currentTimeMillis()); - - // write fake events history file - File eventFile = new File(appDir, - APPLICATION_ID_1 + "-0-" + _endDate.getTime() + "-user1-SUCCEEDED." + Constants.HISTFILE_SUFFIX); - DatumWriter userDatumWriter = new SpecificDatumWriter<>(Event.class); - DataFileWriter dataFileWriter = new DataFileWriter<>(userDatumWriter); - dataFileWriter.create(event0.getSchema(), eventFile); - dataFileWriter.append(event0); - dataFileWriter.append(event1); - dataFileWriter.append(event2); - dataFileWriter.close(); - } - - private static void setupTestTonyConfDir() throws IOException { - Configuration testTonyConf = new Configuration(false); - testTonyConf.set(TonyConfigurationKeys.TONY_HISTORY_INTERMEDIATE, _intermediateDir.getPath()); - testTonyConf.set(TonyConfigurationKeys.TONY_HISTORY_FINISHED, _finishedDir.getPath()); - - File confDir = Files.createTempDir(); - _tonyConfDir = confDir.getPath(); - File tonySiteFile = new File(confDir, Constants.TONY_SITE_CONF); - testTonyConf.writeXml(new FileOutputStream(tonySiteFile)); - } - - @Test - public void testFetchDataIntermediateDir() throws Exception { - testHelper(APPLICATION_ID_1); - } - - @Test - public void testFetchDataFinishedDir() throws Exception { - testHelper(APPLICATION_ID_2); - } - - private static void testHelper(String appId) throws Exception { - FetcherConfigurationData configData = new FetcherConfigurationData(null, null, - ImmutableMap.of(Constants.TONY_CONF_DIR, _tonyConfDir, - TonyConfigurationKeys.TONY_HISTORY_FINISHED_DIR_TIMEZONE, - TonyConfigurationKeys.DEFAULT_TONY_HISTORY_FINISHED_DIR_TIMEZONE)); - TonyFetcher tonyFetcher = new TonyFetcher(configData); - - AnalyticJob job = new AnalyticJob(); - ApplicationType tonyAppType = new ApplicationType(Constants.APP_TYPE); - job.setFinishTime(_endDate.getTime()); - job.setAppId(appId); - job.setAppType(tonyAppType); - TonyApplicationData appData = tonyFetcher.fetchData(job); - - Assert.assertEquals(appId, appData.getAppId()); - Assert.assertEquals(tonyAppType, appData.getApplicationType()); - Assert.assertEquals("bar", appData.getConf().get("foo")); - Map> metricsMap = appData.getTaskMap(); - Assert.assertEquals(2, metricsMap.size()); - Assert.assertEquals(2, metricsMap.get("worker").size()); - Assert.assertEquals(1, metricsMap.get("ps").size()); - } -} \ No newline at end of file diff --git a/test/com/linkedin/drelephant/tony/heuristics/TaskMemoryHeuristicTest.java b/test/com/linkedin/drelephant/tony/heuristics/TaskMemoryHeuristicTest.java deleted file mode 100644 index 0a82bac96..000000000 --- a/test/com/linkedin/drelephant/tony/heuristics/TaskMemoryHeuristicTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2019 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.linkedin.drelephant.tony.heuristics; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HeuristicResult; -import com.linkedin.drelephant.analysis.Severity; -import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; -import com.linkedin.drelephant.tony.data.TonyApplicationData; -import com.linkedin.drelephant.util.Utils; -import com.linkedin.tony.Constants; -import com.linkedin.tony.TonyConfigurationKeys; -import com.linkedin.tony.events.Event; -import com.linkedin.tony.events.EventType; -import com.linkedin.tony.events.Metric; -import com.linkedin.tony.events.TaskFinished; -import com.linkedin.tony.rpc.impl.TaskStatus; -import controllers.Application; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Test; - - -public class TaskMemoryHeuristicTest { - - /** - * 10g workers requested, max worker memory < 50% - */ - @Test - public void testCritical() { - testHelper( - ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{ - 1.2e9, - 1.1e9, - 1e9, - 1.3e9 - }, Constants.PS_JOB_NAME, new double[]{0.5e9}), - ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g", Constants.PS_JOB_NAME, "2g"), - Severity.CRITICAL, - Severity.CRITICAL.getValue() * 4 - ); - } - - /** - * 10g workers requested, max worker memory < 70%; 10g ps requested, max ps memory < 60% - */ - @Test - public void testSevere() { - testHelper( - ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{ - 6.5e9, - 6.6e9, - }, Constants.PS_JOB_NAME, new double[]{5.84e9}), - ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g", Constants.PS_JOB_NAME, "10g"), - Severity.SEVERE, - Severity.MODERATE.getValue() * 2 /* workers */ + Severity.SEVERE.getValue() * 1 - ); - } - - /** - * 10g workers requested, max worker memory < 70% - */ - @Test - public void testModerate() { - testHelper( - ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{ - 6.5e9, - 6.6e9, - }), - ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g"), - Severity.MODERATE, - Severity.MODERATE.getValue() * 2 - ); - } - - /** - * 10g workers requested, max worker memory < 80% - */ - @Test - public void testLow() { - testHelper( - ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{ - 7.56e9, - 7.45e9, - }), - ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g"), - Severity.LOW, - 0 - ); - } - - /** - * 10g workers requested, max worker memory > 80% - */ - @Test - public void testNone() { - testHelper( - ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{ - 8.5e9, - 8.6e9, - }), - ImmutableMap.of(Constants.WORKER_JOB_NAME, "10g"), - Severity.NONE, - 0 - ); - } - - /** - * Low memory utilization but default container size, so pass. - */ - @Test - public void testLowUtilizationDefaultContainerSize() { - testHelper( - ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{ - 0.5e9, - 0.6e9, - }), - ImmutableMap.of(Constants.WORKER_JOB_NAME, "2g"), - Severity.NONE, - 0 - ); - } - - /** - * Though memory utilization is about 50%, severity should be none - * because requested memory is within the default 2 GB grace headroom of the - * max used memory. - */ - @Test - public void testRequestedSizeWithinGraceHeadroomSeverity() { - testHelper( - ImmutableMap.of(Constants.WORKER_JOB_NAME, new double[]{ - 1.5e9, - 1.6e9, - }), - ImmutableMap.of(Constants.WORKER_JOB_NAME, "3g"), - Severity.NONE, - 0 - ); - } - - /** - * Verifies that no exception is thrown when the task map is empty. - */ - @Test - public void testEmptyTaskMap() { - ApplicationType appType = new ApplicationType(Constants.APP_TYPE); - Configuration conf = new Configuration(false); - conf.setInt(TonyConfigurationKeys.getInstancesKey(Constants.PS_JOB_NAME), 0); - TonyApplicationData data = new TonyApplicationData("application_123_456", - appType, conf, Collections.EMPTY_LIST); - new TaskMemoryHeuristic(new HeuristicConfigurationData("ignored", - "ignored", "ignored", appType, Collections.EMPTY_MAP)).apply(data); - - } - - public void testHelper(Map memUsed, Map memRequested, Severity expectedSeverity, - int expectedScore) { - Configuration conf = new Configuration(false); - List events = new ArrayList<>(); - for (Map.Entry entry : memRequested.entrySet()) { - String taskType = entry.getKey(); - conf.set(TonyConfigurationKeys.getResourceKey(taskType, Constants.MEMORY), entry.getValue()); - conf.setInt(TonyConfigurationKeys.getInstancesKey(taskType), memUsed.get(taskType).length); - - for (int i = 0; i < memUsed.get(taskType).length; i++) { - events.add(new Event(EventType.TASK_FINISHED, - new TaskFinished(taskType, i, TaskStatus.SUCCEEDED.toString(), - ImmutableList.of(new Metric(Constants.MAX_MEMORY_BYTES, memUsed.get(taskType)[i]))), - System.currentTimeMillis())); - } - } - - ApplicationType appType = new ApplicationType(Constants.APP_TYPE); - TonyApplicationData data = new TonyApplicationData("application_123_456", appType, conf, events); - - TaskMemoryHeuristic heuristic = new TaskMemoryHeuristic(new HeuristicConfigurationData("ignored", - "ignored", "ignored", appType, Collections.EMPTY_MAP)); - HeuristicResult result = heuristic.apply(data); - Assert.assertEquals(expectedSeverity, result.getSeverity()); - Assert.assertEquals(expectedScore, result.getScore()); - } -} diff --git a/test/com/linkedin/drelephant/tony/util/TonyUtilsTest.java b/test/com/linkedin/drelephant/tony/util/TonyUtilsTest.java deleted file mode 100644 index e305e9edd..000000000 --- a/test/com/linkedin/drelephant/tony/util/TonyUtilsTest.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.linkedin.drelephant.tony.util; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.linkedin.drelephant.tony.data.TonyTaskData; -import com.linkedin.tony.Constants; -import com.linkedin.tony.events.Metric; -import java.util.Map; -import java.util.TreeMap; -import org.junit.Assert; -import org.junit.Test; - - -public class TonyUtilsTest { - /** - * Worker 0 is missing metrics, but worker 1 has metrics; we should use worker 1's - * max memory metrics. - */ - @Test - public void testGetMaxMemorySomeTasksMissingMetrics() { - Map taskDataMap = new TreeMap<>(); - TonyTaskData worker0Data = new TonyTaskData("worker", 0); - TonyTaskData worker1Data = new TonyTaskData("worker", 1); - double worker1MaxMemoryBytes = 123d; - worker1Data.setMetrics(ImmutableList.of(new Metric(Constants.MAX_MEMORY_BYTES, worker1MaxMemoryBytes))); - - taskDataMap.put(0, worker0Data); - taskDataMap.put(1, worker1Data); - - Assert.assertEquals(worker1MaxMemoryBytes, - TonyUtils.getMaxMetricForTaskTypeAndMetricName(ImmutableMap.of("worker", taskDataMap), "worker", - Constants.MAX_MEMORY_BYTES), 0); - } - - @Test - public void testGetMaxMetricForTaskTypeAndMetricNameMissingTask() { - Map taskDataMap = new TreeMap<>(); - TonyTaskData worker0Data = new TonyTaskData("worker", 0); - - taskDataMap.put(0, worker0Data); - - Assert.assertEquals(-1.0d, - TonyUtils.getMaxMetricForTaskTypeAndMetricName(ImmutableMap.of("worker", taskDataMap), "ps", - Constants.MAX_MEMORY_BYTES), 0); - } - - @Test - public void testGetMaxMetricForTaskTypeAndMetricName() { - Map taskDataMap = new TreeMap<>(); - TonyTaskData worker0Data = new TonyTaskData("worker", 0); - TonyTaskData worker1Data = new TonyTaskData("worker", 1); - - double worker0MaxGPUUtilization = 20.0d; - double worker1MaxGPUUtilization = 21.0d; - double worker0MaxGPUFBMemoryUsage = 22.0d; - double worker1MaxGPUFBMemoryUsage = 23.0d; - double worker0MaxGPUMainMemoryUsage = 2.0d; - double worker1MaxGPUMainMemoryUsage = -1.0d; - - worker0Data.setMetrics(ImmutableList.of( - new Metric(Constants.MAX_GPU_UTILIZATION, worker0MaxGPUUtilization), - new Metric(Constants.MAX_GPU_FB_MEMORY_USAGE, worker0MaxGPUFBMemoryUsage), - new Metric(Constants.MAX_GPU_MAIN_MEMORY_USAGE, worker0MaxGPUMainMemoryUsage) - )); - worker1Data.setMetrics(ImmutableList.of( - new Metric(Constants.MAX_GPU_UTILIZATION, worker1MaxGPUUtilization), - new Metric(Constants.MAX_GPU_FB_MEMORY_USAGE, worker1MaxGPUFBMemoryUsage), - new Metric(Constants.MAX_GPU_MAIN_MEMORY_USAGE, worker1MaxGPUMainMemoryUsage) - )); - - taskDataMap.put(0, worker0Data); - taskDataMap.put(1, worker1Data); - - Assert.assertEquals(worker1MaxGPUUtilization, - TonyUtils.getMaxMetricForTaskTypeAndMetricName(ImmutableMap.of("worker", taskDataMap), "worker", - Constants.MAX_GPU_UTILIZATION), 0); - Assert.assertEquals(worker1MaxGPUFBMemoryUsage, - TonyUtils.getMaxMetricForTaskTypeAndMetricName(ImmutableMap.of("worker", taskDataMap), "worker", - Constants.MAX_GPU_FB_MEMORY_USAGE), 0); - Assert.assertEquals(worker0MaxGPUMainMemoryUsage, - TonyUtils.getMaxMetricForTaskTypeAndMetricName(ImmutableMap.of("worker", taskDataMap), "worker", - Constants.MAX_GPU_MAIN_MEMORY_USAGE), 0); - } - - @Test - public void testGetAvgMetricForTaskTypeAndMetricName() { - Map taskDataMap = new TreeMap<>(); - TonyTaskData worker0Data = new TonyTaskData("worker", 0); - TonyTaskData worker1Data = new TonyTaskData("worker", 1); - - double worker0AvgGPUUtilization = 10.0d; - double worker1AvgGPUUtilization = 20.0d; - double worker0AvgGPUFBMemoryUsage = 30.0d; - double worker1AvgGPUFBMemoryUsage = 0.0d; - double worker0AvgGPUMainMemoryUsage = 40.0d; - double worker1AvgGPUMainMemoryUsage = -1.0d; - - worker0Data.setMetrics(ImmutableList.of( - new Metric(Constants.AVG_GPU_UTILIZATION, worker0AvgGPUUtilization), - new Metric(Constants.AVG_GPU_FB_MEMORY_USAGE, worker0AvgGPUFBMemoryUsage), - new Metric(Constants.AVG_GPU_MAIN_MEMORY_USAGE, worker0AvgGPUMainMemoryUsage)) - ); - worker1Data.setMetrics(ImmutableList.of( - new Metric(Constants.AVG_GPU_UTILIZATION, worker1AvgGPUUtilization), - new Metric(Constants.AVG_GPU_FB_MEMORY_USAGE, worker1AvgGPUFBMemoryUsage), - new Metric(Constants.AVG_GPU_MAIN_MEMORY_USAGE, worker1AvgGPUMainMemoryUsage) - )); - - taskDataMap.put(0, worker0Data); - taskDataMap.put(1, worker1Data); - - Assert.assertEquals(15.0d, - TonyUtils.getAvgMetricForTaskTypeAndMetricName(ImmutableMap.of("worker", taskDataMap), "worker", - Constants.AVG_GPU_UTILIZATION), 0); - Assert.assertEquals(30.0d, - TonyUtils.getAvgMetricForTaskTypeAndMetricName(ImmutableMap.of("worker", taskDataMap), "worker", - Constants.AVG_GPU_FB_MEMORY_USAGE), 0); - Assert.assertEquals(40.0d, - TonyUtils.getAvgMetricForTaskTypeAndMetricName(ImmutableMap.of("worker", taskDataMap), "worker", - Constants.AVG_GPU_MAIN_MEMORY_USAGE), 0); - } -} diff --git a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java deleted file mode 100644 index 255268255..000000000 --- a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.tuning; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeType; -import com.linkedin.drelephant.DrElephant; -import com.linkedin.drelephant.ElephantContext; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import models.JobDefinition; -import models.JobExecution; -import models.JobSuggestedParamSet; -import models.JobSuggestedParamValue; -import models.TuningAlgorithm; -import models.TuningJobDefinition; -import models.TuningParameter; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.LoggerFactory; -import play.Application; -import play.GlobalSettings; -import play.libs.Json; -import play.test.FakeApplication; -import org.apache.hadoop.conf.Configuration; - - -import static common.DBTestUtil.*; -import static common.TestConstants.*; -import static org.junit.Assert.*; -import static play.test.Helpers.*; - - -public class PSOParamGeneratorTest { - - private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(PSOParamGeneratorTest.class); - private static FakeApplication fakeApp; - private int numParametersToTune; - - @Before - public void setup() { - Map dbConn = new HashMap(); - dbConn.put(DB_DEFAULT_DRIVER_KEY, DB_DEFAULT_DRIVER_VALUE); - dbConn.put(DB_DEFAULT_URL_KEY, DB_DEFAULT_URL_VALUE); - dbConn.put(EVOLUTION_PLUGIN_KEY, EVOLUTION_PLUGIN_VALUE); - dbConn.put(APPLY_EVOLUTIONS_DEFAULT_KEY, APPLY_EVOLUTIONS_DEFAULT_VALUE); - - GlobalSettings gs = new GlobalSettings() { - @Override - public void onStart(Application app) { - LOGGER.info("Starting FakeApplication"); - } - }; - - fakeApp = fakeApplication(dbConn, gs); - Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - Boolean autoTuningEnabled = configuration.getBoolean(DrElephant.AUTO_TUNING_ENABLED, false); - org.junit.Assume.assumeTrue(autoTuningEnabled); - } - - private void populateTestData() { - try { - initDB(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Test - public void generateParamSetTest() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - - populateTestData(); - JobDefinition jobDefinition = JobDefinition.find.byId(100003); - TuningJobDefinition tuningJobDefinition = - TuningJobDefinition.find.where().eq("job.id", jobDefinition.id).findUnique(); - TuningAlgorithm tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; - - List tuningParameterList = TuningParameter.find.where() - .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, tuningAlgorithm.id) - .eq(TuningParameter.TABLE.isDerived, 0) - .findList(); - - numParametersToTune = tuningParameterList.size(); - - LOGGER.info("PSOParamGeneratorTest parameter list: " + Json.toJson(tuningParameterList)); - - JobTuningInfo jobTuningInfo = new JobTuningInfo(); - jobTuningInfo.setTuningJob(jobDefinition); - jobTuningInfo.setParametersToTune(tuningParameterList); - jobTuningInfo.setTunerState("{}"); - jobTuningInfo.setJobType(TuningAlgorithm.JobType.PIG); - - PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); - - JobTuningInfo updatedJobTuningInfo = psoParamGenerator.generateParamSet(jobTuningInfo); - assertTrue("Updated JobTuningInfo: Job definition mismatch", - updatedJobTuningInfo.getTuningJob().equals(jobDefinition)); - assertTrue("Updated JobTuningInfo: Parameter list mismatch", - updatedJobTuningInfo.getParametersToTune().equals(tuningParameterList)); - - String tunerState = updatedJobTuningInfo.getTunerState(); - JsonNode jsonTunerState = Json.parse(tunerState); - - assertTrue("Updated JobTuningInfo: Current population not found", - jsonTunerState.has(JSON_CURRENT_POPULATION_KEY)); - assertTrue("Updated JobTuningInfo: Previous population not found", - jsonTunerState.has(JSON_PREVIOUS_POPULATION_KEY)); - assertTrue("Updated JobTuningInfo: Archive not found", jsonTunerState.has(JSON_ARCHIVE_KEY)); - assertTrue("Updated JobTuningInfo: Random number state not found", jsonTunerState.has(JSON_RND_STATE_KEY)); - - JsonNode currentPopulation = jsonTunerState.get(JSON_CURRENT_POPULATION_KEY); - assertEquals("Current population not of type array", JsonNodeType.ARRAY, currentPopulation.getNodeType()); - assertEquals("Current population size not equal to swarm size", SWARM_SIZE, currentPopulation.size()); - - JsonNode previousPopulation = jsonTunerState.get(JSON_PREVIOUS_POPULATION_KEY); - assertEquals("Previous population not of type array", JsonNodeType.ARRAY, previousPopulation.getNodeType()); - assertEquals("Previous population size not equal to swarm size", 0, previousPopulation.size()); - - JsonNode archive = jsonTunerState.get(JSON_ARCHIVE_KEY); - assertEquals("Archive population not of type array", JsonNodeType.ARRAY, archive.getNodeType()); - assertEquals("Archive population size not equal to swarm size", SWARM_SIZE, archive.size()); - - JsonNode particle = currentPopulation.get(0); - assertTrue("Particle doesn't contain candidate", particle.has(JSON_PARTICLE_CANDIDATE_KEY)); - assertTrue("Particle doesn't contain birthday", particle.has(JSON_PARTICLE_BIRTHDATE_KEY)); - assertTrue("Particle doesn't contain maximize", particle.has(JSON_PARTICLE_MAXIMIZE_KEY)); - assertTrue("Particle doesn't contain fitness", particle.has(JSON_PARTICLE_FITNESS_KEY)); - - JsonNode candidate = particle.get(JSON_PARTICLE_CANDIDATE_KEY); - assertEquals("Candidate not of type array", JsonNodeType.ARRAY, candidate.getNodeType()); - assertEquals("Candidate size not equal to tuning parameters size", numParametersToTune, candidate.size()); - - JsonNode randomNumberState = jsonTunerState.get(JSON_RND_STATE_KEY); - assertEquals("Random number state not of type string", JsonNodeType.STRING, randomNumberState.getNodeType()); - - jobTuningInfo.setTunerState(updatedJobTuningInfo.getTunerState()); - updatedJobTuningInfo = psoParamGenerator.generateParamSet(jobTuningInfo); - assertTrue("Updated JobTuningInfo: Job definition mismatch", - updatedJobTuningInfo.getTuningJob().equals(jobDefinition)); - assertTrue("Updated JobTuningInfo: Parameter list mismatch", - updatedJobTuningInfo.getParametersToTune().equals(tuningParameterList)); - - tunerState = updatedJobTuningInfo.getTunerState(); - jsonTunerState = Json.parse(tunerState); - - assertTrue("Updated JobTuningInfo: Current population not found", - jsonTunerState.has(JSON_CURRENT_POPULATION_KEY)); - assertTrue("Updated JobTuningInfo: Previous population not found", - jsonTunerState.has(JSON_PREVIOUS_POPULATION_KEY)); - assertTrue("Updated JobTuningInfo: Archive not found", jsonTunerState.has(JSON_ARCHIVE_KEY)); - assertTrue("Updated JobTuningInfo: Random number state not found", jsonTunerState.has(JSON_RND_STATE_KEY)); - - currentPopulation = jsonTunerState.get(JSON_CURRENT_POPULATION_KEY); - assertEquals("Current population not of type array", JsonNodeType.ARRAY, currentPopulation.getNodeType()); - assertEquals("Current population size not equal to swarm size", SWARM_SIZE, currentPopulation.size()); - - previousPopulation = jsonTunerState.get(JSON_PREVIOUS_POPULATION_KEY); - assertEquals("Previous population not of type array", JsonNodeType.ARRAY, previousPopulation.getNodeType()); - assertEquals("Previous population size not equal to swarm size", SWARM_SIZE, previousPopulation.size()); - - archive = jsonTunerState.get(JSON_ARCHIVE_KEY); - assertEquals("Archive population not of type array", JsonNodeType.ARRAY, archive.getNodeType()); - assertEquals("Archive population size not equal to swarm size", SWARM_SIZE, archive.size()); - - particle = currentPopulation.get(0); - assertTrue("Particle doesn't contain candidate", particle.has(JSON_PARTICLE_CANDIDATE_KEY)); - assertTrue("Particle doesn't contain birthday", particle.has(JSON_PARTICLE_BIRTHDATE_KEY)); - assertTrue("Particle doesn't contain maximize", particle.has(JSON_PARTICLE_MAXIMIZE_KEY)); - assertTrue("Particle doesn't contain fitness", particle.has(JSON_PARTICLE_FITNESS_KEY)); - - candidate = particle.get(JSON_PARTICLE_CANDIDATE_KEY); - assertEquals("Candidate not of type array", JsonNodeType.ARRAY, candidate.getNodeType()); - assertEquals("Candidate size not equal to tuning parameters size", numParametersToTune, candidate.size()); - - randomNumberState = jsonTunerState.get(JSON_RND_STATE_KEY); - assertEquals("Random number state not of type string", JsonNodeType.STRING, randomNumberState.getNodeType()); - } - }); - } - - @Test - public void getParamsTest() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); - psoParamGenerator.getParams(); - - List jobSuggestedParamSetList = JobSuggestedParamSet.find.where() - .eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED) - .findList(); - assertEquals("Swarm size did not match", SWARM_SIZE, jobSuggestedParamSetList.size()); - - JobSuggestedParamSet jobSuggestedParamSet = jobSuggestedParamSetList.get(0); - - List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, - jobSuggestedParamSet.id) - .findList(); - - TuningAlgorithm tuningAlgorithm = jobSuggestedParamSet.tuningAlgorithm; - List tuningParameterList = TuningParameter.find.where() - .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, tuningAlgorithm.id) - .findList(); - numParametersToTune = tuningParameterList.size(); - - assertEquals("Number of parameters didn't match", numParametersToTune, jobSuggestedParamValueList.size()); - } - }); - } -} diff --git a/test/com/linkedin/drelephant/util/HadoopUtilsTest.scala b/test/com/linkedin/drelephant/util/HadoopUtilsTest.scala deleted file mode 100644 index 753e087a7..000000000 --- a/test/com/linkedin/drelephant/util/HadoopUtilsTest.scala +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.util - -import java.io.{ByteArrayInputStream, IOException} -import java.net.{HttpURLConnection, URL} - -import com.fasterxml.jackson.databind.JsonNode -import com.fasterxml.jackson.databind.node.JsonNodeFactory -import org.apache.hadoop.conf.Configuration -import org.apache.log4j.Logger -import org.mockito.Mockito -import org.scalatest.{FunSpec, Matchers} -import org.scalatest.mockito.MockitoSugar - -class HadoopUtilsTest extends FunSpec with Matchers { - import HadoopUtilsTest._ - - describe("HadoopUtils") { - describe(".findHaNameNodeAddress") { - it("returns the first active HA name node it can find") { - val hadoopUtils = HadoopUtilsTest.newFakeHadoopUtilsForNameNode( - ("sample-ha1.grid.example.com", ("sample-ha1.grid.example.com", "standby")), - ("sample-ha2.grid.example.com", ("sample-ha2.grid.example.com", "active")) - ) - val conf = new Configuration(false) - conf.addResource("core-site.xml") - val haNameNodeAddress = hadoopUtils.findHaNameNodeAddress(conf) - haNameNodeAddress should be(Some("sample-ha2.grid.example.com:50070")) - } - - it("returns no HA name node if one isn't configured") { - val hadoopUtils = HadoopUtilsTest.newFakeHadoopUtilsForNameNode( - ("sample-ha1.grid.example.com", ("sample-ha1.grid.example.com", "standby")), - ("sample-ha2.grid.example.com", ("sample-ha2.grid.example.com", "active")) - ) - val conf = new Configuration(false) - val haNameNodeAddress = hadoopUtils.findHaNameNodeAddress(conf) - haNameNodeAddress should be(None) - } - } - - describe(".httpNameNodeAddress") { - it("returns the default name node") { - val hadoopUtils = HadoopUtilsTest.newFakeHadoopUtilsForNameNode( - ("sample-ha1.grid.example.com", ("sample-ha1.grid.example.com", "standby")), - ("sample-ha2.grid.example.com", ("sample-ha2.grid.example.com", "active")) - ) - val conf = new Configuration(false) - conf.addResource("core-site.xml") - val haNameNodeAddress = hadoopUtils.httpNameNodeAddress(conf) - haNameNodeAddress should be(Some("sample.grid.example.com:50070")) - } - } - - describe(".isActiveNameNode") { - it("returns true for active name nodes") { - val hadoopUtils = - newFakeHadoopUtilsForNameNode(Map(("nn1.grid.example.com", ("nn1-ha1.grid.example.com", "active")))) - hadoopUtils.isActiveNameNode("nn1.grid.example.com") should be(true) - } - - it("returns false for standby name nodes") { - val hadoopUtils = - newFakeHadoopUtilsForNameNode(Map(("nn1.grid.example.com", ("nn1-ha1.grid.example.com", "standby")))) - hadoopUtils.isActiveNameNode("nn1.grid.example.com") should be(false) - } - } - } -} - -object HadoopUtilsTest extends MockitoSugar { - import scala.annotation.varargs - - @varargs - def newFakeHadoopUtilsForNameNode(nameNodeHostsAndStatesByJmxHost: (String, (String, String))*): HadoopUtils = - newFakeHadoopUtilsForNameNode(nameNodeHostsAndStatesByJmxHost.toMap) - - def newFakeHadoopUtilsForNameNode(nameNodeHostsAndStatesByJmxHost: Map[String, (String, String)]): HadoopUtils = - new HadoopUtils { - override lazy val logger = mock[Logger] - - override def newAuthenticatedConnection(url: URL): HttpURLConnection = { - val conn = mock[HttpURLConnection] - val jmxHost = url.getHost - nameNodeHostsAndStatesByJmxHost.get(jmxHost) match { - case Some((host, state)) => { - val jsonNode = newFakeNameNodeStatus(host, state) - val bytes = jsonNode.toString.getBytes("UTF-8") - Mockito.when(conn.getInputStream()).thenReturn(new ByteArrayInputStream(bytes)) - } - case None => { - Mockito.when(conn.getInputStream()).thenThrow(new IOException()) - } - } - conn - } - } - - def newFakeNameNodeStatus(host: String, state: String): JsonNode = { - val jsonNodeFactory = JsonNodeFactory.instance; - - val beanJsonNode = - jsonNodeFactory.objectNode() - .put("name", "Hadoop:service=NameNode, name=NameNodeStatus") - .put("modelerType", "org.apache.hadoop.hdfs.server.namenode.NameNode") - .put("NNRole", "NameNode") - .put("HostAndPort", "s${host}:9000") - .put("SecurityEnabled", "true") - .put("State", state) - - val beansJsonNode = - jsonNodeFactory.arrayNode().add(beanJsonNode) - - jsonNodeFactory.objectNode().set("beans", beansJsonNode) - } -} diff --git a/test/com/linkedin/drelephant/util/InfoExtractorTest.java b/test/com/linkedin/drelephant/util/InfoExtractorTest.java deleted file mode 100644 index b1c262dbf..000000000 --- a/test/com/linkedin/drelephant/util/InfoExtractorTest.java +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.util; - -import com.linkedin.drelephant.analysis.ApplicationType; -import com.linkedin.drelephant.analysis.HadoopApplicationData; -import com.linkedin.drelephant.configurations.scheduler.SchedulerConfigurationData; -import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; -import com.linkedin.drelephant.schedulers.AirflowScheduler; -import com.linkedin.drelephant.schedulers.AzkabanScheduler; -import com.linkedin.drelephant.schedulers.OozieScheduler; -import com.linkedin.drelephant.schedulers.Scheduler; - -import com.linkedin.drelephant.spark.data.SparkApplicationData; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.ApplicationAttemptInfoImpl; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.ApplicationInfoImpl; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummaryImpl; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.JobDataImpl; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.StageDataImpl; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.ApplicationInfo; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.ApplicationAttemptInfo; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.JobData; -import com.linkedin.drelephant.spark.fetchers.statusapiv1.StageData; -import java.util.ArrayList; -import java.util.Properties; -import models.AppResult; - -import org.apache.log4j.Logger; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -import mockit.Expectations; -import mockit.Mocked; -import mockit.integration.junit4.JMockit; -import org.apache.commons.lang.StringUtils; -import org.apache.oozie.client.OozieClient; -import org.apache.oozie.client.WorkflowJob; - -import play.test.FakeApplication; -import play.test.Helpers; - -import scala.Tuple2; -import scala.collection.immutable.Map; -import scala.collection.immutable.HashMap; -import scala.collection.immutable.Vector; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - - -@RunWith(JMockit.class) -public class InfoExtractorTest { - @Mocked - OozieClient oozieClient; - - @Mocked - WorkflowJob workflowJob; - - @Mocked - WorkflowJob parentWorkflowJob; - - private FakeApplication app; - - @Before - public void startApp() throws Exception { - app = Helpers.fakeApplication(Helpers.inMemoryDatabase()); - Helpers.start(app); - } - - @After - public void stopApp() throws Exception { - Helpers.stop(app); - } - - @Test - public void testGetSchedulerInstanceAzkaban() { - Properties properties = new Properties(); - properties.put(AzkabanScheduler.AZKABAN_WORKFLOW_URL, "azkaban_workflow_url"); - properties.put(AzkabanScheduler.AZKABAN_JOB_URL, "azkaba_job_url"); - properties.put(AzkabanScheduler.AZKABAN_EXECUTION_URL, "azkaban_execution_url"); - properties.put(AzkabanScheduler.AZKABAN_ATTEMPT_URL, "azkaba_attempt_url"); - properties.put(AzkabanScheduler.AZKABAN_JOB_NAME, "azkaba_job_name"); - - Scheduler scheduler = InfoExtractor.getSchedulerInstance("id", properties); - assertEquals(true, scheduler instanceof AzkabanScheduler); - assertEquals("azkaban_workflow_url", scheduler.getFlowDefId()); - assertEquals("azkaba_job_url", scheduler.getJobDefId()); - assertEquals("azkaban_execution_url", scheduler.getFlowExecId()); - assertEquals("azkaba_attempt_url", scheduler.getJobExecId()); - assertEquals("azkaba_job_name", scheduler.getJobName()); - assertEquals("azkaban", scheduler.getSchedulerName()); - } - - @Test - public void testGetSchedulerInstanceAirflow() { - Properties properties = new Properties(); - properties.put(AirflowScheduler.AIRFLOW_DAG_ID, "airflow_dag_id"); - properties.put(AirflowScheduler.AIRFLOW_DAG_RUN_EXECUTION_DATE, "airflow_dag_run_execution_date"); - properties.put(AirflowScheduler.AIRFLOW_TASK_ID, "airflow_task_id"); - properties.put(AirflowScheduler.AIRFLOW_TASK_INSTANCE_EXECUTION_DATE, "airflow_task_instance_execution_date"); - - Scheduler scheduler = InfoExtractor.getSchedulerInstance("id", properties); - assertEquals(true, scheduler instanceof AirflowScheduler); - assertEquals("airflow_dag_id", scheduler.getFlowDefId()); - assertEquals("airflow_dag_id/airflow_dag_run_execution_date", scheduler.getFlowExecId()); - assertEquals("airflow_dag_id/airflow_task_id", scheduler.getJobDefId()); - assertEquals("airflow_dag_id/airflow_dag_run_execution_date/airflow_task_id/airflow_task_instance_execution_date", scheduler.getJobExecId()); - assertEquals("airflow_task_id", scheduler.getJobName()); - assertEquals("airflow", scheduler.getSchedulerName()); - } - - @Test - public void testGetSchedulerInstanceOozie() throws Exception { - final String jobInfo = "0004167-160629080632562-oozie-oozi-W"; - final String jobParentInfo = "0004166-160629080632562-oozie-oozi-W"; - Properties properties = new Properties(); - properties.put("oozie.action.id", jobInfo + "@some-action"); - properties.put("oozie.job.id", jobInfo); - - new Expectations() {{ - workflowJob.getId(); - result = jobInfo; - - workflowJob.getParentId(); - result = jobParentInfo; - - oozieClient.getJobInfo(jobInfo); - result = workflowJob; - - parentWorkflowJob.getId(); - result = jobParentInfo; - - parentWorkflowJob.getParentId(); - result = null; - - oozieClient.getJobInfo(jobParentInfo); - result = parentWorkflowJob; - }}; - - Scheduler scheduler = InfoExtractor.getSchedulerInstance("id", properties); - assertEquals(true, scheduler instanceof OozieScheduler); - assertEquals("oozie", scheduler.getSchedulerName()); - assertEquals(jobParentInfo, scheduler.getFlowDefId()); - assertEquals(jobParentInfo, scheduler.getFlowExecId()); - assertEquals(jobInfo, scheduler.getJobDefId()); - assertEquals(jobInfo, scheduler.getJobExecId()); - assertEquals(jobInfo, scheduler.getJobName()); - } - - @Test - public void testGetSchedulerInstanceNull() { - Properties properties = new Properties(); - - Scheduler scheduler = InfoExtractor.getSchedulerInstance("id", properties); - assertEquals(null, scheduler); - } - - @Test - public void testLoadSchedulerInfo() { - Properties properties = new Properties(); - properties.put(AzkabanScheduler.AZKABAN_JOB_URL, - "https://grid.example.com:9000/manager?project=project-name&flow=flow-name&job=job-name"); - properties.put(AzkabanScheduler.AZKABAN_ATTEMPT_URL, - "https://grid.example.com:9000/executor?execid=123456&job=job-name&attempt=0"); - properties.put(AzkabanScheduler.AZKABAN_WORKFLOW_URL, - "https://grid.example.com:9000/manager?project=project-name&flow=flow-name"); - properties.put(AzkabanScheduler.AZKABAN_EXECUTION_URL, - "https://grid.example.com:9000/executor?execid=123456"); - properties.put(AzkabanScheduler.AZKABAN_JOB_NAME, "job-name"); - - SchedulerConfigurationData schedulerConfigurationData = new SchedulerConfigurationData("azkaban", null, null); - - Scheduler scheduler = new AzkabanScheduler("id", properties, schedulerConfigurationData); - - AppResult result = new AppResult(); - - HadoopApplicationData data = - new HadoopApplicationData() { - String appId = "application_5678"; - Properties conf = new Properties(); - ApplicationType applicationType = new ApplicationType("foo"); - - @Override - public String getAppId() { - return appId; - } - - @Override - public Properties getConf() { - return conf; - } - - @Override - public ApplicationType getApplicationType() { - return applicationType; - } - - @Override - public boolean isEmpty() { - return false; - } - }; - - InfoExtractor.loadSchedulerInfo(result, data, scheduler); - - assertEquals(result.scheduler, "azkaban"); - assertFalse(StringUtils.isEmpty(result.getJobExecId())); - assertFalse(StringUtils.isEmpty(result.getJobDefId())); - assertFalse(StringUtils.isEmpty(result.getFlowExecId())); - assertFalse(StringUtils.isEmpty(result.getFlowDefId())); - assertFalse(StringUtils.isEmpty(result.getJobExecUrl())); - assertFalse(StringUtils.isEmpty(result.getJobDefUrl())); - assertFalse(StringUtils.isEmpty(result.getFlowExecUrl())); - assertFalse(StringUtils.isEmpty(result.getFlowDefUrl())); - } - - @Test - public void testLoadInfoMapReduce() { - final String JOB_DEF_URL = "https://grid.example.com:9000/manager?project=project-name&flow=flow-name&job=job-name"; - final String JOB_EXEC_URL = "https://grid.example.com:9000/executor?execid=123456&job=job-name&attempt=0"; - final String FLOW_DEF_URL = "https://grid.example.com:9000/manager?project=project-name&flow=flow-name"; - final String FLOW_EXEC_URL = "https://grid.example.com:9000/executor?execid=123456"; - final String JOB_NAME = "job-name"; - Properties properties = new Properties(); - properties.put(AzkabanScheduler.AZKABAN_JOB_URL, JOB_DEF_URL); - properties.put(AzkabanScheduler.AZKABAN_ATTEMPT_URL, JOB_EXEC_URL ); - properties.put(AzkabanScheduler.AZKABAN_WORKFLOW_URL, FLOW_DEF_URL); - properties.put(AzkabanScheduler.AZKABAN_EXECUTION_URL, FLOW_EXEC_URL); - properties.put(AzkabanScheduler.AZKABAN_JOB_NAME, JOB_NAME); - - AppResult result = new AppResult(); - - HadoopApplicationData data = - (new MapReduceApplicationData()).setAppId("application_5678").setJobConf(properties); - - InfoExtractor.loadInfo(result, data); - - assertTrue(result.jobDefId.equals(JOB_DEF_URL)); - assertTrue(result.jobExecId.equals(JOB_EXEC_URL)); - assertTrue(result.flowDefId.equals(FLOW_DEF_URL)); - assertTrue(result.flowExecId.equals(FLOW_EXEC_URL)); - } - - @Test - public void testLoadInfoSpark() { - final String JOB_DEF_URL = "https://grid.example.com:9000/manager?project=project-name&flow=flow-name&job=job-name"; - final String JOB_EXEC_URL = "https://grid.example.com:9000/executor?execid=123456&job=job-name&attempt=0"; - final String FLOW_DEF_URL = "https://grid.example.com:9000/manager?project=project-name&flow=flow-name"; - final String FLOW_EXEC_URL = "https://grid.example.com:9000/executor?execid=123456"; - final String JAVA_EXTRA_OPTIONS = "spark.driver.extraJavaOptions"; - Map properties = new HashMap(); - properties = properties.$plus(new Tuple2(JAVA_EXTRA_OPTIONS, "-Dazkaban.link.workflow.url=" + FLOW_DEF_URL + - " -Dazkaban.link.job.url=" + JOB_DEF_URL + - " -Dazkaban.link.execution.url=" + FLOW_EXEC_URL + - " -Dazkaban.link.attempt.url=" + JOB_EXEC_URL)); - - AppResult result = new AppResult(); - - HadoopApplicationData data = new SparkApplicationData("application_5678", - properties, - new ApplicationInfoImpl("", "", new Vector(0,1,0)), - new Vector(0,1,0), - new Vector(0,1,0), - new Vector(0,1,0)); - - InfoExtractor.loadInfo(result, data); - - assertTrue(result.jobDefId.equals(JOB_DEF_URL)); - assertTrue(result.jobExecId.equals(JOB_EXEC_URL)); - assertTrue(result.flowDefId.equals(FLOW_DEF_URL)); - assertTrue(result.flowExecId.equals(FLOW_EXEC_URL)); - } - - @Test - public void testLoadInfoSparkNoConfig() { - Map properties = new HashMap(); - - AppResult result = new AppResult(); - - HadoopApplicationData data = new SparkApplicationData("application_5678", - properties, - new ApplicationInfoImpl("", "", new Vector(0,1,0)), - new Vector(0,1,0), - new Vector(0,1,0), - new Vector(0,1,0)); - - // test to make sure loadInfo does not throw exception if properties are not defined - InfoExtractor.loadInfo(result, data); - - assertTrue(result.jobDefId.isEmpty()); - assertTrue(result.jobExecId.isEmpty()); - assertTrue(result.flowDefId.isEmpty()); - assertTrue(result.flowExecId.isEmpty()); - } -} diff --git a/test/com/linkedin/drelephant/util/MemoryFormatUtilsTest.java b/test/com/linkedin/drelephant/util/MemoryFormatUtilsTest.java deleted file mode 100644 index d7de08a11..000000000 --- a/test/com/linkedin/drelephant/util/MemoryFormatUtilsTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.util; - -import java.util.Random; -import junit.framework.TestCase; - - -/** - * This class test MemoryFormatUtils - */ -public class MemoryFormatUtilsTest extends TestCase { - public void testBytesToString() { - // Check integer values - assertEquals("0 B", MemoryFormatUtils.bytesToString(0L)); - assertEquals("1 B", MemoryFormatUtils.bytesToString(1L)); - assertEquals("1 KB", MemoryFormatUtils.bytesToString(1L << 10)); - assertEquals("1 MB", MemoryFormatUtils.bytesToString(1L << 20)); - assertEquals("1 GB", MemoryFormatUtils.bytesToString(1L << 30)); - assertEquals("1,000 GB", MemoryFormatUtils.bytesToString(1000L << 30)); - assertEquals("1 TB", MemoryFormatUtils.bytesToString(1L << 40)); - assertEquals("1,024 TB", MemoryFormatUtils.bytesToString(1L << 50)); - - // Check double values - assertEquals("8.79 KB", MemoryFormatUtils.bytesToString(9000L)); - assertEquals("8.79 MB", MemoryFormatUtils.bytesToString(9000L << 10)); - assertEquals("8.79 GB", MemoryFormatUtils.bytesToString(9000L << 20)); - assertEquals("8.79 TB", MemoryFormatUtils.bytesToString(9000L << 30)); - assertEquals("87,890.62 TB", MemoryFormatUtils.bytesToString((9000L * 10000) << 30)); - } - - public void testScientificStringNumberToBytes() { - long expectedBytes = (long ) (8.79 * (1 << 20)); - assertEquals(expectedBytes, MemoryFormatUtils.stringToBytes("8.79e0 MB")); - assertEquals(expectedBytes, MemoryFormatUtils.stringToBytes("8.79E0 MB")); - assertEquals(expectedBytes, MemoryFormatUtils.stringToBytes("0.879e1 MB")); - assertEquals(expectedBytes, MemoryFormatUtils.stringToBytes("+0.879e+1 MB")); - assertEquals(expectedBytes, MemoryFormatUtils.stringToBytes("87.9e-1 MB")); - assertEquals(expectedBytes, MemoryFormatUtils.stringToBytes("8,790e-3 MB")); - assertEquals(expectedBytes, MemoryFormatUtils.stringToBytes("8,790,000e-6 MB")); - } - - public void testStringToBytes() { - // Null - assertEquals(0L, MemoryFormatUtils.stringToBytes(null)); - - // Integer tests - String[] units = new String[]{"", "K", "M", "G", "T"}; - long[] multiplers = new long[]{1L, 1L << 10, 1L << 20, 1L << 30, 1L << 40}; - Random random = new Random(System.currentTimeMillis()); - for (int i = 0; i < units.length; i++) { - String unit = units[i]; - - // Int values - int[] integerNums = - units.equals("T") ? new int[]{0, random.nextInt(1024), 2048} : new int[]{0, random.nextInt(1024)}; - for (int num : integerNums) { - long expectedNum = num * multiplers[i]; - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit)); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit)); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit + "b")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit + "B")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit + "B")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit + "b")); - - unit = unit.toLowerCase(); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit)); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit)); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit + "b")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit + "B")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit + "B")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit + "b")); - } - - // Double values, notice that converting double values to long might lose some precision - double[] doublleNums = new double[]{random.nextDouble(), random.nextDouble() + random.nextInt(1024), 8.79d}; - for (double num : doublleNums) { - long expectedNum = (long) (num * multiplers[i]); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit)); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit)); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit + "b")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit + "B")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit + "B")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit + "b")); - - unit = unit.toLowerCase(); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit)); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit)); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit + "b")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + unit + "B")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit + "B")); - assertEquals(expectedNum, MemoryFormatUtils.stringToBytes(num + " " + unit + "b")); - } - } - } -} diff --git a/test/com/linkedin/drelephant/util/SparkUtilsTest.scala b/test/com/linkedin/drelephant/util/SparkUtilsTest.scala deleted file mode 100644 index f8efab9d5..000000000 --- a/test/com/linkedin/drelephant/util/SparkUtilsTest.scala +++ /dev/null @@ -1,319 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.util - -import java.io.{ByteArrayInputStream, ByteArrayOutputStream, InputStream} -import java.net.URI - -import org.apache.commons.io.IOUtils -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FSDataInputStream, FileStatus, FileSystem, Path, PathFilter, PositionedReadable} -import org.apache.hadoop.io.compress.CompressionInputStream -import org.apache.log4j.Logger -import org.apache.spark.SparkConf -import org.apache.spark.io.LZ4CompressionCodec -import org.mockito.BDDMockito -import org.scalatest.{FunSpec, OptionValues} -import org.scalatest.mockito.MockitoSugar -import org.xerial.snappy.SnappyOutputStream - - -class SparkUtilsTest extends FunSpec with org.scalatest.Matchers with OptionValues with MockitoSugar { - describe("SparkUtils") { - describe(".fileSystemAndPathForEventLogDir") { - it("returns a filesystem + path based on uri from fetcherConfg") { - val hadoopConfiguration = new Configuration(false) - val sparkConf = new SparkConf() - val sparkUtils = new SparkUtils { - override lazy val logger = mock[Logger] - override lazy val hadoopUtils = mock[HadoopUtils] - override lazy val defaultEnv = Map.empty[String, String] - } - - val (fs, path) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, - sparkConf, - Some("webhdfs://nn1.grid.example.com:50070/logs/spark")) - fs.getUri.toString should be("webhdfs://nn1.grid.example.com:50070") - path should be(new Path("/logs/spark")) - } - - it("returns a webhdfs filesystem + path based on spark.eventLog.dir when it is a webhdfs URL") { - val hadoopConfiguration = new Configuration(false) - val sparkConf = new SparkConf().set("spark.eventLog.dir", "webhdfs://nn1.grid.example.com:50070/logs/spark") - val sparkUtils = new SparkUtils { - override lazy val logger = mock[Logger] - override lazy val hadoopUtils = mock[HadoopUtils] - override lazy val defaultEnv = Map.empty[String, String] - } - - val (fs, path) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) - fs.getUri.toString should be("webhdfs://nn1.grid.example.com:50070") - path should be(new Path("/logs/spark")) - } - - it("returns a webhdfs filesystem + path based on spark.eventLog.dir when it is an hdfs URL") { - val hadoopConfiguration = new Configuration(false) - val sparkConf = new SparkConf().set("spark.eventLog.dir", "hdfs://nn1.grid.example.com:9000/logs/spark") - val sparkUtils = new SparkUtils { - override lazy val logger = mock[Logger] - override lazy val hadoopUtils = mock[HadoopUtils] - override lazy val defaultEnv = Map.empty[String, String] - } - - val (fs, path) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) - fs.getUri.toString should be("webhdfs://nn1.grid.example.com:50070") - path should be(new Path("/logs/spark")) - } - - it("returns a webhdfs filesystem + path based on dfs.nameservices and spark.eventLog.dir when the latter is a path and the dfs.nameservices is configured and available") { - val hadoopConfiguration = new Configuration(false) - hadoopConfiguration.set("dfs.nameservices", "sample") - hadoopConfiguration.set("dfs.ha.namenodes.sample", "ha1,ha2") - hadoopConfiguration.set("dfs.namenode.http-address.sample.ha1", "sample-ha1.grid.example.com:50070") - hadoopConfiguration.set("dfs.namenode.http-address.sample.ha2", "sample-ha2.grid.example.com:50070") - - val sparkConf = new SparkConf().set("spark.eventLog.dir", "/logs/spark") - - val sparkUtils = new SparkUtils { - override lazy val logger = mock[Logger] - - override lazy val hadoopUtils = HadoopUtilsTest.newFakeHadoopUtilsForNameNode( - ("sample-ha1.grid.example.com", ("sample-ha1.grid.example.com", "standby")), - ("sample-ha2.grid.example.com", ("sample-ha2.grid.example.com", "active")) - ) - - override lazy val defaultEnv = Map.empty[String, String] - } - - val (fs, path) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) - fs.getUri.toString should be("webhdfs://sample-ha2.grid.example.com:50070") - path should be(new Path("/logs/spark")) - } - - it("returns a webhdfs filesystem + path based on dfs.nameservices and spark.eventLog.dir when the latter is a path and the dfs.nameservices is configured but unavailable") { - val hadoopConfiguration = new Configuration(false) - hadoopConfiguration.set("dfs.nameservices", "sample") - hadoopConfiguration.set("dfs.ha.namenodes.sample", "ha1,ha2") - hadoopConfiguration.set("dfs.namenode.http-address.sample.ha1", "sample-ha1.grid.example.com:50070") - hadoopConfiguration.set("dfs.namenode.http-address.sample.ha2", "sample-ha2.grid.example.com:50070") - hadoopConfiguration.set("dfs.namenode.http-address", "sample.grid.example.com:50070") - - val sparkConf = new SparkConf().set("spark.eventLog.dir", "/logs/spark") - - val sparkUtils = new SparkUtils { - override lazy val logger = mock[Logger] - - override lazy val hadoopUtils = HadoopUtilsTest.newFakeHadoopUtilsForNameNode( - ("sample-ha1.grid.example.com", ("sample-ha1.grid.example.com", "standby")), - ("sample-ha2.grid.example.com", ("sample-ha2.grid.example.com", "standby")) - ) - - override lazy val defaultEnv = Map.empty[String, String] - } - - val (fs, path) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) - fs.getUri.toString should be("webhdfs://sample.grid.example.com:50070") - path should be(new Path("/logs/spark")) - } - - it("returns a webhdfs filesystem + path based on dfs.namenode.http-address and spark.eventLog.dir when the latter is a path and dfs.nameservices is not configured") { - val hadoopConfiguration = new Configuration(false) - hadoopConfiguration.set("dfs.namenode.http-address", "sample.grid.example.com:50070") - - val sparkConf = new SparkConf().set("spark.eventLog.dir", "/logs/spark") - - val sparkUtils = new SparkUtils { - override lazy val logger = mock[Logger] - - override lazy val hadoopUtils = HadoopUtilsTest.newFakeHadoopUtilsForNameNode( - ("sample-ha1.grid.example.com", ("sample-ha1.grid.example.com", "standby")), - ("sample-ha2.grid.example.com", ("sample-ha2.grid.example.com", "active")) - ) - - override lazy val defaultEnv = Map.empty[String, String] - } - - val (fs, path) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) - fs.getUri.toString should be("webhdfs://sample.grid.example.com:50070") - path should be(new Path("/logs/spark")) - } - - it("throws an exception when spark.eventLog.dir is a path and no namenode is configured at all") { - val hadoopConfiguration = new Configuration(false) - - val sparkConf = new SparkConf().set("spark.eventLog.dir", "/logs/spark") - - val sparkUtils = new SparkUtils { - override lazy val logger = mock[Logger] - override lazy val hadoopUtils = mock[HadoopUtils] - override lazy val defaultEnv = Map.empty[String, String] - } - - an[Exception] should be thrownBy { sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) } - } - } - - describe(".pathAndCodecforEventLog") { - it("returns the path and codec for the event log, given the base path and app/attempt information") { - val hadoopConfiguration = new Configuration(false) - - val sparkConf = - new SparkConf() - .set("spark.eventLog.dir", "/logs/spark") - .set("spark.eventLog.compress", "true") - - val sparkUtils = SparkUtilsTest.newFakeSparkUtilsForEventLog( - new URI("webhdfs://nn1.grid.example.com:50070"), - new Path("/logs/spark"), - new Path("application_1_1.lz4"), - Array.empty[Byte] - ) - - val (fs, basePath) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) - - val (path, codec) = - sparkUtils.pathAndCodecforEventLog(sparkConf: SparkConf, fs: FileSystem, basePath: Path, "application_1", Some("1")) - - path should be(new Path("webhdfs://nn1.grid.example.com:50070/logs/spark/application_1_1.lz4")) - codec.value should be(a[LZ4CompressionCodec]) - } - it("returns the path and codec for the event log, given the base path and appid. Extracts attempt and codec from path") { - val hadoopConfiguration = new Configuration(false) - - val sparkConf = - new SparkConf() - .set("spark.eventLog.dir", "/logs/spark") - .set("spark.eventLog.compress", "true") - - val sparkUtils = SparkUtilsTest.newFakeSparkUtilsForEventLog( - new URI("webhdfs://nn1.grid.example.com:50070"), - new Path("/logs/spark"), - new Path("application_1_1.lz4"), - Array.empty[Byte] - ) - - val (fs, basePath) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) - - val (path, codec) = - sparkUtils.pathAndCodecforEventLog(sparkConf: SparkConf, fs: FileSystem, basePath: Path, "application_1", None) - - path should be(new Path("webhdfs://nn1.grid.example.com:50070/logs/spark/application_1_1.lz4")) - codec.value should be(a[LZ4CompressionCodec]) - } - } - - describe(".withEventLog") { - it("loans the input stream for the event log") { - val expectedLog = - """{"Event":"SparkListenerApplicationStart","App Name":"app","App ID":"application_1","Timestamp":1,"User":"foo"}""" - - val eventLogBytes = { - val bout = new ByteArrayOutputStream() - for { - in <- resource.managed(new ByteArrayInputStream(expectedLog.getBytes("UTF-8"))) - out <- resource.managed(new SnappyOutputStream(bout)) - } { - IOUtils.copy(in, out) - } - bout.toByteArray - } - - val hadoopConfiguration = new Configuration(false) - - val sparkConf = - new SparkConf() - .set("spark.eventLog.dir", "/logs/spark") - .set("spark.eventLog.compress", "true") - - val sparkUtils = SparkUtilsTest.newFakeSparkUtilsForEventLog( - new URI("webhdfs://nn1.grid.example.com:50070"), - new Path("/logs/spark"), - new Path("application_1_1.snappy"), - eventLogBytes - ) - - val (fs, basePath) = sparkUtils.fileSystemAndPathForEventLogDir(hadoopConfiguration, sparkConf, None) - - val (path, codec) = - sparkUtils.pathAndCodecforEventLog(sparkConf: SparkConf, fs: FileSystem, basePath: Path, "application_1", None) - - sparkUtils.withEventLog(fs, path, codec) { in => - val bout = new ByteArrayOutputStream() - IOUtils.copy(in, bout) - - val actualLog = new String(bout.toByteArray, "UTF-8") - actualLog should be(expectedLog) - } - } - } - } -} - -object SparkUtilsTest extends MockitoSugar { - def newFakeSparkUtilsForEventLog( - fileSystemUri: URI, - basePath: Path, - filename: Path, - bytes: Array[Byte] - ): SparkUtils = new SparkUtils() { - override lazy val logger = mock[Logger] - override lazy val hadoopUtils = mock[HadoopUtils] - override lazy val defaultEnv = Map.empty[String, String] - - override def fileSystemAndPathForEventLogDir( - hadoopConfiguration: Configuration, - sparkConf: SparkConf, - uriFromFetcherConf: Option[String] - ): (FileSystem, Path) = { - val fs = mock[FileSystem] - val expectedPath = new Path(new Path(fileSystemUri), new Path(basePath, filename)) - val expectedFileStatus = { - val fileStatus = mock[FileStatus] - BDDMockito.given(fileStatus.getLen).willReturn(bytes.length.toLong) - BDDMockito.given(fileStatus.getPath()).willReturn(expectedPath) - fileStatus - } - val expectedStatusArray = Array(expectedFileStatus) - - val filter = new PathFilter() { - override def accept(file: Path): Boolean = { - file.getName().startsWith("mockAppId"); - } - } - - BDDMockito.given(fs.getUri).willReturn(fileSystemUri) - BDDMockito.given(fs.exists(expectedPath)).willReturn(true) - BDDMockito.given(fs.getFileStatus(expectedPath)).willReturn(expectedFileStatus) - BDDMockito.given(fs.listStatus(org.mockito.Matchers.refEq(new Path( new Path(fileSystemUri), basePath)), - org.mockito.Matchers.any(filter.getClass))). - willReturn(expectedStatusArray) - BDDMockito.given(fs.open(expectedPath)).willReturn( - new FSDataInputStream(new FakeCompressionInputStream(new ByteArrayInputStream(bytes))) - ) - (fs, basePath) - } - } - - class FakeCompressionInputStream(in: InputStream) extends CompressionInputStream(in) with PositionedReadable { - override def read(): Int = in.read() - override def read(b: Array[Byte], off: Int, len: Int): Int = in.read(b, off, len) - override def read(pos: Long, buffer: Array[Byte], off: Int, len: Int): Int = ??? - override def readFully(pos: Long, buffer: Array[Byte], off: Int, len: Int): Unit = ??? - override def readFully(pos: Long, buffer: Array[Byte]): Unit = ??? - override def resetState(): Unit = ??? - } -} diff --git a/test/com/linkedin/drelephant/util/UtilsTest.java b/test/com/linkedin/drelephant/util/UtilsTest.java deleted file mode 100644 index 69f5509c4..000000000 --- a/test/com/linkedin/drelephant/util/UtilsTest.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.util; - - -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.conf.Configuration; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; - - -/** - * This class tests the Utils class - */ -public class UtilsTest { - - @Test - public void testParseJavaOptions() { - Map options1 = Utils.parseJavaOptions("-Dfoo=bar"); - assertEquals(1, options1.size()); - assertEquals("bar", options1.get("foo")); - - Map options2 = Utils.parseJavaOptions(" -Dfoo=bar -Dfoo2=bar2 -Dfoo3=bar3"); - assertEquals(3, options2.size()); - assertEquals("bar", options2.get("foo")); - assertEquals("bar2", options2.get("foo2")); - assertEquals("bar3", options2.get("foo3")); - } - - @Test - public void testParseJavaOptionsIgnoresNonStandardOptions() { - Map options1 = Utils.parseJavaOptions("-Dfoo=bar -XX:+UseCompressedOops -XX:MaxPermSize=512m -Dfoo2=bar2"); - assertEquals(2, options1.size()); - assertEquals("bar", options1.get("foo")); - assertEquals("bar2", options1.get("foo2")); - } - - @Test(expected = IllegalArgumentException.class) - public void testParseJavaOptionsThrowsIllegalArgumentExceptionForMissingAssignment() { - Utils.parseJavaOptions("-Dfoo"); - } - - @Test(expected = IllegalArgumentException.class) - public void testParseJavaOptionsThrowsIllegalArgumentExceptionForUnexpectedProperties() { - Utils.parseJavaOptions("-foo"); - } - - @Test - public void testGetParam() { - Map paramMap = new HashMap(); - paramMap.put("test_severity_1", "10, 50, 100, 200"); - paramMap.put("test_severity_2", "2, 4, 8"); - paramMap.put("test_param_1", "2&"); - paramMap.put("test_param_2", "2"); - paramMap.put("test_param_3", ""); - paramMap.put("test_param_4", null); - - double limits1[] = Utils.getParam(paramMap.get("test_severity_1"), 4); - assertEquals(10d, limits1[0], 0); - assertEquals(50d, limits1[1], 0); - assertEquals(100d, limits1[2], 0); - assertEquals(200d, limits1[3], 0); - - double limits2[] = Utils.getParam(paramMap.get("test_severity_2"), 4); - assertEquals(null, limits2); - - double limits3[] = Utils.getParam(paramMap.get("test_param_1"), 1); - assertEquals(null, limits3); - - double limits4[] = Utils.getParam(paramMap.get("test_param_2"), 1); - assertEquals(2d, limits4[0], 0); - - double limits5[] = Utils.getParam(paramMap.get("test_param_3"), 1); - assertEquals(null, limits5); - - double limits6[] = Utils.getParam(paramMap.get("test_param_4"), 1); - assertEquals(null, limits6); - } - - @Test - public void testCommaSeparated() { - String commaSeparated1 = Utils.commaSeparated("foo"); - assertEquals("foo", commaSeparated1); - - String commaSeparated2 = Utils.commaSeparated("foo", "bar", ""); - assertEquals("foo,bar", commaSeparated2); - - String commaSeparated3 = Utils.commaSeparated("foo", "bar", null); - assertEquals("foo,bar", commaSeparated3); - - String commaSeparated4 = Utils.commaSeparated(); - assertEquals("", commaSeparated4); - } - - @Test - public void testTruncateField() { - String truncatedField1 = Utils.truncateField("foo-bar", 7, "id"); - assertEquals("foo-bar", truncatedField1); - - String truncatedField2 = Utils.truncateField("foo-bar", 6, "id"); - assertEquals("foo...", truncatedField2); - - String truncatedField3 = Utils.truncateField("foo-bar", -1, "id"); - assertEquals("foo-bar", truncatedField3); - - String truncatedField4 = Utils.truncateField(null, 5, "id"); - assertEquals(null, truncatedField4); - } - - @Test - public void testParseCsKeyValue() { - Map properties = Utils.parseCsKeyValue(""); - assertEquals(0, properties.size()); - - Map properties1 = Utils.parseCsKeyValue("foo=bar"); - assertEquals(1, properties1.size()); - assertEquals("bar", properties1.get("foo")); - - Map properties2 = Utils.parseCsKeyValue("foo1=bar1,foo2=bar2,foo3=bar3"); - assertEquals(3, properties2.size()); - assertEquals("bar1", properties2.get("foo1")); - assertEquals("bar2", properties2.get("foo2")); - assertEquals("bar3", properties2.get("foo3")); - } - - @Test - public void testGetNonNegativeInt() { - Configuration conf = new Configuration(); - conf.set("foo1", "100"); - conf.set("foo2", "-100"); - conf.set("foo3", "0"); - conf.set("foo4", "0.5"); - conf.set("foo5", "9999999999999999"); - conf.set("foo6", "bar"); - - int defaultValue = 50; - assertEquals(100, Utils.getNonNegativeInt(conf, "foo1", defaultValue)); - assertEquals(0, Utils.getNonNegativeInt(conf, "foo2", defaultValue)); - assertEquals(0, Utils.getNonNegativeInt(conf, "foo3", defaultValue)); - assertEquals(defaultValue, Utils.getNonNegativeInt(conf, "foo4", defaultValue)); - assertEquals(defaultValue, Utils.getNonNegativeInt(conf, "foo5", defaultValue)); - assertEquals(defaultValue, Utils.getNonNegativeInt(conf, "foo6", defaultValue)); - assertEquals(defaultValue, Utils.getNonNegativeInt(conf, "foo7", defaultValue)); - } - - @Test - public void testGetNonNegativeLong() { - Configuration conf = new Configuration(); - - conf.set("foo1", "100"); - conf.set("foo2", "-100"); - conf.set("foo3", "0"); - conf.set("foo4", "0.5"); - conf.set("foo5", "9999999999999999"); - conf.set("foo6", "bar"); - - long defaultValue = 50; - assertEquals(100, Utils.getNonNegativeLong(conf, "foo1", defaultValue)); - assertEquals(0, Utils.getNonNegativeLong(conf, "foo2", defaultValue)); - assertEquals(0, Utils.getNonNegativeLong(conf, "foo3", defaultValue)); - assertEquals(defaultValue, Utils.getNonNegativeLong(conf, "foo4", defaultValue)); - assertEquals(9999999999999999L, Utils.getNonNegativeLong(conf, "foo5", defaultValue)); - assertEquals(defaultValue, Utils.getNonNegativeLong(conf, "foo6", defaultValue)); - assertEquals(defaultValue, Utils.getNonNegativeLong(conf, "foo7", defaultValue)); - } - - @Test - public void testFormatStringOrNull() { - assertEquals("Hello world!", Utils.formatStringOrNull("%s %s!", "Hello", "world")); - assertEquals(null, Utils.formatStringOrNull("%s %s!", "Hello", null)); - } - - @Test - public void testGetDurationBreakdown() { - long []durations = {13423,432344,23423562,23,324252,1132141414141L}; - assertEquals("0:00:13", Utils.getDurationBreakdown(durations[0])); - assertEquals("0:07:12", Utils.getDurationBreakdown(durations[1])); - assertEquals("6:30:23", Utils.getDurationBreakdown(durations[2])); - assertEquals("0:00:00", Utils.getDurationBreakdown(durations[3])); - assertEquals("0:05:24", Utils.getDurationBreakdown(durations[4])); - assertEquals("314483:43:34", Utils.getDurationBreakdown(durations[5])); - } - @Test - public void testGetPercentage() { - long []numerators = {10,20,30,40,50}; - long []denominators = {100,200,100,52,70}; - - assertEquals("10.00 %", Utils.getPercentage(numerators[0],denominators[0])); - assertEquals("10.00 %", Utils.getPercentage(numerators[1],denominators[1])); - assertEquals("30.00 %", Utils.getPercentage(numerators[2],denominators[2])); - assertEquals("76.92 %", Utils.getPercentage(numerators[3],denominators[3])); - assertEquals("71.43 %", Utils.getPercentage(numerators[4],denominators[4])); - assertEquals("NaN", Utils.getPercentage(0,0)); - } - - @Test - public void testGetDurationInGBHours() { - - long []durations = {10000, 213234343, 23424, 635322, 213}; - - assertEquals("0.003 GB Hours", Utils.getResourceInGBHours(durations[0])); - assertEquals("57.844 GB Hours", Utils.getResourceInGBHours(durations[1])); - assertEquals("0.006 GB Hours", Utils.getResourceInGBHours(durations[2])); - assertEquals("0.172 GB Hours", Utils.getResourceInGBHours(durations[3])); - assertEquals("0 GB Hours", Utils.getResourceInGBHours(durations[4])); - - } - -} diff --git a/test/common/DBTestUtil.java b/test/common/DBTestUtil.java deleted file mode 100644 index f15455ee4..000000000 --- a/test/common/DBTestUtil.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package common; - -import java.io.FileInputStream; -import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import org.apache.commons.io.IOUtils; -import play.db.DB; - -import static common.TestConstants.TEST_DATA_FILE; -import static common.TestConstants.TEST_AUTO_TUNING_DATA_FILE1;; - - -public class DBTestUtil { - - public static void initDB() throws IOException, SQLException { - initDBUtil(TEST_DATA_FILE); - } - - public static void initAutoTuningDB1() throws IOException, SQLException { - initDBUtil(TEST_AUTO_TUNING_DATA_FILE1); - } - - public static void initDBUtil(String fileName) throws IOException, SQLException { - String query = ""; - FileInputStream inputStream = new FileInputStream(fileName); - - try { - query = IOUtils.toString(inputStream); - } finally { - inputStream.close(); - } - - Connection connection = DB.getConnection(); - - try { - Statement statement = connection.createStatement(); - statement.execute(query); - } finally { - connection.close(); - } - } - -} diff --git a/test/common/TestConstants.java b/test/common/TestConstants.java deleted file mode 100644 index 9541c37a5..000000000 --- a/test/common/TestConstants.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package common; - -public class TestConstants { - - // Test data constants - public static final int TEST_SERVER_PORT = 9001; - public static final String BASE_URL = "http://localhost:" + TEST_SERVER_PORT; - public static final String TEST_DATA_FILE = "test/resources/test-init.sql"; - public static final String TEST_AUTO_TUNING_DATA_FILE1 = "test/resources/tunein-test1.sql"; - - public static final int RESPONSE_TIMEOUT = 3000; // milliseconds - - public static final String TEST_JOB_ID1 = "application_1458194917883_1453361"; - public static final String TEST_JOB_ID2 = "application_1458194917883_1453362"; - public static final String TEST_JOB_NAME = "Email Overwriter"; - public static final String TEST_JOB_TYPE = "HadoopJava"; - public static final String TEST_APP_TYPE = "HadoopJava"; - public static final String TEST_USERNAME = "growth"; - public static final String TEST_DEFAULT_QUEUE_NAME = "default"; - - public static final String TEST_JOB_EXEC_ID1 = - "https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0"; - public static final String TEST_JOB_EXEC_ID2 = - "https://elephant.linkedin.com:8443/executor?execid=1654677&job=overwriter-reminder2&attempt=0"; - - public static final String TEST_FLOW_EXEC_ID1 = - "https://elephant.linkedin.com:8443/executor?execid=1654676"; - public static final String TEST_FLOW_EXEC_ID2 = - "https://elephant.linkedin.com:8443/executor?execid=1654677"; - public static final String TEST_FLOW_DEF_ID1 = - "https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder"; - public static final String TEST_JOB_DEF_ID1 = - "https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2"; - - public static final String TEST_START_TIME1 = "2016-04-18"; - public static final String TEST_END_TIME1 = "2016-04-19"; - // DB connection strings - public static final String DB_DEFAULT_DRIVER_KEY = "db.default.driver"; - public static final String DB_DEFAULT_DRIVER_VALUE = "org.h2.Driver"; - public static final String DB_DEFAULT_URL_KEY = "db.default.url"; - public static final String DB_DEFAULT_URL_VALUE = "jdbc:h2:mem:test;MODE=MySQL;"; - public static final String EVOLUTION_PLUGIN_KEY = "evolutionplugin"; - public static final String EVOLUTION_PLUGIN_VALUE = "enabled"; - public static final String APPLY_EVOLUTIONS_DEFAULT_KEY = "applyEvolutions.default"; - public static final String APPLY_EVOLUTIONS_DEFAULT_VALUE = "true"; - - // Application configurations - public static final String METRICS_ENABLE_KEY = "metrics"; - - // Paths to the rest end-points - public static final String REST_APP_RESULT_PATH = "/rest/job"; - public static final String REST_JOB_EXEC_RESULT_PATH = "/rest/jobexec"; - public static final String REST_FLOW_EXEC_RESULT_PATH = "/rest/flowexec"; - public static final String REST_SEARCH_PATH = "/rest/search"; - public static final String REST_COMPARE_PATH = "/rest/compare"; - public static final String REST_FLOW_GRAPH_DATA_PATH = "/rest/flowgraphdata"; - public static final String REST_JOB_GRAPH_DATA_PATH = "/rest/jobgraphdata"; - public static final String REST_JOB_METRICS_GRAPH_DATA_PATH = "/rest/jobmetricsgraphdata"; - public static final String REST_FLOW_METRICS_GRAPH_DATA_PATH = "/rest/flowmetricsgraphdata"; - public static final String REST_USER_RESOURCE_USAGE_PATH = "/rest/resourceusagedatabyuser"; - public static final String REST_GET_CURRENT_RUN_PARAMETERS = "/rest/getCurrentRunParameters"; - - // Paths to metrics end-points. - public static final String METRICS_ENDPOINT = "/metrics"; - - public static final String DEFAULT_ENCODING = "UTF-8"; - - //Paths to the UI rest end points - public static final String REST_WORKFLOW_SUMMARIES_PATH = "/rest/workflow-summaries"; - public static final String REST_JOB_SUMMARIES_PATH = "/rest/job-summaries"; - public static final String REST_APPLICATION_SUMMARIES_PATH = "/rest/application-summaries"; - public static final String REST_WORKFLOWS_PATH = "/rest/workflows"; - public static final String REST_JOBS_PATH = "/rest/jobs"; - public static final String REST_APPLICATIONS_PATH = "/rest/applications"; - public static final String REST_SEARCH_RESULTS = "/rest/search-results"; - - // Sample mapreduce constants - public static final String FILENAME_JOBCOUNTER = "mrdata/sampleJobCounter.properties"; - public static final String FILENAME_MAPPERTASK = "mrdata/mapperTaskCounter$.properties"; - public static final String FILENAME_REDUCERTASK = "mrdata/reducerTaskCounter$.properties"; - public static final String FILENAME_JOBCONF = "mrdata/sampleJobConf.properties"; - - public static final String TEST_TRACKING_URL = "http://hostname/jobhistory/job/job_1460381439677_0001"; - public static final String TEST_JOBCONF_NAME = "mapred.child.java.opts"; - public static final String TEST_JOBCONF_PATTERN = ".*."; - public static final String TEST_SCHEDULER = "azkaban"; - public static final long TEST_SCORE = 0; - public static final String TEST_SEVERITY = "NONE"; - - // Tuning constant - public static final String JSON_CURRENT_POPULATION_KEY = "current_population"; - public static final String JSON_PREVIOUS_POPULATION_KEY = "prev_population"; - public static final String JSON_ARCHIVE_KEY = "archive"; - public static final String JSON_RND_STATE_KEY = "rnd_state"; - public static final int SWARM_SIZE = 3; - public static final String JSON_PARTICLE_CANDIDATE_KEY = "_candidate"; - public static final String JSON_PARTICLE_BIRTHDATE_KEY = "birthdate"; - public static final String JSON_PARTICLE_MAXIMIZE_KEY = "maximize"; - public static final String JSON_PARTICLE_FITNESS_KEY = "fitness"; - - -} diff --git a/test/common/TestUtil.java b/test/common/TestUtil.java deleted file mode 100644 index a547c2a8c..000000000 --- a/test/common/TestUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package common; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class TestUtil { - - private static final Logger logger = LoggerFactory.getLogger(TestUtil.class); - - // private on purpose - private TestUtil() {} - - public static Properties loadProperties(String filePath) - throws IOException { - Properties properties = new Properties(); - InputStream inputStream = TestUtil.class.getClassLoader().getResourceAsStream(filePath); - if (inputStream == null) { - logger.info("Configuation file not present in classpath. File: " + filePath); - throw new RuntimeException("Unable to read " + filePath); - } - properties.load(inputStream); - logger.info("Configuation file loaded. File: " + filePath); - return properties; - } - -} diff --git a/test/controllers/ApplicationTest.java b/test/controllers/ApplicationTest.java deleted file mode 100644 index 1aae11388..000000000 --- a/test/controllers/ApplicationTest.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package controllers; - -import com.avaje.ebean.Query; -import models.AppResult; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import play.api.mvc.Content; -import play.test.FakeApplication; -import play.test.Helpers; -import views.html.page.homePage; -import views.html.results.searchResults; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -public class ApplicationTest { - - @Test - public void testRenderHomePage() { - Content html = homePage.render(5, 2, 3, searchResults.render("Latest analysis", null)); - assertEquals("text/html", html.contentType()); - assertTrue(html.body().contains("Hello there, I've been busy!")); - assertTrue(html.body().contains("I looked through 5 jobs today.")); - assertTrue(html.body().contains("About 2 of them could use some tuning.")); - assertTrue(html.body().contains("About 3 of them need some serious attention!")); - } - - @Test - public void testRenderSearch() { - Content html = searchResults.render("Latest analysis", null); - assertEquals("text/html", html.contentType()); - assertTrue(html.body().contains("Latest analysis")); - } - - public static FakeApplication app; - - @BeforeClass - public static void startApp() { - app = Helpers.fakeApplication(Helpers.inMemoryDatabase()); - Helpers.start(app); - } - - @AfterClass - public static void stopApp() { - Helpers.stop(app); - } - - @Test - public void testGenerateSearchQuery() { - - Map searchParams = new HashMap(); - - // Null searchParams Check - Query query1 = Application.generateSearchQuery("*", null); - assertNotNull(query1.findList()); - String sql1 = query1.getGeneratedSql(); - assertTrue(sql1.contains("select t0.id c0")); - assertTrue(sql1.contains("from yarn_app_result t0 order by t0.finish_time desc")); - - // No searchParams Check - Query query2 = Application.generateSearchQuery("*", searchParams); - assertNotNull(query2.findList()); - String sql2 = query2.getGeneratedSql(); - assertTrue(sql2.contains("select t0.id c0")); - assertTrue(sql2.contains("from yarn_app_result t0 order by t0.finish_time desc")); - - // Query by username - searchParams.put(Application.USERNAME, "username"); - query2 = Application.generateSearchQuery("*", searchParams); - assertNotNull(query2.findList()); - sql2 = query2.getGeneratedSql(); - assertTrue(sql2.contains("select t0.id c0")); - assertTrue(sql2.contains("from yarn_app_result t0 where")); - assertTrue(sql2.contains("t0.username = ? order by t0.finish_time desc")); - - // Query by queuename - searchParams.put(Application.QUEUE_NAME, "queueName"); - query2 = Application.generateSearchQuery("*", searchParams); - assertNotNull(query2.findList()); - sql2 = query2.getGeneratedSql(); - assertTrue(sql2.contains("select t0.id c0")); - assertTrue(sql2.contains("from yarn_app_result t0 where")); - assertTrue(sql2.contains("t0.queue_name = ? order by t0.finish_time desc")); - - - // Query by jobtype - searchParams.put(Application.JOB_TYPE, "Pig"); - query2 = Application.generateSearchQuery("*", searchParams); - assertNotNull(query2.findList()); - sql2 = query2.getGeneratedSql(); - assertTrue(sql2.contains("select t0.id c0")); - assertTrue(sql2.contains("from yarn_app_result t0 where")); - assertTrue(sql2.contains("t0.username = ?")); - assertTrue(sql2.contains("t0.job_type = ?")); - assertTrue(sql2.contains("order by t0.finish_time desc")); - - // Query by username, jobtype and start time - searchParams.put(Application.STARTED_TIME_BEGIN, "1459713751000"); - searchParams.put(Application.STARTED_TIME_END, "1459713751000"); - Query query3 = Application.generateSearchQuery("*", searchParams); - assertNotNull(query3.findList()); - String sql3 = query3.getGeneratedSql(); - assertTrue(sql3.contains("select t0.id c0")); - assertTrue(sql3.contains("from yarn_app_result t0 where")); - assertTrue(sql3.contains("t0.username = ?")); - assertTrue(sql3.contains("t0.start_time >= ?")); - assertTrue(sql3.contains("t0.start_time <= ?")); - assertTrue(sql3.contains("order by t0.start_time desc")); - - // Query by finish time - searchParams = new HashMap(); - searchParams.put(Application.FINISHED_TIME_BEGIN, "1459713751000"); - searchParams.put(Application.FINISHED_TIME_END, "1459713751000"); - Query query4 = Application.generateSearchQuery("*", searchParams); - assertNotNull(query4.findList()); - String sql4 = query4.getGeneratedSql(); - assertTrue(sql4.contains("select t0.id c0")); - assertTrue(sql4.contains("from yarn_app_result t0 where")); - assertTrue(sql4.contains("t0.finish_time >= ?")); - assertTrue(sql4.contains("t0.finish_time <= ?")); - assertTrue(sql4.contains("order by t0.finish_time desc")); - } -} diff --git a/test/controllers/MetricsControllerTest.java b/test/controllers/MetricsControllerTest.java deleted file mode 100644 index f129ecb67..000000000 --- a/test/controllers/MetricsControllerTest.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package controllers; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.junit.Before; -import org.junit.Test; -import play.libs.WS; -import play.test.FakeApplication; - -import static common.TestConstants.*; -import static play.test.Helpers.fakeApplication; -import static play.test.Helpers.running; -import static play.test.Helpers.testServer; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - - -/** - * Tests the class {@link MetricsController}. - */ -public class MetricsControllerTest { - - private static FakeApplication fakeApp; - - @Before - public void setup() { - Map additionalConfig = new HashMap(); - additionalConfig.put(DB_DEFAULT_DRIVER_KEY, DB_DEFAULT_DRIVER_VALUE); - additionalConfig.put(DB_DEFAULT_URL_KEY, DB_DEFAULT_URL_VALUE); - additionalConfig.put(EVOLUTION_PLUGIN_KEY, EVOLUTION_PLUGIN_VALUE); - additionalConfig.put(APPLY_EVOLUTIONS_DEFAULT_KEY, APPLY_EVOLUTIONS_DEFAULT_VALUE); - additionalConfig.put(METRICS_ENABLE_KEY, "true"); - fakeApp = fakeApplication(additionalConfig); - } - - /** - * Test fetching metrics for queue sizes from Dr.Elephant's "/metrics" endpoint. - */ - @Test - public void testQueueSizeMetrics() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - // Initialize the metrics and set queue sizes for the main job queue, first retry queue and - // second retry queue. - MetricsController.init(); - MetricsController.setQueueSize(6); - MetricsController.setRetryQueueSize(4); - MetricsController.setSecondRetryQueueSize(2); - - // Initiate a request to metrics endpoint and verify the response. - JsonNode jsonResponse = getMetricsEndpointResponse(); - JsonNode metricsNode = getAndVerifyJsonNode(jsonResponse, "metrics"); - assertMetricsIntValue( - metricsNode, "AnalyticJob.jobQueue.size", 6); - assertMetricsIntValue( - metricsNode, "AnalyticJob.retryQueue.size", 4); - assertMetricsIntValue( - metricsNode, "AnalyticJob.secondRetryQueue.size", 2); - } - }); - } - - private static JsonNode getMetricsEndpointResponse() { - WS.Response response = WS.url(BASE_URL + METRICS_ENDPOINT). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - return response.asJson(); - } - - private static JsonNode getAndVerifyJsonNode(JsonNode rootNode, String nodeName) { - JsonNode node = rootNode.path(nodeName); - assertNotNull(nodeName + " node should have been returned", node); - return node; - } - - private static void assertMetricsIntValue(JsonNode metricsNode, String metricName, - int expectedValue) { - JsonNode metricNameNode = getAndVerifyJsonNode(metricsNode, metricName); - JsonNode valueNode = metricNameNode.path("value"); - assertNotNull("value node inside " + metricName + " node should have " - + "been returned", valueNode); - assertEquals(expectedValue, valueNode.asInt()); - } -} diff --git a/test/org/apache/spark/deploy/history/SparkDataCollectionTest.java b/test/org/apache/spark/deploy/history/SparkDataCollectionTest.java deleted file mode 100644 index 0ed76b2e5..000000000 --- a/test/org/apache/spark/deploy/history/SparkDataCollectionTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.spark.deploy.history; - -import com.linkedin.drelephant.spark.legacydata.SparkJobProgressData; -import java.io.IOException; -import org.apache.spark.SparkConf; -import org.apache.spark.scheduler.ApplicationEventListener; -import org.apache.spark.scheduler.ReplayListenerBus; -import org.apache.spark.storage.StorageStatusListener; -import org.apache.spark.storage.StorageStatusTrackingListener; -import org.apache.spark.ui.env.EnvironmentListener; -import org.apache.spark.ui.exec.ExecutorsListener; -import org.apache.spark.ui.jobs.JobProgressListener; -import org.apache.spark.ui.storage.StorageListener; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.BufferedInputStream; -import java.io.InputStream; - -import static org.junit.Assert.assertNotNull; - -public class SparkDataCollectionTest { - - private static final String event_log_dir = "spark_event_logs/"; - - @Test - public void testCollectJobProgressData() throws IOException { - SparkDataCollection dataCollection = new SparkDataCollection(); - - InputStream in = new BufferedInputStream( - SparkDataCollectionTest.class.getClassLoader().getResourceAsStream(event_log_dir + "event_log_1")); - dataCollection.load(in, in.toString()); - in.close(); - - SparkJobProgressData jobProgressData = dataCollection.getJobProgressData(); - assertNotNull("can't get job progress data", jobProgressData); - } - -} diff --git a/test/org/apache/spark/deploy/history/SparkFsFetcherTest.scala b/test/org/apache/spark/deploy/history/SparkFsFetcherTest.scala deleted file mode 100644 index 50995b2a8..000000000 --- a/test/org/apache/spark/deploy/history/SparkFsFetcherTest.scala +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.spark.deploy.history - -import java.io.{ByteArrayInputStream, ByteArrayOutputStream} -import java.net.URI - -import com.linkedin.drelephant.analysis.AnalyticJob -import com.linkedin.drelephant.configurations.fetcher.{FetcherConfiguration, FetcherConfigurationData} -import com.linkedin.drelephant.util.{SparkUtils, SparkUtilsTest} -import javax.xml.parsers.DocumentBuilderFactory -import org.apache.commons.io.IOUtils -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.spark.SparkConf -import org.scalatest.{FunSpec, Matchers} -import org.scalatest.mockito.MockitoSugar -import org.w3c.dom.Document -import org.xerial.snappy.SnappyOutputStream - -class SparkFsFetcherTest extends FunSpec with Matchers with MockitoSugar { - import SparkFsFetcherTest._ - - describe("SparkFsFetcher") { - describe("constructor") { - it("handles fetcher configurations with supplied values") { - val fetcher = newFetcher("configurations/fetcher/FetcherConfTest5.xml") - fetcher.eventLogSizeLimitMb should be(50) - } - - it("handles fetcher configurations with empty values") { - val fetcher = newFetcher("configurations/fetcher/FetcherConfTest6.xml") - fetcher.eventLogSizeLimitMb should be(SparkFSFetcher.DEFAULT_EVENT_LOG_SIZE_LIMIT_MB) - } - - it("handles fetcher configurations with missing values") { - val fetcher = newFetcher("configurations/fetcher/FetcherConfTest7.xml") - fetcher.eventLogSizeLimitMb should be(SparkFSFetcher.DEFAULT_EVENT_LOG_SIZE_LIMIT_MB) - } - } - - describe(".fetchData") { - it("returns the data collected from the Spark event log for the given analytic job") { - val eventLogBytes = { - val eventLog = - """{"Event":"SparkListenerApplicationStart","App Name":"app","App ID":"application_1","Timestamp":1,"User":"foo"}""" - val bout = new ByteArrayOutputStream() - for { - in <- resource.managed(new ByteArrayInputStream(eventLog.getBytes("UTF-8"))) - out <- resource.managed(new SnappyOutputStream(bout)) - } { - IOUtils.copy(in, out) - } - bout.toByteArray - } - - val fetcherConfigurationData = newFetcherConfigurationData("configurations/fetcher/FetcherConfTest7.xml") - val fetcher = new SparkFSFetcher(fetcherConfigurationData) { - override lazy val hadoopConfiguration = new Configuration(false) - - override lazy val sparkConf = - new SparkConf() - .set("spark.eventLog.dir", "webhdfs://nn1.grid.example.com:50070/logs/spark") - .set("spark.eventLog.compress", "true") - .set("spark.io.compression.codec", "snappy") - - override lazy val sparkUtils = SparkUtilsTest.newFakeSparkUtilsForEventLog( - new URI("webhdfs://nn1.grid.example.com:50070"), - new Path("/logs/spark"), - new Path("application_1_1.snappy"), - eventLogBytes - ) - - override protected def doAsPrivilegedAction[T](action: () => T): T = action() - } - val analyticJob = new AnalyticJob().setAppId("application_1") - - val data = fetcher.fetchData(analyticJob) - data.getAppId should be("application_1") - - val generalData = data.getGeneralData - generalData.getApplicationId should be("application_1") - generalData.getApplicationName should be("app") - generalData.getSparkUser should be("foo") - } - } - } -} - -object SparkFsFetcherTest { - def newFetcher(confResourcePath: String): SparkFSFetcher = { - val fetcherConfData = newFetcherConfigurationData(confResourcePath) - val fetcherClass = getClass.getClassLoader.loadClass(fetcherConfData.getClassName) - fetcherClass.getConstructor(classOf[FetcherConfigurationData]).newInstance(fetcherConfData).asInstanceOf[SparkFSFetcher] - } - - def newFetcherConfigurationData(confResourcePath: String): FetcherConfigurationData = { - val document = parseDocument(confResourcePath) - val fetcherConf = new FetcherConfiguration(document.getDocumentElement()) - fetcherConf.getFetchersConfigurationData().get(0) - } - - def parseDocument(resourcePath: String): Document = { - val factory = DocumentBuilderFactory.newInstance() - val builder = factory.newDocumentBuilder() - builder.parse(getClass.getClassLoader.getResourceAsStream(resourcePath)) - } -} diff --git a/test/resources/AggregatorConf.xml b/test/resources/AggregatorConf.xml deleted file mode 100644 index 23586d587..000000000 --- a/test/resources/AggregatorConf.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - mapreduce - com.linkedin.drelephant.mapreduce.MapReduceMetricsAggregator - - - spark - com.linkedin.drelephant.spark.SparkMetricsAggregator - - 0.5 - - - diff --git a/test/resources/AutoTuningConf.xml b/test/resources/AutoTuningConf.xml deleted file mode 100644 index 024ce0824..000000000 --- a/test/resources/AutoTuningConf.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - - autotuning.enabled - true - Auto Tuning is enabled or not - - - autotuning.default.allowed_max_resource_usage_percent - 150 - Default value for allowed max resource usage percent - - - autotuning.default.allowed_max_execution_time_percent - 150 - Default value for allowed max execution time percent - - - autotuning.daemon.wait.interval.ms - 600 - Auto tuning daemon wait interval - - - baseline.execution.count - 30 - Baseline to be done on recent number of execution - - - fitness.compute.wait_interval.ms - 0 - Wait time after the job is completed till fitness is computer - - - dr.elephant.api.url - http://ltx1-holdemdre01.grid.linkedin.com:8080 - API URL for Dr Elephant - - - - - - - - diff --git a/test/resources/FetcherConf.xml b/test/resources/FetcherConf.xml deleted file mode 100644 index f81aad5e6..000000000 --- a/test/resources/FetcherConf.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - - - - - - - mapreduce - com.linkedin.drelephant.mapreduce.fetchers.MapReduceFSFetcherHadoop2 - - false - 500 - PST - - - - - - - spark - com.linkedin.drelephant.spark.fetchers.FSFetcher - - - - diff --git a/test/resources/GeneralConf.xml b/test/resources/GeneralConf.xml deleted file mode 100644 index 1d7da9ff2..000000000 --- a/test/resources/GeneralConf.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - drelephant.analysis.thread.count - 1 - Number of threads to analyze the completed jobs - - - drelephant.analysis.fetch.interval - 60000 - Interval between fetches in milliseconds - - - drelephant.analysis.retry.interval - 60000 - Interval between retries in milliseconds - - - drelephant.application.search.match.partial - true - If this property is "false", search will only make exact matches - - - - diff --git a/test/resources/JobTypeConf.xml b/test/resources/JobTypeConf.xml deleted file mode 100644 index c7d002898..000000000 --- a/test/resources/JobTypeConf.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - - Spark - spark - spark.app.id - - - - Pig - mapreduce - pig.script - - - Hive - mapreduce - hive.mapred.mode - - - OozieLauncher - mapreduce - oozie.launcher.action.main.class - - - Cascading - mapreduce - cascading.app.frameworks - - - Voldemort - mapreduce - mapred.reducer.class - voldemort.store.readonly.mr.* - - - Kafka - mapreduce - kafka.url - - - HadoopJava - mapreduce - mapred.child.java.opts - - - - TonY - TONY - tony.application.name - - - diff --git a/test/resources/SchedulerConf.xml b/test/resources/SchedulerConf.xml deleted file mode 100644 index c4a29f5e3..000000000 --- a/test/resources/SchedulerConf.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - airflow - com.linkedin.drelephant.schedulers.AirflowScheduler - - http://localhost:8000 - - - - - azkaban - com.linkedin.drelephant.schedulers.AzkabanScheduler - - - - oozie - com.linkedin.drelephant.schedulers.OozieScheduler - - http://oozie/ - - - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest1.xml b/test/resources/configurations/fetcher/FetcherConfTest1.xml deleted file mode 100644 index e69e71c6b..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest1.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - mapreduce - com.linkedin.drelephant.mapreduce.fetchers.MapReduceFetcherHadoop2 - - - spark - org.apache.spark.deploy.history.SparkFSFetcher - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest10.xml b/test/resources/configurations/fetcher/FetcherConfTest10.xml deleted file mode 100644 index ec593f782..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest10.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - mapreduce - com.linkedin.drelephant.mapreduce.fetchers.MapReduceFSFetcherHadoop2 - - true - 200 - PST - - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest11.xml b/test/resources/configurations/fetcher/FetcherConfTest11.xml deleted file mode 100644 index 432d731a6..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest11.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - mapreduce - com.linkedin.drelephant.mapreduce.fetchers.MapReduceFSFetcherHadoop2 - - - - - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest2.xml b/test/resources/configurations/fetcher/FetcherConfTest2.xml deleted file mode 100644 index b025abe0f..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest2.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - mapreduce - com.linkedin.drelephant.mapreduce.fetchers.MapReduceFetcherHadoop2 - - - spark - org.apache.spark.deploy.history.SparkFSFetcher - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest3.xml b/test/resources/configurations/fetcher/FetcherConfTest3.xml deleted file mode 100644 index 46abe5bc1..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest3.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - mapreduce - - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest4.xml b/test/resources/configurations/fetcher/FetcherConfTest4.xml deleted file mode 100644 index bcb7ebe29..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest4.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - com.linkedin.drelephant.mapreduce.fetchers.MapReduceFetcherHadoop2 - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest5.xml b/test/resources/configurations/fetcher/FetcherConfTest5.xml deleted file mode 100644 index 4004c5e3e..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest5.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - spark - org.apache.spark.deploy.history.SparkFSFetcher - - 50 - - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest6.xml b/test/resources/configurations/fetcher/FetcherConfTest6.xml deleted file mode 100644 index a09588dfc..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest6.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - spark - org.apache.spark.deploy.history.SparkFSFetcher - - - - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest7.xml b/test/resources/configurations/fetcher/FetcherConfTest7.xml deleted file mode 100644 index 1564cb313..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest7.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - spark - org.apache.spark.deploy.history.SparkFSFetcher - - - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest8.xml b/test/resources/configurations/fetcher/FetcherConfTest8.xml deleted file mode 100644 index 00fe9c055..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest8.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - spark - org.apache.spark.deploy.history.SparkFSFetcher - - sample-ha3.grid.example.com:50070,sample-ha4.grid.example.com:50070 - - - diff --git a/test/resources/configurations/fetcher/FetcherConfTest9.xml b/test/resources/configurations/fetcher/FetcherConfTest9.xml deleted file mode 100644 index ef7c2c0be..000000000 --- a/test/resources/configurations/fetcher/FetcherConfTest9.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - mapreduce - com.linkedin.drelephant.mapreduce.fetchers.MapReduceFSFetcherHadoop2 - - diff --git a/test/resources/configurations/heuristic/HeuristicConfTest1.xml b/test/resources/configurations/heuristic/HeuristicConfTest1.xml deleted file mode 100644 index 540b81b00..000000000 --- a/test/resources/configurations/heuristic/HeuristicConfTest1.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - mapreduce - Mapper Skew - com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic - views.html.help.mapreduce.helpMapperSkew - - 10, 50, 100, 200 - 2, 4, 8, 16 - 1/8, 1/4, 1/2, 1 - - - - - mapreduce - Mapper GC - com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic - views.html.help.mapreduce.helpGC - - - - mapreduce - Mapper Time - com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic - views.html.help.mapreduce.helpMapperTime - - - \ No newline at end of file diff --git a/test/resources/configurations/heuristic/HeuristicConfTest2.xml b/test/resources/configurations/heuristic/HeuristicConfTest2.xml deleted file mode 100644 index dd755d696..000000000 --- a/test/resources/configurations/heuristic/HeuristicConfTest2.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - mapreduce - Mapper Data Skew - com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic - views.html.help.mapreduce.helpMapperSkew - - 10, 50, 100, 200 - 2, 4, 8, 16 - 1/8, 1/4, 1/2, 1 - - - - - mapreduce - Mapper GC - com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic - views.html.help.mapreduce.helpGC - - - - mapreduce - Mapper Time - com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic - views.html.help.mapreduce.helpMapperTime - - - diff --git a/test/resources/configurations/heuristic/HeuristicConfTest3.xml b/test/resources/configurations/heuristic/HeuristicConfTest3.xml deleted file mode 100644 index 7f4e5bd72..000000000 --- a/test/resources/configurations/heuristic/HeuristicConfTest3.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - mapreduce - Mapper Data Skew - com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic - views.html.help.mapreduce.helpMapperSkew - - 10, 50, 100, 200 - 2, 4, 8, 16 - 1/8, 1/4, 1/2, 1 - - - - - mapreduce - Mapper GC - com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic - views.html.help.mapreduce.helpGC - - - - mapreduce - Mapper Time - com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic - views.html.help.mapreduce.helpMapperTime - - - \ No newline at end of file diff --git a/test/resources/configurations/heuristic/HeuristicConfTest4.xml b/test/resources/configurations/heuristic/HeuristicConfTest4.xml deleted file mode 100644 index 5686e8b35..000000000 --- a/test/resources/configurations/heuristic/HeuristicConfTest4.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - - - mapreduce - Mapper Data Skew - com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic - views.html.help.mapreduce.helpMapperSkew - - 10, 50, 100, 200 - 2, 4, 8, 16 - 1/8, 1/4, 1/2, 1 - - - - - mapreduce - Mapper GC - com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic - views.html.help.mapreduce.helpGC - - - - mapreduce - Mapper Time - com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic - views.html.help.mapreduce.helpMapperTime - - - \ No newline at end of file diff --git a/test/resources/configurations/heuristic/HeuristicConfTest5.xml b/test/resources/configurations/heuristic/HeuristicConfTest5.xml deleted file mode 100644 index 612aa0834..000000000 --- a/test/resources/configurations/heuristic/HeuristicConfTest5.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - - mapreduce - Mapper Data Skew - com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic - views.html.help.mapreduce.helpMapperSkew - - 10, 50, 100, 200 - 2, 4, 8, 16 - 1/8, 1/4, 1/2, 1 - - - - - Mapper GC - com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic - views.html.help.mapreduce.helpGC - - - - mapreduce - Mapper Time - com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic - views.html.help.mapreduce.helpMapperTime - - - \ No newline at end of file diff --git a/test/resources/configurations/jobtype/JobTypeConfTest1.xml b/test/resources/configurations/jobtype/JobTypeConfTest1.xml deleted file mode 100644 index aaff92137..000000000 --- a/test/resources/configurations/jobtype/JobTypeConfTest1.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - Spark - spark - spark.app.id - - - - Pig - mapreduce - pig.script - - - Hive - mapreduce - hive.mapred.mode - - - Cascading - mapreduce - cascading.app.frameworks - - - HadoopJava - mapreduce - mapred.child.java.opts - - - diff --git a/test/resources/configurations/jobtype/JobTypeConfTest2.xml b/test/resources/configurations/jobtype/JobTypeConfTest2.xml deleted file mode 100644 index 760a11997..000000000 --- a/test/resources/configurations/jobtype/JobTypeConfTest2.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - Spark - spark - spark.app.id - - - - Pig - mapreduce - pig.script - - - mapreduce - hive.mapred.mode - - - Cascading - mapreduce - cascading.app.frameworks - - - HadoopJava - mapreduce - mapred.child.java.opts - - - \ No newline at end of file diff --git a/test/resources/configurations/jobtype/JobTypeConfTest3.xml b/test/resources/configurations/jobtype/JobTypeConfTest3.xml deleted file mode 100644 index 9da172cd7..000000000 --- a/test/resources/configurations/jobtype/JobTypeConfTest3.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - Spark - spark - - - - Pig - mapreduce - - - Hive - mapreduce - hive.mapred.mode - - - Cascading - mapreduce - cascading.app.frameworks - - - HadoopJava - mapreduce - mapred.child.java.opts - - - \ No newline at end of file diff --git a/test/resources/configurations/jobtype/JobTypeConfTest4.xml b/test/resources/configurations/jobtype/JobTypeConfTest4.xml deleted file mode 100644 index b02b767ad..000000000 --- a/test/resources/configurations/jobtype/JobTypeConfTest4.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - Spark - spark - spark.app.id - - - - Pig - pig.script - - - Hive - hive.mapred.mode - - - Cascading - mapreduce - cascading.app.frameworks - - - HadoopJava - mapreduce - mapred.child.java.opts - - - \ No newline at end of file diff --git a/test/resources/configurations/jobtype/JobTypeConfTest5.xml b/test/resources/configurations/jobtype/JobTypeConfTest5.xml deleted file mode 100644 index b1f7953ae..000000000 --- a/test/resources/configurations/jobtype/JobTypeConfTest5.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - Voldemort - mapreduce - mapred.reducer.class - [(voldemort) - - - diff --git a/test/resources/configurations/jobtype/JobTypeConfTest6.xml b/test/resources/configurations/jobtype/JobTypeConfTest6.xml deleted file mode 100644 index 7878dfaaf..000000000 --- a/test/resources/configurations/jobtype/JobTypeConfTest6.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - Spark - spark - spark.app.id - - - - Pig - mapreduce - pig.script - - - - Hive - mapreduce - hive.mapred.mode - - - - Cascading - mapreduce - cascading.app.frameworks - - - HadoopJava - mapreduce - mapred.child.java.opts - - - diff --git a/test/resources/configurations/scheduler/SchedulerConfTest1.xml b/test/resources/configurations/scheduler/SchedulerConfTest1.xml deleted file mode 100644 index ac710874c..000000000 --- a/test/resources/configurations/scheduler/SchedulerConfTest1.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - airflow - com.linkedin.drelephant.schedulers.AirflowScheduler - - http://localhost:8000 - - - - - azkaban - com.linkedin.drelephant.schedulers.AzkabanScheduler - - - diff --git a/test/resources/configurations/scheduler/SchedulerConfTest2.xml b/test/resources/configurations/scheduler/SchedulerConfTest2.xml deleted file mode 100644 index cc64a051f..000000000 --- a/test/resources/configurations/scheduler/SchedulerConfTest2.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - airflow - - http://localhost:8000 - - - - - azkaban - com.linkedin.drelephant.schedulers.AzkabanScheduler - - - diff --git a/test/resources/configurations/scheduler/SchedulerConfTest3.xml b/test/resources/configurations/scheduler/SchedulerConfTest3.xml deleted file mode 100644 index 1b524d59b..000000000 --- a/test/resources/configurations/scheduler/SchedulerConfTest3.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - airflow - com.linkedin.drelephant.schedulers.AirflowScheduler - - http://localhost:8000 - - - - - com.linkedin.drelephant.schedulers.AzkabanScheduler - - - diff --git a/test/resources/core-site.xml b/test/resources/core-site.xml deleted file mode 100644 index 7c26750fb..000000000 --- a/test/resources/core-site.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - - - mapreduce.framework.name - yarn - - - - yarn.resourcemanager.webapp.address - localhost:19888 - - - - dfs.nameservices - sample - - - - dfs.ha.namenodes.sample - ha1,ha2 - - - - dfs.namenode.http-address.sample.ha1 - sample-ha1.grid.example.com:50070 - - - - dfs.namenode.http-address.sample.ha2 - sample-ha2.grid.example.com:50070 - - - - dfs.namenode.http-address - sample.grid.example.com:50070 - - diff --git a/test/resources/elephant.conf b/test/resources/elephant.conf deleted file mode 100644 index c58444367..000000000 --- a/test/resources/elephant.conf +++ /dev/null @@ -1,55 +0,0 @@ -# Play application server port -port=8080 - -# Secret key -# The secret key is used to secure cryptographics functions. -# If you deploy your application to several instances be sure to use the same key! -# You can set the key using the env variable APPLICATION_SECRET or set it here -# Setting it here has higher precedence than env variable. -# application_secret="changeme" - -# Database configuration -db_url=aragrawa-ld1.linkedin.biz -db_name=drelephant -db_user=hadooptuner -db_password=1@Hadooptuner - -# Enable web analytics for the application. -# By default analytics is not turned on. Set this property -# to true and paste the javascript snippet into 'public/analytics/track.js' for -# enabling web analytics for the application. You may configure an analytics application -# like piwik. More information on piwik at piwik.org -enable_analytics=false - -# Set the keytab user and the path to the keytab file if security is enabled. - keytab_user="mkumar1" - keytab_location="/etc/krb5.keytab" - krb_conf_file="/etc/krb5.conf" -# Additional Configuration -# Check https://www.playframework.com/documentation/2.2.x/ProductionConfiguration -jvm_args="-Devolutionplugin=enabled -DapplyEvolutions.default=true" - -# Property enables dropwizard metrics for the application. -# More info on Dropwizard metrics at http://metrics.dropwizard.io -# By default metrics are turned which provides several useful stats for -# the application. The following endpoints can be queried once the application is up. -# /ping -# /healthcheck -# /metrics -metrics=true - -# -# Property enables an agent jar to be loaded along with the Dr. Elephant application. -# The intention with this property is to have the agent publish metrics to other -# applications, although it can do anything else. This property is disabled by default -# and users wishing to make use of it should provide their own implementation of the agent. -# -# More information on writing a jvm agent can be found under the following. -# https://docs.oracle.com/javase/8/docs/api/java/lang/instrument/package-summary.html -# -# Sample configuration of this property is shown below. -# metrics_agent_jar="-javaagent:lib/your_agent.jar" -# -# Sample configuration of the agent with additional options. -# metrics_agent_jar="-javaagent:lib/your_agent.jar=app-name=dr-elephant,app-host=foo" - diff --git a/test/resources/history/done/2018/05/17/000000/job_1526555215992_0001-1526555642677-user-QuasiMonteCarlo-1526555658299-1-1-SUCCEEDED-default-1526555648931.jhist b/test/resources/history/done/2018/05/17/000000/job_1526555215992_0001-1526555642677-user-QuasiMonteCarlo-1526555658299-1-1-SUCCEEDED-default-1526555648931.jhist deleted file mode 100644 index 86c46a12f..000000000 --- a/test/resources/history/done/2018/05/17/000000/job_1526555215992_0001-1526555642677-user-QuasiMonteCarlo-1526555658299-1-1-SUCCEEDED-default-1526555648931.jhist +++ /dev/null @@ -1,27 +0,0 @@ -Avro-Json -{"type":"record","name":"Event","namespace":"org.apache.hadoop.mapreduce.jobhistory","fields":[{"name":"type","type":{"type":"enum","name":"EventType","symbols":["JOB_SUBMITTED","JOB_INITED","JOB_FINISHED","JOB_PRIORITY_CHANGED","JOB_STATUS_CHANGED","JOB_QUEUE_CHANGED","JOB_FAILED","JOB_KILLED","JOB_ERROR","JOB_INFO_CHANGED","TASK_STARTED","TASK_FINISHED","TASK_FAILED","TASK_UPDATED","NORMALIZED_RESOURCE","MAP_ATTEMPT_STARTED","MAP_ATTEMPT_FINISHED","MAP_ATTEMPT_FAILED","MAP_ATTEMPT_KILLED","REDUCE_ATTEMPT_STARTED","REDUCE_ATTEMPT_FINISHED","REDUCE_ATTEMPT_FAILED","REDUCE_ATTEMPT_KILLED","SETUP_ATTEMPT_STARTED","SETUP_ATTEMPT_FINISHED","SETUP_ATTEMPT_FAILED","SETUP_ATTEMPT_KILLED","CLEANUP_ATTEMPT_STARTED","CLEANUP_ATTEMPT_FINISHED","CLEANUP_ATTEMPT_FAILED","CLEANUP_ATTEMPT_KILLED","AM_STARTED"]}},{"name":"event","type":[{"type":"record","name":"JobFinished","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"failedMaps","type":"int"},{"name":"failedReduces","type":"int"},{"name":"totalCounters","type":{"type":"record","name":"JhCounters","fields":[{"name":"name","type":"string"},{"name":"groups","type":{"type":"array","items":{"type":"record","name":"JhCounterGroup","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"counts","type":{"type":"array","items":{"type":"record","name":"JhCounter","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"value","type":"long"}]}}}]}}}]}},{"name":"mapCounters","type":"JhCounters"},{"name":"reduceCounters","type":"JhCounters"}]},{"type":"record","name":"JobInfoChange","fields":[{"name":"jobid","type":"string"},{"name":"submitTime","type":"long"},{"name":"launchTime","type":"long"}]},{"type":"record","name":"JobInited","fields":[{"name":"jobid","type":"string"},{"name":"launchTime","type":"long"},{"name":"totalMaps","type":"int"},{"name":"totalReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"uberized","type":"boolean"}]},{"type":"record","name":"AMStarted","fields":[{"name":"applicationAttemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"containerId","type":"string"},{"name":"nodeManagerHost","type":"string"},{"name":"nodeManagerPort","type":"int"},{"name":"nodeManagerHttpPort","type":"int"}]},{"type":"record","name":"JobPriorityChange","fields":[{"name":"jobid","type":"string"},{"name":"priority","type":"string"}]},{"type":"record","name":"JobQueueChange","fields":[{"name":"jobid","type":"string"},{"name":"jobQueueName","type":"string"}]},{"type":"record","name":"JobStatusChanged","fields":[{"name":"jobid","type":"string"},{"name":"jobStatus","type":"string"}]},{"type":"record","name":"JobSubmitted","fields":[{"name":"jobid","type":"string"},{"name":"jobName","type":"string"},{"name":"userName","type":"string"},{"name":"submitTime","type":"long"},{"name":"jobConfPath","type":"string"},{"name":"acls","type":{"type":"map","values":"string"}},{"name":"jobQueueName","type":"string"},{"name":"workflowId","type":["null","string"],"default":null},{"name":"workflowName","type":["null","string"],"default":null},{"name":"workflowNodeName","type":["null","string"],"default":null},{"name":"workflowAdjacencies","type":["null","string"],"default":null},{"name":"workflowTags","type":["null","string"],"default":null}]},{"type":"record","name":"JobUnsuccessfulCompletion","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"diagnostics","type":["null","string"],"default":null}]},{"type":"record","name":"MapAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"mapFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"ReduceAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"shuffleFinishTime","type":"long"},{"name":"sortFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"finishTime","type":"long"},{"name":"rackname","type":"string"},{"name":"hostname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"}]},{"type":"record","name":"TaskAttemptStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"trackerName","type":"string"},{"name":"httpPort","type":"int"},{"name":"shufflePort","type":"int"},{"name":"containerId","type":"string"},{"name":"locality","type":["null","string"],"default":null},{"name":"avataar","type":["null","string"],"default":null}]},{"type":"record","name":"TaskAttemptUnsuccessfulCompletion","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"status","type":"string"},{"name":"error","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskFailed","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"error","type":"string"},{"name":"failedDueToAttempt","type":["null","string"]},{"name":"status","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null}]},{"type":"record","name":"TaskFinished","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"status","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"successfulAttemptId","type":["null","string"],"default":null}]},{"type":"record","name":"TaskStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"startTime","type":"long"},{"name":"splitLocations","type":"string"}]},{"type":"record","name":"TaskUpdated","fields":[{"name":"taskid","type":"string"},{"name":"finishTime","type":"long"}]}]}]} -{"type":"AM_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.AMStarted":{"applicationAttemptId":"appattempt_1526555215992_0001_000001","startTime":1526555646448,"containerId":"container_1526555215992_0001_01_000001","nodeManagerHost":"172.23.244.182","nodeManagerPort":54690,"nodeManagerHttpPort":8042}}} - -{"type":"JOB_SUBMITTED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobSubmitted":{"jobid":"job_1526555215992_0001","jobName":"QuasiMonteCarlo","userName":"user","submitTime":1526555642677,"jobConfPath":"hdfs://127.0.0.1:9000/tmp/hadoop-yarn/staging/user/.staging/job_1526555215992_0001/job.xml","acls":{},"jobQueueName":"default","workflowId":{"string":""},"workflowName":{"string":""},"workflowNodeName":{"string":""},"workflowAdjacencies":{"string":""},"workflowTags":{"string":""}}}} - -{"type":"JOB_INITED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInited":{"jobid":"job_1526555215992_0001","launchTime":1526555648931,"totalMaps":1,"totalReduces":1,"jobStatus":"INITED","uberized":false}}} - -{"type":"JOB_INFO_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInfoChange":{"jobid":"job_1526555215992_0001","submitTime":1526555642677,"launchTime":1526555648931}}} - -{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1526555215992_0001_m_000000","taskType":"MAP","startTime":1526555648958,"splitLocations":"172.23.244.182"}}} - -{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1526555215992_0001_r_000000","taskType":"REDUCE","startTime":1526555648960,"splitLocations":""}}} - -{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1526555215992_0001_m_000000","taskType":"MAP","attemptId":"attempt_1526555215992_0001_m_000000_0","startTime":1526555651139,"trackerName":"172.23.244.182","httpPort":8042,"shufflePort":13562,"containerId":"container_1526555215992_0001_01_000002","locality":{"string":"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}} - -{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1526555215992_0001_m_000000","attemptId":"attempt_1526555215992_0001_m_000000_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1526555653170,"finishTime":1526555653236,"hostname":"172.23.244.182","port":54690,"rackname":"/default-rack","state":"map","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117831},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":268},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":150},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":40},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"clockSplits":[2041,3,3,3,3,3,3,3,3,3,3,4],"cpuUsages":[0,0,0,0,0,0,0,0,0,0,0,0],"vMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0],"physMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0]}}} - -{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1526555215992_0001_m_000000","taskType":"MAP","finishTime":1526555653236,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117831},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":268},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":150},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":40},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"successfulAttemptId":{"string":"attempt_1526555215992_0001_m_000000_0"}}}} - -{"type":"REDUCE_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1526555215992_0001_r_000000","taskType":"REDUCE","attemptId":"attempt_1526555215992_0001_r_000000_0","startTime":1526555656049,"trackerName":"172.23.244.182","httpPort":8042,"shufflePort":13562,"containerId":"container_1526555215992_0001_01_000003","locality":{"string":"OFF_SWITCH"},"avataar":{"string":"VIRGIN"}}}} - -{"type":"REDUCE_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinished":{"taskid":"task_1526555215992_0001_r_000000","attemptId":"attempt_1526555215992_0001_r_000000_0","taskType":"REDUCE","taskStatus":"SUCCEEDED","shuffleFinishTime":1526555658031,"sortFinishTime":1526555658048,"finishTime":1526555658260,"hostname":"172.23.244.182","port":54690,"rackname":"/default-rack","state":"reduce > reduce","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117776},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":40},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":145752064}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"clockSplits":[2017,17,17,16,17,17,16,17,17,16,17,17],"cpuUsages":[0,0,0,0,0,0,0,0,0,0,0,0],"vMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0],"physMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0]}}} - -{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1526555215992_0001_r_000000","taskType":"REDUCE","finishTime":1526555658260,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117776},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":40},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":145752064}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"successfulAttemptId":{"string":"attempt_1526555215992_0001_r_000000_0"}}}} - -{"type":"JOB_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobFinished":{"jobid":"job_1526555215992_0001","finishTime":1526555658299,"finishedMaps":1,"finishedReduces":1,"failedMaps":0,"failedReduces":0,"totalCounters":{"name":"TOTAL_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":235607},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":268},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":7},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.JobCounter","displayName":"Job Counters ","counts":[{"name":"TOTAL_LAUNCHED_MAPS","displayName":"Launched map tasks","value":1},{"name":"TOTAL_LAUNCHED_REDUCES","displayName":"Launched reduce tasks","value":1},{"name":"DATA_LOCAL_MAPS","displayName":"Data-local map tasks","value":1},{"name":"SLOTS_MILLIS_MAPS","displayName":"Total time spent by all maps in occupied slots (ms)","value":2097},{"name":"SLOTS_MILLIS_REDUCES","displayName":"Total time spent by all reduces in occupied slots (ms)","value":2211},{"name":"MILLIS_MAPS","displayName":"Total time spent by all map tasks (ms)","value":2097},{"name":"MILLIS_REDUCES","displayName":"Total time spent by all reduce tasks (ms)","value":2211},{"name":"VCORES_MILLIS_MAPS","displayName":"Total vcore-milliseconds taken by all map tasks","value":2097},{"name":"VCORES_MILLIS_REDUCES","displayName":"Total vcore-milliseconds taken by all reduce tasks","value":2211},{"name":"MB_MILLIS_MAPS","displayName":"Total megabyte-milliseconds taken by all map tasks","value":2147328},{"name":"MB_MILLIS_REDUCES","displayName":"Total megabyte-milliseconds taken by all reduce tasks","value":2264064}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":150},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":4},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":80},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":347078656}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"mapCounters":{"name":"MAP_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117831},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":268},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":150},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":40},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"reduceCounters":{"name":"REDUCE_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117776},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":40},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":145752064}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]}}}} diff --git a/test/resources/history/done/2018/05/17/000000/job_1526555215992_0001_conf.xml b/test/resources/history/done/2018/05/17/000000/job_1526555215992_0001_conf.xml deleted file mode 100644 index 17576bfaf..000000000 --- a/test/resources/history/done/2018/05/17/000000/job_1526555215992_0001_conf.xml +++ /dev/null @@ -1,168 +0,0 @@ - -mapreduce.job.maxtaskfailures.per.tracker3mapred-default.xmljob.xml -mapreduce.job.speculative.retry-after-speculate15000mapred-default.xmljob.xml -mapreduce.reduce.markreset.buffer.percent0.0mapred-default.xmljob.xml -mapreduce.jobhistory.max-age-ms604800000mapred-default.xmljob.xml -mapreduce.job.ubertask.enablefalsemapred-default.xmljob.xml -mapreduce.input.fileinputformat.numinputfiles1programaticallyjob.xml -mapreduce.jobhistory.datestring.cache.size200000mapred-default.xmljob.xml -mapreduce.job.inputformat.classorg.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormatprogramaticallyjob.xml -mapreduce.job.application.attempt.id1programatically -mapreduce.task.profile.reduce.params${mapreduce.task.profile.params}mapred-default.xmljob.xml -mapreduce.job.split.metainfo.maxsize10000000mapred-default.xmljob.xml -mapreduce.input.lineinputformat.linespermap1mapred-default.xmljob.xml -mapreduce.reduce.shuffle.input.buffer.percent0.70mapred-default.xmljob.xml -mapreduce.reduce.maxattempts4mapred-default.xmljob.xml -mapreduce.jobhistory.admin.acl*mapred-default.xmljob.xml -mapreduce.map.maxattempts4mapred-default.xmljob.xml -mapreduce.jobhistory.cleaner.interval-ms86400000mapred-default.xmljob.xml -mapreduce.job.map.classorg.apache.hadoop.examples.QuasiMonteCarlo$QmcMapperprogramaticallyjob.xml -mapreduce.reduce.cpu.vcores1mapred-default.xmljob.xml -mapreduce.jobtracker.heartbeats.in.second100mapred-default.xmljob.xml -mapreduce.output.fileoutputformat.compressfalsemapred-default.xmljob.xml -mapreduce.shuffle.max.threads0mapred-default.xmljob.xml -mapreduce.map.speculativefalseprogramaticallyjob.xml -mapreduce.job.speculative.slowtaskthreshold1.0mapred-default.xmljob.xml -mapreduce.tasktracker.http.threads40mapred-default.xmljob.xml -mapreduce.jobhistory.http.policyHTTP_ONLYmapred-default.xmljob.xml -mapreduce.task.timeout600000mapred-default.xmljob.xml -hadoop.security.groups.cache.secs300core-default.xmljob.xml -mapreduce.input.fileinputformat.split.minsize0mapred-default.xmljob.xml -mapreduce.shuffle.port13562mapred-default.xmljob.xml -mapreduce.jobtracker.retiredjobs.cache.size1000mapred-default.xmljob.xml -mapreduce.jobtracker.addresslocalmapred-default.xmljob.xml -hadoop.http.authentication.kerberos.principalHTTP/_HOST@LOCALHOSTcore-default.xmljob.xml -mapreduce.task.profile.reduces0-2mapred-default.xmljob.xml -mapreduce.job.end-notification.max.retry.interval5000mapred-default.xml -mapreduce.job.speculative.minimum-allowed-tasks10mapred-default.xmljob.xml -mapreduce.reduce.shuffle.fetch.retry.timeout-ms30000mapred-default.xmljob.xml -mapreduce.tasktracker.dns.nameserverdefaultmapred-default.xmljob.xml -hadoop.security.random.device.file.path/dev/urandomcore-default.xmljob.xml -mapreduce.task.merge.progress.records10000mapred-default.xmljob.xml -hadoop.registry.securefalsecore-default.xmljob.xml -mapreduce.job.counters.max120mapred-default.xmljob.xml -mapreduce.task.profilefalsemapred-default.xmljob.xml -mapreduce.task.combine.progress.records10000mapred-default.xmljob.xml -mapreduce.app-submission.cross-platformfalsemapred-default.xmljob.xml -mapreduce.job.nameQuasiMonteCarloprogramaticallyjob.xml -mapreduce.jobtracker.restart.recoverfalsemapred-default.xmljob.xml -mapreduce.map.skip.proc.count.autoincrtruemapred-default.xmljob.xml -hadoop.security.instrumentation.requires.adminfalsecore-default.xmljob.xml -io.compression.codec.bzip2.librarysystem-nativecore-default.xmljob.xml -mapreduce.task.userlog.limit.kb0mapred-default.xmljob.xml -mapreduce.reduce.speculativefalseprogramaticallyjob.xml -mapreduce.job.submithostnamevasaxena-mn1.linkedin.bizprogramaticallyjob.xml -mapreduce.jobhistory.recovery.enablefalsemapred-default.xmljob.xml -mapreduce.reduce.shuffle.memory.limit.percent0.25mapred-default.xmljob.xml -mapreduce.job.reduce.shuffle.consumer.plugin.classorg.apache.hadoop.mapreduce.task.reduce.Shufflemapred-default.xmljob.xml -mapreduce.job.jvm.numtasks1mapred-default.xmljob.xml -mapreduce.am.max-attempts2mapred-default.xmljob.xml -mapreduce.shuffle.connection-keep-alive.timeout5mapred-default.xmljob.xml -mapreduce.job.reduces1programaticallyjob.xml -mapreduce.tasktracker.reduce.tasks.maximum2mapred-default.xmljob.xml -mapreduce.cluster.local.dir${hadoop.tmp.dir}/mapred/localmapred-default.xmljob.xml -mapreduce.client.output.filterFAILEDmapred-default.xmljob.xml -mapreduce.tasktracker.healthchecker.script.timeout600000mapred-default.xmljob.xml -mapreduce.reduce.shuffle.retry-delay.max.ms60000mapred-default.xmljob.xml -mapreduce.job.committer.setup.cleanup.neededtruemapred-default.xmljob.xml -mapreduce.job.end-notification.retry.attempts0mapred-default.xmljob.xml -mapreduce.map.output.compressfalsemapred-default.xmljob.xml -mapreduce.jobhistory.cleaner.enabletruemapred-default.xmljob.xml -mapreduce.job.running.reduce.limit0mapred-default.xmljob.xml -io.seqfile.local.dir${hadoop.tmp.dir}/io/localcore-default.xmljob.xml -mapreduce.job.queuenamedefaultmapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.directory.search.timeout10000core-default.xmljob.xml -mapreduce.job.token.tracking.ids.enabledfalsemapred-default.xmljob.xml -mapreduce.map.output.compress.codecorg.apache.hadoop.io.compress.DefaultCodecmapred-default.xmljob.xml -mapreduce.job.user.namevasaxenaprogramatically -mapreduce.tasktracker.dns.interfacedefaultmapred-default.xmljob.xml -mapreduce.job.speculative.speculative-cap-total-tasks0.01mapred-default.xmljob.xml -mapreduce.job.output.value.classorg.apache.hadoop.io.LongWritableprogramaticallyjob.xml -mapreduce.job.acl-modify-job mapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.search.filter.group(objectClass=group)core-default.xmljob.xml -mapreduce.map.cpu.vcores1mapred-default.xmljob.xml -mapreduce.map.log.levelINFOmapred-default.xmljob.xml -mapreduce.job.output.key.classorg.apache.hadoop.io.BooleanWritableprogramaticallyjob.xml -mapreduce.job.running.map.limit0mapred-default.xmljob.xml -mapreduce.jobtracker.taskcache.levels2mapred-default.xmljob.xml -mapreduce.client.completion.pollinterval5000mapred-default.xmljob.xml -mapreduce.job.maps1programaticallyjob.xml -mapreduce.job.acl-view-job mapred-default.xmljob.xml -mapreduce.shuffle.max.connections0mapred-default.xmljob.xml -mapreduce.jobtracker.expire.trackers.interval600000mapred-default.xmljob.xml -mapreduce.task.io.sort.factor10mapred-default.xmljob.xml -mapreduce.input.fileinputformat.list-status.num-threads1mapred-default.xmljob.xml -mapreduce.jobhistory.move.thread-count3mapred-default.xmljob.xml -mapreduce.task.profile.params-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%smapred-default.xmljob.xml -mapreduce.jobhistory.keytab/etc/security/keytab/jhs.service.keytabmapred-default.xmljob.xml -mapreduce.reduce.shuffle.fetch.retry.interval-ms1000mapred-default.xmljob.xml -hadoop.user.group.static.mapping.overridesdr.who=;core-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.dir/jobtracker/jobsInfomapred-default.xmljob.xml -hadoop.security.group.mappingorg.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallbackcore-default.xmljob.xml -mapreduce.job.emit-timeline-datafalsemapred-default.xmljob.xml -mapreduce.map.memory.mb1024mapred-default.xmljob.xml -hadoop.http.cross-origin.max-age1800core-default.xmljob.xml -mapreduce.tasktracker.healthchecker.interval60000mapred-default.xmljob.xml -mapreduce.job.speculative.retry-after-no-speculate1000mapred-default.xmljob.xml -mapreduce.task.skip.start.attempts2mapred-default.xmljob.xml -hadoop.hdconfiguration.version1ddefault.xmljob.xml -mapreduce.map.skip.maxrecords0mapred-default.xmljob.xml -mapreduce.jobtracker.system.dir${hadoop.tmp.dir}/mapred/systemmapred-default.xmljob.xml -hadoop.registry.rm.enabledfalsecore-default.xmljob.xml -mapreduce.job.reducer.preempt.delay.sec0mapred-default.xmljob.xml -hadoop.tmp.dir/tmp/hadoop-${user.name}core-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.hours1mapred-default.xmljob.xml -mapreduce.task.io.sort.mb100mapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.search.attr.group.namecncore-default.xmljob.xml -mapreduce.output.fileoutputformat.compress.typeRECORDmapred-default.xmljob.xml -mapreduce.reduce.skip.proc.count.autoincrtruemapred-default.xmljob.xml -mapreduce.job.userlog.retain.hours24mapred-default.xmljob.xml -mapreduce.reduce.input.buffer.percent0.0mapred-default.xmljob.xml -mapreduce.tasktracker.local.dir.minspacestart0mapred-default.xmljob.xml -mapreduce.task.files.preserve.failedtasksfalsemapred-default.xmljob.xml -mapreduce.output.fileoutputformat.compress.codecorg.apache.hadoop.io.compress.DefaultCodecmapred-default.xmljob.xml -map.sort.classorg.apache.hadoop.util.QuickSortmapred-default.xmljob.xml -mapreduce.jobtracker.tasktracker.maxblacklists4mapred-default.xmljob.xml -mapreduce.task.profile.maps0-2mapred-default.xmljob.xml -mapreduce.jobtracker.staging.root.dir${hadoop.tmp.dir}/mapred/stagingmapred-default.xmljob.xml -mapreduce.job.reduce.classorg.apache.hadoop.examples.QuasiMonteCarlo$QmcReducerprogramaticallyjob.xml -hadoop.http.cross-origin.allowed-origins*core-default.xmljob.xml -mapreduce.tasktracker.local.dir.minspacekill0mapred-default.xmljob.xml -mapreduce.tasktracker.taskmemorymanager.monitoringinterval5000mapred-default.xmljob.xml -mapreduce.job.end-notification.retry.interval1000mapred-default.xmljob.xml -mapreduce.jobhistory.loadedjobs.cache.size5mapred-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.activetruemapred-default.xmljob.xml -mapreduce.reduce.shuffle.parallelcopies5mapred-default.xmljob.xml -mapreduce.jobhistory.principaljhs/_HOST@REALM.TLDmapred-default.xmljob.xml -hadoop.http.authentication.simple.anonymous.allowedtruecore-default.xmljob.xml -mapreduce.client.genericoptionsparser.usedtrueprogramaticallyjob.xml -mapreduce.job.ubertask.maxreduces1mapred-default.xmljob.xml -mapreduce.shuffle.transfer.buffer.size131072mapred-default.xmljob.xml -mapreduce.jobtracker.jobhistory.task.numberprogresssplits12mapred-default.xmljob.xml -mapreduce.jobhistory.client.thread-count10mapred-default.xmljob.xml -mapreduce.fileoutputcommitter.algorithm.version1mapred-default.xmljob.xml -mapreduce.reduce.skip.maxgroups0mapred-default.xmljob.xml -mapreduce.reduce.memory.mb1024mapred-default.xmljob.xml -mapreduce.client.progressmonitor.pollinterval1000mapred-default.xmljob.xml -mapreduce.job.outputformat.classorg.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormatprogramaticallyjob.xml -hadoop.http.authentication.signature.secret.file${user.home}/hadoop-http-auth-signature-secretcore-default.xmljob.xml -mapreduce.jobhistory.move.interval-ms180000mapred-default.xmljob.xml -hadoop.security.authorizationfalsecore-default.xmljob.xml -dstorage.policy.enabledtrueddefault.xmljob.xml -mapreduce.shuffle.connection-keep-alive.enablefalsemapred-default.xmljob.xml -hadoop.common.configuration.version0.23.0core-default.xmljob.xml -hadoop.security.groups.negative-cache.secs30core-default.xmljob.xml -mapreduce.ifile.readaheadtruemapred-default.xmljob.xml -mapreduce.job.max.split.locations10mapred-default.xmljob.xml -mapreduce.jobhistory.joblist.cache.size2mapred-site.xmljob.xml -mapreduce.job.end-notification.max.attempts5mapred-default.xml -dimage.transfer.timeout60000ddefault.xmljob.xml -mapreduce.reduce.shuffle.connect.timeout180000mapred-default.xmljob.xml -mapreduce.jobhistory.webapp.address0.0.0.0:19888mapred-default.xmljob.xml -hadoop.http.authentication.token.validity36000core-default.xmljob.xml -mapreduce.tasktracker.map.tasks.maximum2mapred-default.xmljob.xml -mapreduce.job.speculative.speculative-cap-running-tasks0.1mapred-default.xmljob.xml -mapreduce.map.sort.spill.percent0.80mapred-default.xmljob.xml -mapreduce.job.reduce.slowstart.completedmaps0.05mapred-default.xmljob.xml -mapreduce.cluster.temp.dir${hadoop.tmp.dir}/mapred/tempmapred-default.xmljob.xml - diff --git a/test/resources/history/done/2018/05/17/000000/job_1526555215992_0002-1526555694494-user-QuasiMonteCarlo-1526555709175-1-1-SUCCEEDED-default-1526555699708.jhist b/test/resources/history/done/2018/05/17/000000/job_1526555215992_0002-1526555694494-user-QuasiMonteCarlo-1526555709175-1-1-SUCCEEDED-default-1526555699708.jhist deleted file mode 100644 index b82a380d4..000000000 --- a/test/resources/history/done/2018/05/17/000000/job_1526555215992_0002-1526555694494-user-QuasiMonteCarlo-1526555709175-1-1-SUCCEEDED-default-1526555699708.jhist +++ /dev/null @@ -1,27 +0,0 @@ -Avro-Json -{"type":"record","name":"Event","namespace":"org.apache.hadoop.mapreduce.jobhistory","fields":[{"name":"type","type":{"type":"enum","name":"EventType","symbols":["JOB_SUBMITTED","JOB_INITED","JOB_FINISHED","JOB_PRIORITY_CHANGED","JOB_STATUS_CHANGED","JOB_QUEUE_CHANGED","JOB_FAILED","JOB_KILLED","JOB_ERROR","JOB_INFO_CHANGED","TASK_STARTED","TASK_FINISHED","TASK_FAILED","TASK_UPDATED","NORMALIZED_RESOURCE","MAP_ATTEMPT_STARTED","MAP_ATTEMPT_FINISHED","MAP_ATTEMPT_FAILED","MAP_ATTEMPT_KILLED","REDUCE_ATTEMPT_STARTED","REDUCE_ATTEMPT_FINISHED","REDUCE_ATTEMPT_FAILED","REDUCE_ATTEMPT_KILLED","SETUP_ATTEMPT_STARTED","SETUP_ATTEMPT_FINISHED","SETUP_ATTEMPT_FAILED","SETUP_ATTEMPT_KILLED","CLEANUP_ATTEMPT_STARTED","CLEANUP_ATTEMPT_FINISHED","CLEANUP_ATTEMPT_FAILED","CLEANUP_ATTEMPT_KILLED","AM_STARTED"]}},{"name":"event","type":[{"type":"record","name":"JobFinished","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"failedMaps","type":"int"},{"name":"failedReduces","type":"int"},{"name":"totalCounters","type":{"type":"record","name":"JhCounters","fields":[{"name":"name","type":"string"},{"name":"groups","type":{"type":"array","items":{"type":"record","name":"JhCounterGroup","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"counts","type":{"type":"array","items":{"type":"record","name":"JhCounter","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"value","type":"long"}]}}}]}}}]}},{"name":"mapCounters","type":"JhCounters"},{"name":"reduceCounters","type":"JhCounters"}]},{"type":"record","name":"JobInfoChange","fields":[{"name":"jobid","type":"string"},{"name":"submitTime","type":"long"},{"name":"launchTime","type":"long"}]},{"type":"record","name":"JobInited","fields":[{"name":"jobid","type":"string"},{"name":"launchTime","type":"long"},{"name":"totalMaps","type":"int"},{"name":"totalReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"uberized","type":"boolean"}]},{"type":"record","name":"AMStarted","fields":[{"name":"applicationAttemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"containerId","type":"string"},{"name":"nodeManagerHost","type":"string"},{"name":"nodeManagerPort","type":"int"},{"name":"nodeManagerHttpPort","type":"int"}]},{"type":"record","name":"JobPriorityChange","fields":[{"name":"jobid","type":"string"},{"name":"priority","type":"string"}]},{"type":"record","name":"JobQueueChange","fields":[{"name":"jobid","type":"string"},{"name":"jobQueueName","type":"string"}]},{"type":"record","name":"JobStatusChanged","fields":[{"name":"jobid","type":"string"},{"name":"jobStatus","type":"string"}]},{"type":"record","name":"JobSubmitted","fields":[{"name":"jobid","type":"string"},{"name":"jobName","type":"string"},{"name":"userName","type":"string"},{"name":"submitTime","type":"long"},{"name":"jobConfPath","type":"string"},{"name":"acls","type":{"type":"map","values":"string"}},{"name":"jobQueueName","type":"string"},{"name":"workflowId","type":["null","string"],"default":null},{"name":"workflowName","type":["null","string"],"default":null},{"name":"workflowNodeName","type":["null","string"],"default":null},{"name":"workflowAdjacencies","type":["null","string"],"default":null},{"name":"workflowTags","type":["null","string"],"default":null}]},{"type":"record","name":"JobUnsuccessfulCompletion","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"diagnostics","type":["null","string"],"default":null}]},{"type":"record","name":"MapAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"mapFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"ReduceAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"shuffleFinishTime","type":"long"},{"name":"sortFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"finishTime","type":"long"},{"name":"rackname","type":"string"},{"name":"hostname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"}]},{"type":"record","name":"TaskAttemptStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"trackerName","type":"string"},{"name":"httpPort","type":"int"},{"name":"shufflePort","type":"int"},{"name":"containerId","type":"string"},{"name":"locality","type":["null","string"],"default":null},{"name":"avataar","type":["null","string"],"default":null}]},{"type":"record","name":"TaskAttemptUnsuccessfulCompletion","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"status","type":"string"},{"name":"error","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskFailed","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"error","type":"string"},{"name":"failedDueToAttempt","type":["null","string"]},{"name":"status","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null}]},{"type":"record","name":"TaskFinished","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"status","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"successfulAttemptId","type":["null","string"],"default":null}]},{"type":"record","name":"TaskStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"startTime","type":"long"},{"name":"splitLocations","type":"string"}]},{"type":"record","name":"TaskUpdated","fields":[{"name":"taskid","type":"string"},{"name":"finishTime","type":"long"}]}]}]} -{"type":"AM_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.AMStarted":{"applicationAttemptId":"appattempt_1526555215992_0002_000001","startTime":1526555696951,"containerId":"container_1526555215992_0002_01_000001","nodeManagerHost":"172.23.244.182","nodeManagerPort":54690,"nodeManagerHttpPort":8042}}} - -{"type":"JOB_SUBMITTED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobSubmitted":{"jobid":"job_1526555215992_0002","jobName":"QuasiMonteCarlo","userName":"user","submitTime":1526555694494,"jobConfPath":"hdfs://127.0.0.1:9000/tmp/hadoop-yarn/staging/user/.staging/job_1526555215992_0002/job.xml","acls":{},"jobQueueName":"default","workflowId":{"string":""},"workflowName":{"string":""},"workflowNodeName":{"string":""},"workflowAdjacencies":{"string":""},"workflowTags":{"string":""}}}} - -{"type":"JOB_INITED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInited":{"jobid":"job_1526555215992_0002","launchTime":1526555699708,"totalMaps":1,"totalReduces":1,"jobStatus":"INITED","uberized":false}}} - -{"type":"JOB_INFO_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInfoChange":{"jobid":"job_1526555215992_0002","submitTime":1526555694494,"launchTime":1526555699708}}} - -{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1526555215992_0002_m_000000","taskType":"MAP","startTime":1526555699740,"splitLocations":"172.23.244.182"}}} - -{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1526555215992_0002_r_000000","taskType":"REDUCE","startTime":1526555699742,"splitLocations":""}}} - -{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1526555215992_0002_m_000000","taskType":"MAP","attemptId":"attempt_1526555215992_0002_m_000000_0","startTime":1526555701907,"trackerName":"172.23.244.182","httpPort":8042,"shufflePort":13562,"containerId":"container_1526555215992_0002_01_000002","locality":{"string":"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}} - -{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1526555215992_0002_m_000000","attemptId":"attempt_1526555215992_0002_m_000000_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1526555704106,"finishTime":1526555704164,"hostname":"172.23.244.182","port":54690,"rackname":"/default-rack","state":"map","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117829},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":267},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":149},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":38},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"clockSplits":[2210,3,2,3,2,3,2,3,2,3,2,3],"cpuUsages":[0,0,0,0,0,0,0,0,0,0,0,0],"vMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0],"physMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0]}}} - -{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1526555215992_0002_m_000000","taskType":"MAP","finishTime":1526555704164,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117829},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":267},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":149},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":38},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"successfulAttemptId":{"string":"attempt_1526555215992_0002_m_000000_0"}}}} - -{"type":"REDUCE_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1526555215992_0002_r_000000","taskType":"REDUCE","attemptId":"attempt_1526555215992_0002_r_000000_0","startTime":1526555706801,"trackerName":"172.23.244.182","httpPort":8042,"shufflePort":13562,"containerId":"container_1526555215992_0002_01_000003","locality":{"string":"OFF_SWITCH"},"avataar":{"string":"VIRGIN"}}}} - -{"type":"REDUCE_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinished":{"taskid":"task_1526555215992_0002_r_000000","attemptId":"attempt_1526555215992_0002_r_000000_0","taskType":"REDUCE","taskStatus":"SUCCEEDED","shuffleFinishTime":1526555708861,"sortFinishTime":1526555708877,"finishTime":1526555709132,"hostname":"172.23.244.182","port":54690,"rackname":"/default-rack","state":"reduce > reduce","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117774},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":45},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":146800640}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"clockSplits":[2098,20,20,20,20,20,20,20,20,20,20,20],"cpuUsages":[0,0,0,0,0,0,0,0,0,0,0,0],"vMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0],"physMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0]}}} - -{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1526555215992_0002_r_000000","taskType":"REDUCE","finishTime":1526555709132,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117774},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":45},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":146800640}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"successfulAttemptId":{"string":"attempt_1526555215992_0002_r_000000_0"}}}} - -{"type":"JOB_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobFinished":{"jobid":"job_1526555215992_0002","finishTime":1526555709175,"finishedMaps":1,"finishedReduces":1,"failedMaps":0,"failedReduces":0,"totalCounters":{"name":"TOTAL_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":235603},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":267},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":7},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.JobCounter","displayName":"Job Counters ","counts":[{"name":"TOTAL_LAUNCHED_MAPS","displayName":"Launched map tasks","value":1},{"name":"TOTAL_LAUNCHED_REDUCES","displayName":"Launched reduce tasks","value":1},{"name":"DATA_LOCAL_MAPS","displayName":"Data-local map tasks","value":1},{"name":"SLOTS_MILLIS_MAPS","displayName":"Total time spent by all maps in occupied slots (ms)","value":2257},{"name":"SLOTS_MILLIS_REDUCES","displayName":"Total time spent by all reduces in occupied slots (ms)","value":2331},{"name":"MILLIS_MAPS","displayName":"Total time spent by all map tasks (ms)","value":2257},{"name":"MILLIS_REDUCES","displayName":"Total time spent by all reduce tasks (ms)","value":2331},{"name":"VCORES_MILLIS_MAPS","displayName":"Total vcore-milliseconds taken by all map tasks","value":2257},{"name":"VCORES_MILLIS_REDUCES","displayName":"Total vcore-milliseconds taken by all reduce tasks","value":2331},{"name":"MB_MILLIS_MAPS","displayName":"Total megabyte-milliseconds taken by all map tasks","value":2311168},{"name":"MB_MILLIS_REDUCES","displayName":"Total megabyte-milliseconds taken by all reduce tasks","value":2386944}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":149},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":4},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":83},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":348127232}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"mapCounters":{"name":"MAP_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117829},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":267},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":149},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":38},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"reduceCounters":{"name":"REDUCE_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117774},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":45},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":146800640}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]}}}} diff --git a/test/resources/history/done/2018/05/17/000000/job_1526555215992_0002_conf.xml b/test/resources/history/done/2018/05/17/000000/job_1526555215992_0002_conf.xml deleted file mode 100644 index 17576bfaf..000000000 --- a/test/resources/history/done/2018/05/17/000000/job_1526555215992_0002_conf.xml +++ /dev/null @@ -1,168 +0,0 @@ - -mapreduce.job.maxtaskfailures.per.tracker3mapred-default.xmljob.xml -mapreduce.job.speculative.retry-after-speculate15000mapred-default.xmljob.xml -mapreduce.reduce.markreset.buffer.percent0.0mapred-default.xmljob.xml -mapreduce.jobhistory.max-age-ms604800000mapred-default.xmljob.xml -mapreduce.job.ubertask.enablefalsemapred-default.xmljob.xml -mapreduce.input.fileinputformat.numinputfiles1programaticallyjob.xml -mapreduce.jobhistory.datestring.cache.size200000mapred-default.xmljob.xml -mapreduce.job.inputformat.classorg.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormatprogramaticallyjob.xml -mapreduce.job.application.attempt.id1programatically -mapreduce.task.profile.reduce.params${mapreduce.task.profile.params}mapred-default.xmljob.xml -mapreduce.job.split.metainfo.maxsize10000000mapred-default.xmljob.xml -mapreduce.input.lineinputformat.linespermap1mapred-default.xmljob.xml -mapreduce.reduce.shuffle.input.buffer.percent0.70mapred-default.xmljob.xml -mapreduce.reduce.maxattempts4mapred-default.xmljob.xml -mapreduce.jobhistory.admin.acl*mapred-default.xmljob.xml -mapreduce.map.maxattempts4mapred-default.xmljob.xml -mapreduce.jobhistory.cleaner.interval-ms86400000mapred-default.xmljob.xml -mapreduce.job.map.classorg.apache.hadoop.examples.QuasiMonteCarlo$QmcMapperprogramaticallyjob.xml -mapreduce.reduce.cpu.vcores1mapred-default.xmljob.xml -mapreduce.jobtracker.heartbeats.in.second100mapred-default.xmljob.xml -mapreduce.output.fileoutputformat.compressfalsemapred-default.xmljob.xml -mapreduce.shuffle.max.threads0mapred-default.xmljob.xml -mapreduce.map.speculativefalseprogramaticallyjob.xml -mapreduce.job.speculative.slowtaskthreshold1.0mapred-default.xmljob.xml -mapreduce.tasktracker.http.threads40mapred-default.xmljob.xml -mapreduce.jobhistory.http.policyHTTP_ONLYmapred-default.xmljob.xml -mapreduce.task.timeout600000mapred-default.xmljob.xml -hadoop.security.groups.cache.secs300core-default.xmljob.xml -mapreduce.input.fileinputformat.split.minsize0mapred-default.xmljob.xml -mapreduce.shuffle.port13562mapred-default.xmljob.xml -mapreduce.jobtracker.retiredjobs.cache.size1000mapred-default.xmljob.xml -mapreduce.jobtracker.addresslocalmapred-default.xmljob.xml -hadoop.http.authentication.kerberos.principalHTTP/_HOST@LOCALHOSTcore-default.xmljob.xml -mapreduce.task.profile.reduces0-2mapred-default.xmljob.xml -mapreduce.job.end-notification.max.retry.interval5000mapred-default.xml -mapreduce.job.speculative.minimum-allowed-tasks10mapred-default.xmljob.xml -mapreduce.reduce.shuffle.fetch.retry.timeout-ms30000mapred-default.xmljob.xml -mapreduce.tasktracker.dns.nameserverdefaultmapred-default.xmljob.xml -hadoop.security.random.device.file.path/dev/urandomcore-default.xmljob.xml -mapreduce.task.merge.progress.records10000mapred-default.xmljob.xml -hadoop.registry.securefalsecore-default.xmljob.xml -mapreduce.job.counters.max120mapred-default.xmljob.xml -mapreduce.task.profilefalsemapred-default.xmljob.xml -mapreduce.task.combine.progress.records10000mapred-default.xmljob.xml -mapreduce.app-submission.cross-platformfalsemapred-default.xmljob.xml -mapreduce.job.nameQuasiMonteCarloprogramaticallyjob.xml -mapreduce.jobtracker.restart.recoverfalsemapred-default.xmljob.xml -mapreduce.map.skip.proc.count.autoincrtruemapred-default.xmljob.xml -hadoop.security.instrumentation.requires.adminfalsecore-default.xmljob.xml -io.compression.codec.bzip2.librarysystem-nativecore-default.xmljob.xml -mapreduce.task.userlog.limit.kb0mapred-default.xmljob.xml -mapreduce.reduce.speculativefalseprogramaticallyjob.xml -mapreduce.job.submithostnamevasaxena-mn1.linkedin.bizprogramaticallyjob.xml -mapreduce.jobhistory.recovery.enablefalsemapred-default.xmljob.xml -mapreduce.reduce.shuffle.memory.limit.percent0.25mapred-default.xmljob.xml -mapreduce.job.reduce.shuffle.consumer.plugin.classorg.apache.hadoop.mapreduce.task.reduce.Shufflemapred-default.xmljob.xml -mapreduce.job.jvm.numtasks1mapred-default.xmljob.xml -mapreduce.am.max-attempts2mapred-default.xmljob.xml -mapreduce.shuffle.connection-keep-alive.timeout5mapred-default.xmljob.xml -mapreduce.job.reduces1programaticallyjob.xml -mapreduce.tasktracker.reduce.tasks.maximum2mapred-default.xmljob.xml -mapreduce.cluster.local.dir${hadoop.tmp.dir}/mapred/localmapred-default.xmljob.xml -mapreduce.client.output.filterFAILEDmapred-default.xmljob.xml -mapreduce.tasktracker.healthchecker.script.timeout600000mapred-default.xmljob.xml -mapreduce.reduce.shuffle.retry-delay.max.ms60000mapred-default.xmljob.xml -mapreduce.job.committer.setup.cleanup.neededtruemapred-default.xmljob.xml -mapreduce.job.end-notification.retry.attempts0mapred-default.xmljob.xml -mapreduce.map.output.compressfalsemapred-default.xmljob.xml -mapreduce.jobhistory.cleaner.enabletruemapred-default.xmljob.xml -mapreduce.job.running.reduce.limit0mapred-default.xmljob.xml -io.seqfile.local.dir${hadoop.tmp.dir}/io/localcore-default.xmljob.xml -mapreduce.job.queuenamedefaultmapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.directory.search.timeout10000core-default.xmljob.xml -mapreduce.job.token.tracking.ids.enabledfalsemapred-default.xmljob.xml -mapreduce.map.output.compress.codecorg.apache.hadoop.io.compress.DefaultCodecmapred-default.xmljob.xml -mapreduce.job.user.namevasaxenaprogramatically -mapreduce.tasktracker.dns.interfacedefaultmapred-default.xmljob.xml -mapreduce.job.speculative.speculative-cap-total-tasks0.01mapred-default.xmljob.xml -mapreduce.job.output.value.classorg.apache.hadoop.io.LongWritableprogramaticallyjob.xml -mapreduce.job.acl-modify-job mapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.search.filter.group(objectClass=group)core-default.xmljob.xml -mapreduce.map.cpu.vcores1mapred-default.xmljob.xml -mapreduce.map.log.levelINFOmapred-default.xmljob.xml -mapreduce.job.output.key.classorg.apache.hadoop.io.BooleanWritableprogramaticallyjob.xml -mapreduce.job.running.map.limit0mapred-default.xmljob.xml -mapreduce.jobtracker.taskcache.levels2mapred-default.xmljob.xml -mapreduce.client.completion.pollinterval5000mapred-default.xmljob.xml -mapreduce.job.maps1programaticallyjob.xml -mapreduce.job.acl-view-job mapred-default.xmljob.xml -mapreduce.shuffle.max.connections0mapred-default.xmljob.xml -mapreduce.jobtracker.expire.trackers.interval600000mapred-default.xmljob.xml -mapreduce.task.io.sort.factor10mapred-default.xmljob.xml -mapreduce.input.fileinputformat.list-status.num-threads1mapred-default.xmljob.xml -mapreduce.jobhistory.move.thread-count3mapred-default.xmljob.xml -mapreduce.task.profile.params-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%smapred-default.xmljob.xml -mapreduce.jobhistory.keytab/etc/security/keytab/jhs.service.keytabmapred-default.xmljob.xml -mapreduce.reduce.shuffle.fetch.retry.interval-ms1000mapred-default.xmljob.xml -hadoop.user.group.static.mapping.overridesdr.who=;core-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.dir/jobtracker/jobsInfomapred-default.xmljob.xml -hadoop.security.group.mappingorg.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallbackcore-default.xmljob.xml -mapreduce.job.emit-timeline-datafalsemapred-default.xmljob.xml -mapreduce.map.memory.mb1024mapred-default.xmljob.xml -hadoop.http.cross-origin.max-age1800core-default.xmljob.xml -mapreduce.tasktracker.healthchecker.interval60000mapred-default.xmljob.xml -mapreduce.job.speculative.retry-after-no-speculate1000mapred-default.xmljob.xml -mapreduce.task.skip.start.attempts2mapred-default.xmljob.xml -hadoop.hdconfiguration.version1ddefault.xmljob.xml -mapreduce.map.skip.maxrecords0mapred-default.xmljob.xml -mapreduce.jobtracker.system.dir${hadoop.tmp.dir}/mapred/systemmapred-default.xmljob.xml -hadoop.registry.rm.enabledfalsecore-default.xmljob.xml -mapreduce.job.reducer.preempt.delay.sec0mapred-default.xmljob.xml -hadoop.tmp.dir/tmp/hadoop-${user.name}core-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.hours1mapred-default.xmljob.xml -mapreduce.task.io.sort.mb100mapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.search.attr.group.namecncore-default.xmljob.xml -mapreduce.output.fileoutputformat.compress.typeRECORDmapred-default.xmljob.xml -mapreduce.reduce.skip.proc.count.autoincrtruemapred-default.xmljob.xml -mapreduce.job.userlog.retain.hours24mapred-default.xmljob.xml -mapreduce.reduce.input.buffer.percent0.0mapred-default.xmljob.xml -mapreduce.tasktracker.local.dir.minspacestart0mapred-default.xmljob.xml -mapreduce.task.files.preserve.failedtasksfalsemapred-default.xmljob.xml -mapreduce.output.fileoutputformat.compress.codecorg.apache.hadoop.io.compress.DefaultCodecmapred-default.xmljob.xml -map.sort.classorg.apache.hadoop.util.QuickSortmapred-default.xmljob.xml -mapreduce.jobtracker.tasktracker.maxblacklists4mapred-default.xmljob.xml -mapreduce.task.profile.maps0-2mapred-default.xmljob.xml -mapreduce.jobtracker.staging.root.dir${hadoop.tmp.dir}/mapred/stagingmapred-default.xmljob.xml -mapreduce.job.reduce.classorg.apache.hadoop.examples.QuasiMonteCarlo$QmcReducerprogramaticallyjob.xml -hadoop.http.cross-origin.allowed-origins*core-default.xmljob.xml -mapreduce.tasktracker.local.dir.minspacekill0mapred-default.xmljob.xml -mapreduce.tasktracker.taskmemorymanager.monitoringinterval5000mapred-default.xmljob.xml -mapreduce.job.end-notification.retry.interval1000mapred-default.xmljob.xml -mapreduce.jobhistory.loadedjobs.cache.size5mapred-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.activetruemapred-default.xmljob.xml -mapreduce.reduce.shuffle.parallelcopies5mapred-default.xmljob.xml -mapreduce.jobhistory.principaljhs/_HOST@REALM.TLDmapred-default.xmljob.xml -hadoop.http.authentication.simple.anonymous.allowedtruecore-default.xmljob.xml -mapreduce.client.genericoptionsparser.usedtrueprogramaticallyjob.xml -mapreduce.job.ubertask.maxreduces1mapred-default.xmljob.xml -mapreduce.shuffle.transfer.buffer.size131072mapred-default.xmljob.xml -mapreduce.jobtracker.jobhistory.task.numberprogresssplits12mapred-default.xmljob.xml -mapreduce.jobhistory.client.thread-count10mapred-default.xmljob.xml -mapreduce.fileoutputcommitter.algorithm.version1mapred-default.xmljob.xml -mapreduce.reduce.skip.maxgroups0mapred-default.xmljob.xml -mapreduce.reduce.memory.mb1024mapred-default.xmljob.xml -mapreduce.client.progressmonitor.pollinterval1000mapred-default.xmljob.xml -mapreduce.job.outputformat.classorg.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormatprogramaticallyjob.xml -hadoop.http.authentication.signature.secret.file${user.home}/hadoop-http-auth-signature-secretcore-default.xmljob.xml -mapreduce.jobhistory.move.interval-ms180000mapred-default.xmljob.xml -hadoop.security.authorizationfalsecore-default.xmljob.xml -dstorage.policy.enabledtrueddefault.xmljob.xml -mapreduce.shuffle.connection-keep-alive.enablefalsemapred-default.xmljob.xml -hadoop.common.configuration.version0.23.0core-default.xmljob.xml -hadoop.security.groups.negative-cache.secs30core-default.xmljob.xml -mapreduce.ifile.readaheadtruemapred-default.xmljob.xml -mapreduce.job.max.split.locations10mapred-default.xmljob.xml -mapreduce.jobhistory.joblist.cache.size2mapred-site.xmljob.xml -mapreduce.job.end-notification.max.attempts5mapred-default.xml -dimage.transfer.timeout60000ddefault.xmljob.xml -mapreduce.reduce.shuffle.connect.timeout180000mapred-default.xmljob.xml -mapreduce.jobhistory.webapp.address0.0.0.0:19888mapred-default.xmljob.xml -hadoop.http.authentication.token.validity36000core-default.xmljob.xml -mapreduce.tasktracker.map.tasks.maximum2mapred-default.xmljob.xml -mapreduce.job.speculative.speculative-cap-running-tasks0.1mapred-default.xmljob.xml -mapreduce.map.sort.spill.percent0.80mapred-default.xmljob.xml -mapreduce.job.reduce.slowstart.completedmaps0.05mapred-default.xmljob.xml -mapreduce.cluster.temp.dir${hadoop.tmp.dir}/mapred/tempmapred-default.xmljob.xml - diff --git a/test/resources/history/done_intermediate/user/job_1526555215992_0004-1526567229183-user-QuasiMonteCarlo-1526567243482-1-1-SUCCEEDED-default-1526567234171.jhist b/test/resources/history/done_intermediate/user/job_1526555215992_0004-1526567229183-user-QuasiMonteCarlo-1526567243482-1-1-SUCCEEDED-default-1526567234171.jhist deleted file mode 100644 index a7e9f41bc..000000000 --- a/test/resources/history/done_intermediate/user/job_1526555215992_0004-1526567229183-user-QuasiMonteCarlo-1526567243482-1-1-SUCCEEDED-default-1526567234171.jhist +++ /dev/null @@ -1,27 +0,0 @@ -Avro-Json -{"type":"record","name":"Event","namespace":"org.apache.hadoop.mapreduce.jobhistory","fields":[{"name":"type","type":{"type":"enum","name":"EventType","symbols":["JOB_SUBMITTED","JOB_INITED","JOB_FINISHED","JOB_PRIORITY_CHANGED","JOB_STATUS_CHANGED","JOB_QUEUE_CHANGED","JOB_FAILED","JOB_KILLED","JOB_ERROR","JOB_INFO_CHANGED","TASK_STARTED","TASK_FINISHED","TASK_FAILED","TASK_UPDATED","NORMALIZED_RESOURCE","MAP_ATTEMPT_STARTED","MAP_ATTEMPT_FINISHED","MAP_ATTEMPT_FAILED","MAP_ATTEMPT_KILLED","REDUCE_ATTEMPT_STARTED","REDUCE_ATTEMPT_FINISHED","REDUCE_ATTEMPT_FAILED","REDUCE_ATTEMPT_KILLED","SETUP_ATTEMPT_STARTED","SETUP_ATTEMPT_FINISHED","SETUP_ATTEMPT_FAILED","SETUP_ATTEMPT_KILLED","CLEANUP_ATTEMPT_STARTED","CLEANUP_ATTEMPT_FINISHED","CLEANUP_ATTEMPT_FAILED","CLEANUP_ATTEMPT_KILLED","AM_STARTED"]}},{"name":"event","type":[{"type":"record","name":"JobFinished","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"failedMaps","type":"int"},{"name":"failedReduces","type":"int"},{"name":"totalCounters","type":{"type":"record","name":"JhCounters","fields":[{"name":"name","type":"string"},{"name":"groups","type":{"type":"array","items":{"type":"record","name":"JhCounterGroup","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"counts","type":{"type":"array","items":{"type":"record","name":"JhCounter","fields":[{"name":"name","type":"string"},{"name":"displayName","type":"string"},{"name":"value","type":"long"}]}}}]}}}]}},{"name":"mapCounters","type":"JhCounters"},{"name":"reduceCounters","type":"JhCounters"}]},{"type":"record","name":"JobInfoChange","fields":[{"name":"jobid","type":"string"},{"name":"submitTime","type":"long"},{"name":"launchTime","type":"long"}]},{"type":"record","name":"JobInited","fields":[{"name":"jobid","type":"string"},{"name":"launchTime","type":"long"},{"name":"totalMaps","type":"int"},{"name":"totalReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"uberized","type":"boolean"}]},{"type":"record","name":"AMStarted","fields":[{"name":"applicationAttemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"containerId","type":"string"},{"name":"nodeManagerHost","type":"string"},{"name":"nodeManagerPort","type":"int"},{"name":"nodeManagerHttpPort","type":"int"}]},{"type":"record","name":"JobPriorityChange","fields":[{"name":"jobid","type":"string"},{"name":"priority","type":"string"}]},{"type":"record","name":"JobQueueChange","fields":[{"name":"jobid","type":"string"},{"name":"jobQueueName","type":"string"}]},{"type":"record","name":"JobStatusChanged","fields":[{"name":"jobid","type":"string"},{"name":"jobStatus","type":"string"}]},{"type":"record","name":"JobSubmitted","fields":[{"name":"jobid","type":"string"},{"name":"jobName","type":"string"},{"name":"userName","type":"string"},{"name":"submitTime","type":"long"},{"name":"jobConfPath","type":"string"},{"name":"acls","type":{"type":"map","values":"string"}},{"name":"jobQueueName","type":"string"},{"name":"workflowId","type":["null","string"],"default":null},{"name":"workflowName","type":["null","string"],"default":null},{"name":"workflowNodeName","type":["null","string"],"default":null},{"name":"workflowAdjacencies","type":["null","string"],"default":null},{"name":"workflowTags","type":["null","string"],"default":null}]},{"type":"record","name":"JobUnsuccessfulCompletion","fields":[{"name":"jobid","type":"string"},{"name":"finishTime","type":"long"},{"name":"finishedMaps","type":"int"},{"name":"finishedReduces","type":"int"},{"name":"jobStatus","type":"string"},{"name":"diagnostics","type":["null","string"],"default":null}]},{"type":"record","name":"MapAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"mapFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"ReduceAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"shuffleFinishTime","type":"long"},{"name":"sortFinishTime","type":"long"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskAttemptFinished","fields":[{"name":"taskid","type":"string"},{"name":"attemptId","type":"string"},{"name":"taskType","type":"string"},{"name":"taskStatus","type":"string"},{"name":"finishTime","type":"long"},{"name":"rackname","type":"string"},{"name":"hostname","type":"string"},{"name":"state","type":"string"},{"name":"counters","type":"JhCounters"}]},{"type":"record","name":"TaskAttemptStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"startTime","type":"long"},{"name":"trackerName","type":"string"},{"name":"httpPort","type":"int"},{"name":"shufflePort","type":"int"},{"name":"containerId","type":"string"},{"name":"locality","type":["null","string"],"default":null},{"name":"avataar","type":["null","string"],"default":null}]},{"type":"record","name":"TaskAttemptUnsuccessfulCompletion","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"attemptId","type":"string"},{"name":"finishTime","type":"long"},{"name":"hostname","type":"string"},{"name":"port","type":"int"},{"name":"rackname","type":"string"},{"name":"status","type":"string"},{"name":"error","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null},{"name":"clockSplits","type":{"type":"array","items":"int"}},{"name":"cpuUsages","type":{"type":"array","items":"int"}},{"name":"vMemKbytes","type":{"type":"array","items":"int"}},{"name":"physMemKbytes","type":{"type":"array","items":"int"}}]},{"type":"record","name":"TaskFailed","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"error","type":"string"},{"name":"failedDueToAttempt","type":["null","string"]},{"name":"status","type":"string"},{"name":"counters","type":["null","JhCounters"],"default":null}]},{"type":"record","name":"TaskFinished","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"finishTime","type":"long"},{"name":"status","type":"string"},{"name":"counters","type":"JhCounters"},{"name":"successfulAttemptId","type":["null","string"],"default":null}]},{"type":"record","name":"TaskStarted","fields":[{"name":"taskid","type":"string"},{"name":"taskType","type":"string"},{"name":"startTime","type":"long"},{"name":"splitLocations","type":"string"}]},{"type":"record","name":"TaskUpdated","fields":[{"name":"taskid","type":"string"},{"name":"finishTime","type":"long"}]}]}]} -{"type":"AM_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.AMStarted":{"applicationAttemptId":"appattempt_1526555215992_0004_000001","startTime":1526567231815,"containerId":"container_1526555215992_0004_01_000001","nodeManagerHost":"172.23.244.182","nodeManagerPort":54690,"nodeManagerHttpPort":8042}}} - -{"type":"JOB_SUBMITTED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobSubmitted":{"jobid":"job_1526555215992_0004","jobName":"QuasiMonteCarlo","userName":"user","submitTime":1526567229183,"jobConfPath":"hdfs://127.0.0.1:9000/tmp/hadoop-yarn/staging/user/.staging/job_1526555215992_0004/job.xml","acls":{},"jobQueueName":"default","workflowId":{"string":""},"workflowName":{"string":""},"workflowNodeName":{"string":""},"workflowAdjacencies":{"string":""},"workflowTags":{"string":""}}}} - -{"type":"JOB_INITED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInited":{"jobid":"job_1526555215992_0004","launchTime":1526567234171,"totalMaps":1,"totalReduces":1,"jobStatus":"INITED","uberized":false}}} - -{"type":"JOB_INFO_CHANGED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobInfoChange":{"jobid":"job_1526555215992_0004","submitTime":1526567229183,"launchTime":1526567234171}}} - -{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1526555215992_0004_m_000000","taskType":"MAP","startTime":1526567234196,"splitLocations":"172.23.244.182"}}} - -{"type":"TASK_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskStarted":{"taskid":"task_1526555215992_0004_r_000000","taskType":"REDUCE","startTime":1526567234199,"splitLocations":""}}} - -{"type":"MAP_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1526555215992_0004_m_000000","taskType":"MAP","attemptId":"attempt_1526555215992_0004_m_000000_0","startTime":1526567236332,"trackerName":"172.23.244.182","httpPort":8042,"shufflePort":13562,"containerId":"container_1526555215992_0004_01_000002","locality":{"string":"NODE_LOCAL"},"avataar":{"string":"VIRGIN"}}}} - -{"type":"MAP_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.MapAttemptFinished":{"taskid":"task_1526555215992_0004_m_000000","attemptId":"attempt_1526555215992_0004_m_000000_0","taskType":"MAP","taskStatus":"SUCCEEDED","mapFinishTime":1526567238402,"finishTime":1526567238471,"hostname":"172.23.244.182","port":54690,"rackname":"/default-rack","state":"map","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117831},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":268},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":150},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":35},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"clockSplits":[2081,2,3,2,3,2,2,3,2,3,2,3],"cpuUsages":[0,0,0,0,0,0,0,0,0,0,0,0],"vMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0],"physMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0]}}} - -{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1526555215992_0004_m_000000","taskType":"MAP","finishTime":1526567238471,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117831},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":268},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":150},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":35},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"successfulAttemptId":{"string":"attempt_1526555215992_0004_m_000000_0"}}}} - -{"type":"REDUCE_ATTEMPT_STARTED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskAttemptStarted":{"taskid":"task_1526555215992_0004_r_000000","taskType":"REDUCE","attemptId":"attempt_1526555215992_0004_r_000000_0","startTime":1526567241267,"trackerName":"172.23.244.182","httpPort":8042,"shufflePort":13562,"containerId":"container_1526555215992_0004_01_000003","locality":{"string":"OFF_SWITCH"},"avataar":{"string":"VIRGIN"}}}} - -{"type":"REDUCE_ATTEMPT_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.ReduceAttemptFinished":{"taskid":"task_1526555215992_0004_r_000000","attemptId":"attempt_1526555215992_0004_r_000000_0","taskType":"REDUCE","taskStatus":"SUCCEEDED","shuffleFinishTime":1526567243232,"sortFinishTime":1526567243247,"finishTime":1526567243448,"hostname":"172.23.244.182","port":54690,"rackname":"/default-rack","state":"reduce > reduce","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117776},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":38},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":144703488}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"clockSplits":[1997,16,16,16,16,16,15,16,16,16,16,16],"cpuUsages":[0,0,0,0,0,0,0,0,0,0,0,0],"vMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0],"physMemKbytes":[0,0,0,0,0,0,0,0,0,0,0,0]}}} - -{"type":"TASK_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.TaskFinished":{"taskid":"task_1526555215992_0004_r_000000","taskType":"REDUCE","finishTime":1526567243448,"status":"SUCCEEDED","counters":{"name":"COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117776},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":38},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":144703488}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"successfulAttemptId":{"string":"attempt_1526555215992_0004_r_000000_0"}}}} - -{"type":"JOB_FINISHED","event":{"org.apache.hadoop.mapreduce.jobhistory.JobFinished":{"jobid":"job_1526555215992_0004","finishTime":1526567243482,"finishedMaps":1,"finishedReduces":1,"failedMaps":0,"failedReduces":0,"totalCounters":{"name":"TOTAL_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":235607},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":268},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":7},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.JobCounter","displayName":"Job Counters ","counts":[{"name":"TOTAL_LAUNCHED_MAPS","displayName":"Launched map tasks","value":1},{"name":"TOTAL_LAUNCHED_REDUCES","displayName":"Launched reduce tasks","value":1},{"name":"DATA_LOCAL_MAPS","displayName":"Data-local map tasks","value":1},{"name":"SLOTS_MILLIS_MAPS","displayName":"Total time spent by all maps in occupied slots (ms)","value":2139},{"name":"SLOTS_MILLIS_REDUCES","displayName":"Total time spent by all reduces in occupied slots (ms)","value":2181},{"name":"MILLIS_MAPS","displayName":"Total time spent by all map tasks (ms)","value":2139},{"name":"MILLIS_REDUCES","displayName":"Total time spent by all reduce tasks (ms)","value":2181},{"name":"VCORES_MILLIS_MAPS","displayName":"Total vcore-milliseconds taken by all map tasks","value":2139},{"name":"VCORES_MILLIS_REDUCES","displayName":"Total vcore-milliseconds taken by all reduce tasks","value":2181},{"name":"MB_MILLIS_MAPS","displayName":"Total megabyte-milliseconds taken by all map tasks","value":2190336},{"name":"MB_MILLIS_REDUCES","displayName":"Total megabyte-milliseconds taken by all reduce tasks","value":2233344}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":150},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":4},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":73},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":346030080}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]},"mapCounters":{"name":"MAP_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":0},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117831},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":268},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":0},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":4},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":0}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"MAP_INPUT_RECORDS","displayName":"Map input records","value":1},{"name":"MAP_OUTPUT_RECORDS","displayName":"Map output records","value":2},{"name":"MAP_OUTPUT_BYTES","displayName":"Map output bytes","value":18},{"name":"MAP_OUTPUT_MATERIALIZED_BYTES","displayName":"Map output materialized bytes","value":28},{"name":"SPLIT_RAW_BYTES","displayName":"Input split bytes","value":150},{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":0},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":35},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":201326592}]},{"name":"org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter","displayName":"File Input Format Counters ","counts":[{"name":"BYTES_READ","displayName":"Bytes Read","value":118}]}]},"reduceCounters":{"name":"REDUCE_COUNTERS","groups":[{"name":"org.apache.hadoop.mapreduce.FileSystemCounter","displayName":"File System Counters","counts":[{"name":"FILE_BYTES_READ","displayName":"FILE: Number of bytes read","value":28},{"name":"FILE_BYTES_WRITTEN","displayName":"FILE: Number of bytes written","value":117776},{"name":"FILE_READ_OPS","displayName":"FILE: Number of read operations","value":0},{"name":"FILE_LARGE_READ_OPS","displayName":"FILE: Number of large read operations","value":0},{"name":"FILE_WRITE_OPS","displayName":"FILE: Number of write operations","value":0},{"name":"HDFS_BYTES_READ","displayName":"HDFS: Number of bytes read","value":0},{"name":"HDFS_BYTES_WRITTEN","displayName":"HDFS: Number of bytes written","value":215},{"name":"HDFS_READ_OPS","displayName":"HDFS: Number of read operations","value":3},{"name":"HDFS_LARGE_READ_OPS","displayName":"HDFS: Number of large read operations","value":0},{"name":"HDFS_WRITE_OPS","displayName":"HDFS: Number of write operations","value":3}]},{"name":"org.apache.hadoop.mapreduce.TaskCounter","displayName":"Map-Reduce Framework","counts":[{"name":"COMBINE_INPUT_RECORDS","displayName":"Combine input records","value":0},{"name":"COMBINE_OUTPUT_RECORDS","displayName":"Combine output records","value":0},{"name":"REDUCE_INPUT_GROUPS","displayName":"Reduce input groups","value":2},{"name":"REDUCE_SHUFFLE_BYTES","displayName":"Reduce shuffle bytes","value":28},{"name":"REDUCE_INPUT_RECORDS","displayName":"Reduce input records","value":2},{"name":"REDUCE_OUTPUT_RECORDS","displayName":"Reduce output records","value":0},{"name":"SPILLED_RECORDS","displayName":"Spilled Records","value":2},{"name":"SHUFFLED_MAPS","displayName":"Shuffled Maps ","value":1},{"name":"FAILED_SHUFFLE","displayName":"Failed Shuffles","value":0},{"name":"MERGED_MAP_OUTPUTS","displayName":"Merged Map outputs","value":1},{"name":"GC_TIME_MILLIS","displayName":"GC time elapsed (ms)","value":38},{"name":"CPU_MILLISECONDS","displayName":"CPU time spent (ms)","value":0},{"name":"PHYSICAL_MEMORY_BYTES","displayName":"Physical memory (bytes) snapshot","value":0},{"name":"VIRTUAL_MEMORY_BYTES","displayName":"Virtual memory (bytes) snapshot","value":0},{"name":"COMMITTED_HEAP_BYTES","displayName":"Total committed heap usage (bytes)","value":144703488}]},{"name":"Shuffle Errors","displayName":"Shuffle Errors","counts":[{"name":"BAD_ID","displayName":"BAD_ID","value":0},{"name":"CONNECTION","displayName":"CONNECTION","value":0},{"name":"IO_ERROR","displayName":"IO_ERROR","value":0},{"name":"WRONG_LENGTH","displayName":"WRONG_LENGTH","value":0},{"name":"WRONG_MAP","displayName":"WRONG_MAP","value":0},{"name":"WRONG_REDUCE","displayName":"WRONG_REDUCE","value":0}]},{"name":"org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter","displayName":"File Output Format Counters ","counts":[{"name":"BYTES_WRITTEN","displayName":"Bytes Written","value":97}]}]}}}} diff --git a/test/resources/history/done_intermediate/user/job_1526555215992_0004_conf.xml b/test/resources/history/done_intermediate/user/job_1526555215992_0004_conf.xml deleted file mode 100644 index dfe80de78..000000000 --- a/test/resources/history/done_intermediate/user/job_1526555215992_0004_conf.xml +++ /dev/null @@ -1,167 +0,0 @@ - -mapreduce.job.maxtaskfailures.per.tracker3mapred-default.xmljob.xml -mapreduce.job.speculative.retry-after-speculate15000mapred-default.xmljob.xml -mapreduce.reduce.markreset.buffer.percent0.0mapred-default.xmljob.xml -mapreduce.jobhistory.max-age-ms604800000mapred-default.xmljob.xml -mapreduce.job.ubertask.enablefalsemapred-default.xmljob.xml -mapreduce.input.fileinputformat.numinputfiles1programaticallyjob.xml -mapreduce.jobhistory.datestring.cache.size200000mapred-default.xmljob.xml -mapreduce.job.inputformat.classorg.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormatprogramaticallyjob.xml -mapreduce.job.application.attempt.id1programatically -mapreduce.task.profile.reduce.params${mapreduce.task.profile.params}mapred-default.xmljob.xml -mapreduce.job.split.metainfo.maxsize10000000mapred-default.xmljob.xml -mapreduce.input.lineinputformat.linespermap1mapred-default.xmljob.xml -mapreduce.reduce.shuffle.input.buffer.percent0.70mapred-default.xmljob.xml -mapreduce.reduce.maxattempts4mapred-default.xmljob.xml -mapreduce.jobhistory.admin.acl*mapred-default.xmljob.xml -mapreduce.map.maxattempts4mapred-default.xmljob.xml -mapreduce.jobhistory.cleaner.interval-ms86400000mapred-default.xmljob.xml -mapreduce.job.map.classorg.apache.hadoop.examples.QuasiMonteCarlo$QmcMapperprogramaticallyjob.xml -mapreduce.reduce.cpu.vcores1mapred-default.xmljob.xml -mapreduce.jobtracker.heartbeats.in.second100mapred-default.xmljob.xml -mapreduce.output.fileoutputformat.compressfalsemapred-default.xmljob.xml -mapreduce.shuffle.max.threads0mapred-default.xmljob.xml -mapreduce.map.speculativefalseprogramaticallyjob.xml -mapreduce.job.speculative.slowtaskthreshold1.0mapred-default.xmljob.xml -mapreduce.tasktracker.http.threads40mapred-default.xmljob.xml -mapreduce.jobhistory.http.policyHTTP_ONLYmapred-default.xmljob.xml -mapreduce.task.timeout600000mapred-default.xmljob.xml -hadoop.security.groups.cache.secs300core-default.xmljob.xml -mapreduce.input.fileinputformat.split.minsize0mapred-default.xmljob.xml -mapreduce.shuffle.port13562mapred-default.xmljob.xml -mapreduce.jobtracker.retiredjobs.cache.size1000mapred-default.xmljob.xml -mapreduce.jobtracker.addresslocalmapred-default.xmljob.xml -mapreduce.task.profile.reduces0-2mapred-default.xmljob.xml -mapreduce.job.end-notification.max.retry.interval5000mapred-default.xml -mapreduce.job.speculative.minimum-allowed-tasks10mapred-default.xmljob.xml -mapreduce.reduce.shuffle.fetch.retry.timeout-ms30000mapred-default.xmljob.xml -mapreduce.tasktracker.dns.nameserverdefaultmapred-default.xmljob.xml -hadoop.security.random.device.file.path/dev/urandomcore-default.xmljob.xml -mapreduce.task.merge.progress.records10000mapred-default.xmljob.xml -hadoop.registry.securefalsecore-default.xmljob.xml -mapreduce.job.counters.max120mapred-default.xmljob.xml -mapreduce.task.profilefalsemapred-default.xmljob.xml -mapreduce.task.combine.progress.records10000mapred-default.xmljob.xml -mapreduce.app-submission.cross-platformfalsemapred-default.xmljob.xml -mapreduce.job.nameQuasiMonteCarloprogramaticallyjob.xml -mapreduce.jobtracker.restart.recoverfalsemapred-default.xmljob.xml -mapreduce.map.skip.proc.count.autoincrtruemapred-default.xmljob.xml -hadoop.security.instrumentation.requires.adminfalsecore-default.xmljob.xml -io.compression.codec.bzip2.librarysystem-nativecore-default.xmljob.xml -mapreduce.task.userlog.limit.kb0mapred-default.xmljob.xml -mapreduce.reduce.speculativefalseprogramaticallyjob.xml -mapreduce.job.submithostnamevasaxena-mn1.linkedin.bizprogramaticallyjob.xml -mapreduce.jobhistory.recovery.enablefalsemapred-default.xmljob.xml -mapreduce.reduce.shuffle.memory.limit.percent0.25mapred-default.xmljob.xml -mapreduce.job.reduce.shuffle.consumer.plugin.classorg.apache.hadoop.mapreduce.task.reduce.Shufflemapred-default.xmljob.xml -mapreduce.job.jvm.numtasks1mapred-default.xmljob.xml -mapreduce.am.max-attempts2mapred-default.xmljob.xml -mapreduce.shuffle.connection-keep-alive.timeout5mapred-default.xmljob.xml -mapreduce.job.reduces1programaticallyjob.xml -mapreduce.tasktracker.reduce.tasks.maximum2mapred-default.xmljob.xml -mapreduce.cluster.local.dir${hadoop.tmp.dir}/mapred/localmapred-default.xmljob.xml -mapreduce.client.output.filterFAILEDmapred-default.xmljob.xml -mapreduce.tasktracker.healthchecker.script.timeout600000mapred-default.xmljob.xml -mapreduce.reduce.shuffle.retry-delay.max.ms60000mapred-default.xmljob.xml -mapreduce.job.committer.setup.cleanup.neededtruemapred-default.xmljob.xml -mapreduce.job.end-notification.retry.attempts0mapred-default.xmljob.xml -mapreduce.map.output.compressfalsemapred-default.xmljob.xml -mapreduce.jobhistory.cleaner.enabletruemapred-default.xmljob.xml -mapreduce.job.running.reduce.limit0mapred-default.xmljob.xml -io.seqfile.local.dir${hadoop.tmp.dir}/io/localcore-default.xmljob.xml -mapreduce.job.queuenamedefaultmapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.directory.search.timeout10000core-default.xmljob.xml -mapreduce.job.token.tracking.ids.enabledfalsemapred-default.xmljob.xml -mapreduce.map.output.compress.codecorg.apache.hadoop.io.compress.DefaultCodecmapred-default.xmljob.xml -mapreduce.job.user.namevasaxenaprogramatically -mapreduce.tasktracker.dns.interfacedefaultmapred-default.xmljob.xml -mapreduce.job.speculative.speculative-cap-total-tasks0.01mapred-default.xmljob.xml -mapreduce.job.output.value.classorg.apache.hadoop.io.LongWritableprogramaticallyjob.xml -mapreduce.job.acl-modify-job mapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.search.filter.group(objectClass=group)core-default.xmljob.xml -mapreduce.map.cpu.vcores1mapred-default.xmljob.xml -mapreduce.map.log.levelINFOmapred-default.xmljob.xml -mapreduce.job.output.key.classorg.apache.hadoop.io.BooleanWritableprogramaticallyjob.xml -mapreduce.job.running.map.limit0mapred-default.xmljob.xml -mapreduce.jobtracker.taskcache.levels2mapred-default.xmljob.xml -mapreduce.client.completion.pollinterval5000mapred-default.xmljob.xml -mapreduce.job.maps1programaticallyjob.xml -mapreduce.job.acl-view-job mapred-default.xmljob.xml -mapreduce.shuffle.max.connections0mapred-default.xmljob.xml -mapreduce.jobtracker.expire.trackers.interval600000mapred-default.xmljob.xml -mapreduce.task.io.sort.factor10mapred-default.xmljob.xml -mapreduce.input.fileinputformat.list-status.num-threads1mapred-default.xmljob.xml -mapreduce.jobhistory.move.thread-count3mapred-default.xmljob.xml -mapreduce.task.profile.params-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%smapred-default.xmljob.xml -mapreduce.jobhistory.keytab/etc/security/keytab/jhs.service.keytabmapred-default.xmljob.xml -mapreduce.reduce.shuffle.fetch.retry.interval-ms1000mapred-default.xmljob.xml -hadoop.user.group.static.mapping.overridesdr.who=;core-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.dir/jobtracker/jobsInfomapred-default.xmljob.xml -hadoop.security.group.mappingorg.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallbackcore-default.xmljob.xml -mapreduce.job.emit-timeline-datafalsemapred-default.xmljob.xml -mapreduce.map.memory.mb1024mapred-default.xmljob.xml -hadoop.http.cross-origin.max-age1800core-default.xmljob.xml -mapreduce.tasktracker.healthchecker.interval60000mapred-default.xmljob.xml -mapreduce.job.speculative.retry-after-no-speculate1000mapred-default.xmljob.xml -mapreduce.task.skip.start.attempts2mapred-default.xmljob.xml -hadoop.hdconfiguration.version1ddefault.xmljob.xml -mapreduce.map.skip.maxrecords0mapred-default.xmljob.xml -mapreduce.jobtracker.system.dir${hadoop.tmp.dir}/mapred/systemmapred-default.xmljob.xml -hadoop.registry.rm.enabledfalsecore-default.xmljob.xml -mapreduce.job.reducer.preempt.delay.sec0mapred-default.xmljob.xml -hadoop.tmp.dir/tmp/hadoop-${user.name}core-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.hours1mapred-default.xmljob.xml -mapreduce.task.io.sort.mb100mapred-default.xmljob.xml -hadoop.security.group.mapping.ldap.search.attr.group.namecncore-default.xmljob.xml -mapreduce.output.fileoutputformat.compress.typeRECORDmapred-default.xmljob.xml -mapreduce.reduce.skip.proc.count.autoincrtruemapred-default.xmljob.xml -mapreduce.job.userlog.retain.hours24mapred-default.xmljob.xml -mapreduce.reduce.input.buffer.percent0.0mapred-default.xmljob.xml -mapreduce.tasktracker.local.dir.minspacestart0mapred-default.xmljob.xml -mapreduce.task.files.preserve.failedtasksfalsemapred-default.xmljob.xml -mapreduce.output.fileoutputformat.compress.codecorg.apache.hadoop.io.compress.DefaultCodecmapred-default.xmljob.xml -map.sort.classorg.apache.hadoop.util.QuickSortmapred-default.xmljob.xml -mapreduce.jobtracker.tasktracker.maxblacklists4mapred-default.xmljob.xml -mapreduce.task.profile.maps0-2mapred-default.xmljob.xml -mapreduce.jobtracker.staging.root.dir${hadoop.tmp.dir}/mapred/stagingmapred-default.xmljob.xml -mapreduce.job.reduce.classorg.apache.hadoop.examples.QuasiMonteCarlo$QmcReducerprogramaticallyjob.xml -hadoop.http.cross-origin.allowed-origins*core-default.xmljob.xml -mapreduce.tasktracker.local.dir.minspacekill0mapred-default.xmljob.xml -mapreduce.tasktracker.taskmemorymanager.monitoringinterval5000mapred-default.xmljob.xml -mapreduce.job.end-notification.retry.interval1000mapred-default.xmljob.xml -mapreduce.jobhistory.loadedjobs.cache.size5mapred-default.xmljob.xml -mapreduce.jobtracker.persist.jobstatus.activetruemapred-default.xmljob.xml -mapreduce.reduce.shuffle.parallelcopies5mapred-default.xmljob.xml -mapreduce.jobhistory.principaljhs/_HOST@REALM.TLDmapred-default.xmljob.xml -hadoop.http.authentication.simple.anonymous.allowedtruecore-default.xmljob.xml -mapreduce.client.genericoptionsparser.usedtrueprogramaticallyjob.xml -mapreduce.job.ubertask.maxreduces1mapred-default.xmljob.xml -mapreduce.shuffle.transfer.buffer.size131072mapred-default.xmljob.xml -mapreduce.jobtracker.jobhistory.task.numberprogresssplits12mapred-default.xmljob.xml -mapreduce.jobhistory.client.thread-count10mapred-default.xmljob.xml -mapreduce.fileoutputcommitter.algorithm.version1mapred-default.xmljob.xml -mapreduce.reduce.skip.maxgroups0mapred-default.xmljob.xml -mapreduce.reduce.memory.mb1024mapred-default.xmljob.xml -mapreduce.client.progressmonitor.pollinterval1000mapred-default.xmljob.xml -mapreduce.job.outputformat.classorg.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormatprogramaticallyjob.xml -hadoop.http.authentication.signature.secret.file${user.home}/hadoop-http-auth-signature-secretcore-default.xmljob.xml -mapreduce.jobhistory.move.interval-ms180000mapred-default.xmljob.xml -hadoop.security.authorizationfalsecore-default.xmljob.xml -dstorage.policy.enabledtrueddefault.xmljob.xml -mapreduce.shuffle.connection-keep-alive.enablefalsemapred-default.xmljob.xml -hadoop.common.configuration.version0.23.0core-default.xmljob.xml -hadoop.security.groups.negative-cache.secs30core-default.xmljob.xml -mapreduce.ifile.readaheadtruemapred-default.xmljob.xml -mapreduce.job.max.split.locations10mapred-default.xmljob.xml -mapreduce.jobhistory.joblist.cache.size2mapred-site.xmljob.xml -mapreduce.job.end-notification.max.attempts5mapred-default.xml -dimage.transfer.timeout60000ddefault.xmljob.xml -mapreduce.reduce.shuffle.connect.timeout180000mapred-default.xmljob.xml -mapreduce.jobhistory.webapp.address0.0.0.0:19888mapred-default.xmljob.xml -hadoop.http.authentication.token.validity36000core-default.xmljob.xml -mapreduce.tasktracker.map.tasks.maximum2mapred-default.xmljob.xml -mapreduce.job.speculative.speculative-cap-running-tasks0.1mapred-default.xmljob.xml -mapreduce.map.sort.spill.percent0.80mapred-default.xmljob.xml -mapreduce.job.reduce.slowstart.completedmaps0.05mapred-default.xmljob.xml -mapreduce.cluster.temp.dir${hadoop.tmp.dir}/mapred/tempmapred-default.xmljob.xml - diff --git a/test/resources/log4j.properties b/test/resources/log4j.properties deleted file mode 100644 index f3b3d59e9..000000000 --- a/test/resources/log4j.properties +++ /dev/null @@ -1,27 +0,0 @@ -# -# Copyright 2016 LinkedIn Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# - -#Define the root logger with appender file -log4j.rootLogger = INFO, FA - -#File Appender -log4j.appender.FA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.FA.File=dr_elephant.log -log4j.appender.FA.layout=org.apache.log4j.PatternLayout -log4j.appender.FA.layout.ConversionPattern=%d{MM-dd-yyyy HH:mm:ss} %-5p [%t] %c %x: %m%n -log4j.appender.FA.DatePattern='.'yyyy-MM-dd -log4j.appender.FA.ImmediateFlush=true -log4j.appender.FA.Append=true diff --git a/test/resources/mapred-site.xml b/test/resources/mapred-site.xml deleted file mode 100644 index 4c7d910cb..000000000 --- a/test/resources/mapred-site.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - mapreduce.jobhistory.done-dir - test/resources/history/done - - - - mapreduce.jobhistory.intermediate-done-dir - test/resources/history/done_intermediate - - - diff --git a/test/resources/mrdata/mapperTaskCounter1.properties b/test/resources/mrdata/mapperTaskCounter1.properties deleted file mode 100644 index 424b12fb4..000000000 --- a/test/resources/mrdata/mapperTaskCounter1.properties +++ /dev/null @@ -1,3 +0,0 @@ -org.apache.hadoop.mapreduce.FileSystemCounter={FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, HDFS_READ_OPS=4, HDFS_BYTES_READ=268, HDFS_LARGE_READ_OPS=0, FILE_READ_OPS=0, FILE_BYTES_WRITTEN=117858, FILE_BYTES_READ=0, HDFS_WRITE_OPS=0, HDFS_BYTES_WRITTEN=0} -org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter={BYTES_READ=118} -org.apache.hadoop.mapreduce.TaskCounter={MAP_OUTPUT_MATERIALIZED_BYTES=28, SPILLED_RECORDS=2, MERGED_MAP_OUTPUTS=0, VIRTUAL_MEMORY_BYTES=0, MAP_INPUT_RECORDS=1, SPLIT_RAW_BYTES=150, FAILED_SHUFFLE=0, MAP_OUTPUT_BYTES=18, PHYSICAL_MEMORY_BYTES=0, GC_TIME_MILLIS=46, MAP_OUTPUT_RECORDS=2, COMBINE_INPUT_RECORDS=0, CPU_MILLISECONDS=0, COMMITTED_HEAP_BYTES=201326592} \ No newline at end of file diff --git a/test/resources/mrdata/mapperTaskCounter2.properties b/test/resources/mrdata/mapperTaskCounter2.properties deleted file mode 100644 index 424b12fb4..000000000 --- a/test/resources/mrdata/mapperTaskCounter2.properties +++ /dev/null @@ -1,3 +0,0 @@ -org.apache.hadoop.mapreduce.FileSystemCounter={FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, HDFS_READ_OPS=4, HDFS_BYTES_READ=268, HDFS_LARGE_READ_OPS=0, FILE_READ_OPS=0, FILE_BYTES_WRITTEN=117858, FILE_BYTES_READ=0, HDFS_WRITE_OPS=0, HDFS_BYTES_WRITTEN=0} -org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter={BYTES_READ=118} -org.apache.hadoop.mapreduce.TaskCounter={MAP_OUTPUT_MATERIALIZED_BYTES=28, SPILLED_RECORDS=2, MERGED_MAP_OUTPUTS=0, VIRTUAL_MEMORY_BYTES=0, MAP_INPUT_RECORDS=1, SPLIT_RAW_BYTES=150, FAILED_SHUFFLE=0, MAP_OUTPUT_BYTES=18, PHYSICAL_MEMORY_BYTES=0, GC_TIME_MILLIS=46, MAP_OUTPUT_RECORDS=2, COMBINE_INPUT_RECORDS=0, CPU_MILLISECONDS=0, COMMITTED_HEAP_BYTES=201326592} \ No newline at end of file diff --git a/test/resources/mrdata/mapperTaskCounter3.properties b/test/resources/mrdata/mapperTaskCounter3.properties deleted file mode 100644 index 239c26f0d..000000000 --- a/test/resources/mrdata/mapperTaskCounter3.properties +++ /dev/null @@ -1,3 +0,0 @@ -org.apache.hadoop.mapreduce.FileSystemCounter={FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, HDFS_READ_OPS=4, HDFS_BYTES_READ=268, HDFS_LARGE_READ_OPS=0, FILE_READ_OPS=0, FILE_BYTES_WRITTEN=117858, FILE_BYTES_READ=0, HDFS_WRITE_OPS=0, HDFS_BYTES_WRITTEN=0} -org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter={BYTES_READ=118} -org.apache.hadoop.mapreduce.TaskCounter={MAP_OUTPUT_MATERIALIZED_BYTES=28, SPILLED_RECORDS=2, MERGED_MAP_OUTPUTS=0, VIRTUAL_MEMORY_BYTES=0, MAP_INPUT_RECORDS=1, SPLIT_RAW_BYTES=150, FAILED_SHUFFLE=0, MAP_OUTPUT_BYTES=18, PHYSICAL_MEMORY_BYTES=0, GC_TIME_MILLIS=43, MAP_OUTPUT_RECORDS=2, COMBINE_INPUT_RECORDS=0, CPU_MILLISECONDS=0, COMMITTED_HEAP_BYTES=201326592} \ No newline at end of file diff --git a/test/resources/mrdata/reducerTaskCounter1.properties b/test/resources/mrdata/reducerTaskCounter1.properties deleted file mode 100644 index 00f316618..000000000 --- a/test/resources/mrdata/reducerTaskCounter1.properties +++ /dev/null @@ -1,4 +0,0 @@ -org.apache.hadoop.mapreduce.FileSystemCounter={FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, HDFS_READ_OPS=3, HDFS_BYTES_READ=0, HDFS_LARGE_READ_OPS=0, FILE_READ_OPS=0, FILE_BYTES_WRITTEN=117847, FILE_BYTES_READ=72, HDFS_WRITE_OPS=3, HDFS_BYTES_WRITTEN=215} -org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter={BYTES_WRITTEN=97} -org.apache.hadoop.mapreduce.TaskCounter={REDUCE_INPUT_RECORDS=6, SPILLED_RECORDS=6, MERGED_MAP_OUTPUTS=3, VIRTUAL_MEMORY_BYTES=0, FAILED_SHUFFLE=0, REDUCE_SHUFFLE_BYTES=84, PHYSICAL_MEMORY_BYTES=0, GC_TIME_MILLIS=38, REDUCE_INPUT_GROUPS=2, COMBINE_OUTPUT_RECORDS=0, SHUFFLED_MAPS=3, REDUCE_OUTPUT_RECORDS=0, COMBINE_INPUT_RECORDS=0, CPU_MILLISECONDS=0, COMMITTED_HEAP_BYTES=147849216} -Shuffle Errors={CONNECTION=0, WRONG_LENGTH=0, BAD_ID=0, WRONG_REDUCE=0, IO_ERROR=0, WRONG_MAP=0} \ No newline at end of file diff --git a/test/resources/mrdata/sampleJobConf.properties b/test/resources/mrdata/sampleJobConf.properties deleted file mode 100644 index 4b9781d6c..000000000 --- a/test/resources/mrdata/sampleJobConf.properties +++ /dev/null @@ -1,4 +0,0 @@ -azkaban.link.job.url=https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2 -azkaban.link.attempt.url=https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0 -azkaban.link.workflow.url=https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder -azkaban.link.execution.url=https://elephant.linkedin.com:8443/executor?execid=1654676 diff --git a/test/resources/mrdata/sampleJobCounter.properties b/test/resources/mrdata/sampleJobCounter.properties deleted file mode 100644 index 12f261dbd..000000000 --- a/test/resources/mrdata/sampleJobCounter.properties +++ /dev/null @@ -1,6 +0,0 @@ -org.apache.hadoop.mapreduce.FileSystemCounter={FILE_LARGE_READ_OPS=0, FILE_WRITE_OPS=0, HDFS_READ_OPS=15, HDFS_BYTES_READ=804, HDFS_LARGE_READ_OPS=0, FILE_READ_OPS=0, FILE_BYTES_WRITTEN=471421, FILE_BYTES_READ=72, HDFS_WRITE_OPS=3, HDFS_BYTES_WRITTEN=215} -org.apache.hadoop.mapreduce.JobCounter={TOTAL_LAUNCHED_MAPS=3, VCORES_MILLIS_REDUCES=1870, MB_MILLIS_MAPS=7876608, TOTAL_LAUNCHED_REDUCES=1, SLOTS_MILLIS_REDUCES=1870, VCORES_MILLIS_MAPS=7692, MB_MILLIS_REDUCES=1914880, SLOTS_MILLIS_MAPS=7692, MILLIS_REDUCES=1870, MILLIS_MAPS=7692, DATA_LOCAL_MAPS=3} -org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter={BYTES_WRITTEN=97} -org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter={BYTES_READ=354} -org.apache.hadoop.mapreduce.TaskCounter={MAP_OUTPUT_MATERIALIZED_BYTES=84, REDUCE_INPUT_RECORDS=6, SPILLED_RECORDS=12, MERGED_MAP_OUTPUTS=3, VIRTUAL_MEMORY_BYTES=0, MAP_INPUT_RECORDS=3, SPLIT_RAW_BYTES=450, FAILED_SHUFFLE=0, MAP_OUTPUT_BYTES=54, REDUCE_SHUFFLE_BYTES=84, PHYSICAL_MEMORY_BYTES=0, GC_TIME_MILLIS=173, REDUCE_INPUT_GROUPS=2, COMBINE_OUTPUT_RECORDS=0, SHUFFLED_MAPS=3, REDUCE_OUTPUT_RECORDS=0, MAP_OUTPUT_RECORDS=6, COMBINE_INPUT_RECORDS=0, CPU_MILLISECONDS=0, COMMITTED_HEAP_BYTES=751828992} -Shuffle Errors={CONNECTION=0, WRONG_LENGTH=0, BAD_ID=0, WRONG_REDUCE=0, IO_ERROR=0, WRONG_MAP=0} \ No newline at end of file diff --git a/test/resources/spark-defaults.conf b/test/resources/spark-defaults.conf deleted file mode 100644 index de9cc375a..000000000 --- a/test/resources/spark-defaults.conf +++ /dev/null @@ -1,4 +0,0 @@ -spark.yarn.historyServer.address = jh1.grid.example.com:18080 -spark.eventLog.enabled = true -spark.eventLog.compress = true -spark.eventLog.dir = hdfs://nn1.grid.example.com:9000/logs/spark diff --git a/test/resources/spark_event_logs/event_log_1 b/test/resources/spark_event_logs/event_log_1 deleted file mode 100644 index 9507a9c8c..000000000 --- a/test/resources/spark_event_logs/event_log_1 +++ /dev/null @@ -1,33 +0,0 @@ -{"Event":"SparkListenerLogStart","Spark Version":"1.5.1"} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"10.20.0.71","Port":58838},"Maximum Memory":1111794647,"Timestamp":1458126388757} -{"Event":"SparkListenerEnvironmentUpdate","Spark Properties":{"spark.serializer":"org.apache.spark.serializer.KryoSerializer","spark.storage.memoryFraction":"0.3","spark.driver.memory":"2G","spark.executor.instances":"900","spark.executor.memory":"1g","spark.shuffle.memoryFraction":"0.5"},"JVM Information":{},"System Properties":{},"Classpath Entries":{}} -{"Event":"SparkListenerApplicationStart","App Name":"PythonPi","App ID":"application_1457600942802_0093","App Attempt ID":"1","Timestamp":1458126354336,"User":"hdfs"} -{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1458126390170,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"reduce at pi.py:39","Number of Tasks":10,"RDD Info":[{"RDD ID":1,"Name":"PythonRDD","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use ExternalBlockStore":false,"Deserialized":false,"Replication":1},"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"ExternalBlockStore Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use ExternalBlockStore":false,"Deserialized":false,"Replication":1},"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"ExternalBlockStore Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"","Accumulables":[]}],"Stage IDs":[0],"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"1\",\"name\":\"collect\"}","callSite.short":"reduce at pi.py:39"}} -{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"reduce at pi.py:39","Number of Tasks":10,"RDD Info":[{"RDD ID":1,"Name":"PythonRDD","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use ExternalBlockStore":false,"Deserialized":false,"Replication":1},"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"ExternalBlockStore Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use ExternalBlockStore":false,"Deserialized":false,"Replication":1},"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"ExternalBlockStore Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"","Submission Time":1458126390256,"Accumulables":[]},"Properties":{"spark.rdd.scope.noOverride":"true","spark.rdd.scope":"{\"id\":\"1\",\"name\":\"collect\"}","callSite.short":"reduce at pi.py:39"}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1458126397624,"Executor ID":"2","Executor Info":{"Host":".hello.com","Total Cores":2,"Log Urls":{"stdout":"http://hello.com:8042/node/containerlogs/container_e38_1457600942802_0093_01_000003/hdfs/stdout?start=-4096","stderr":"http://hello.com:8042/node/containerlogs/container_e38_1457600942802_0093_01_000003/hdfs/stderr?start=-4096"}}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"2","Host":".hello.com","Port":36478},"Maximum Memory":2223023063,"Timestamp":1458126398028} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1458126398712,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1458126398726,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerExecutorAdded","Timestamp":1458126398962,"Executor ID":"1","Executor Info":{"Host":".hello.com","Total Cores":2,"Log Urls":{"stdout":"http://.hello.com:8042/node/containerlogs/container_e38_1457600942802_0093_01_000002/hdfs/stdout?start=-4096","stderr":"http://.hello.com:8042/node/containerlogs/container_e38_1457600942802_0093_01_000002/hdfs/stderr?start=-4096"}}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1458126398970,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1458126398973,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":".hello.com","Port":38464},"Maximum Memory":2223023063,"Timestamp":1458126399357} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1458126403532,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":1,"Attempt":0,"Launch Time":1458126398726,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126403558,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":"hello.com","Executor Deserialize Time":1091,"Executor Run Time":408,"Result Size":1018,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1458126403563,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1458126398712,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126403565,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":"hello.com","Executor Deserialize Time":2605,"Executor Run Time":411,"Result Size":1018,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1458126404784,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":3,"Attempt":0,"Launch Time":1458126398973,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126404787,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":".hello.com","Executor Deserialize Time":1375,"Executor Run Time":473,"Result Size":1018,"JVM GC Time":48,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1458126404791,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":2,"Attempt":0,"Launch Time":1458126398970,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126404793,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":".hello.com","Executor Deserialize Time":1385,"Executor Run Time":473,"Result Size":1018,"JVM GC Time":48,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":8,"Attempt":0,"Launch Time":1458126405270,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":4,"Attempt":0,"Launch Time":1458126403532,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126405273,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":"hello.com","Executor Deserialize Time":16,"Executor Run Time":115,"Result Size":1018,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":9,"Attempt":0,"Launch Time":1458126406523,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Accumulables":[]}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":6,"Attempt":0,"Launch Time":1458126404784,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126406526,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":".hello.com","Executor Deserialize Time":14,"Executor Run Time":115,"Result Size":1018,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":5,"Attempt":0,"Launch Time":1458126403563,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126406809,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":"hello.com","Executor Deserialize Time":14,"Executor Run Time":125,"Result Size":1018,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":7,"Attempt":0,"Launch Time":1458126404791,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126408068,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":".hello.com","Executor Deserialize Time":15,"Executor Run Time":118,"Result Size":1018,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":8,"Attempt":0,"Launch Time":1458126405270,"Executor ID":"2","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126408323,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":"hello.com","Executor Deserialize Time":13,"Executor Run Time":116,"Result Size":1018,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":9,"Attempt":0,"Launch Time":1458126406523,"Executor ID":"1","Host":".hello.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1458126409598,"Failed":false,"Accumulables":[]},"Task Metrics":{"Host Name":".hello.com","Executor Deserialize Time":12,"Executor Run Time":116,"Result Size":1018,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0}} -{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"reduce at pi.py:39","Number of Tasks":10,"RDD Info":[{"RDD ID":1,"Name":"PythonRDD","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use ExternalBlockStore":false,"Deserialized":false,"Replication":1},"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"ExternalBlockStore Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use ExternalBlockStore":false,"Deserialized":false,"Replication":1},"Number of Partitions":10,"Number of Cached Partitions":0,"Memory Size":0,"ExternalBlockStore Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"","Submission Time":1458126390256,"Completion Time":1458126409599,"Accumulables":[]}} -{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1458126409602,"Job Result":{"Result":"JobSucceeded"}} -{"Event":"SparkListenerApplicationEnd","Timestamp":1458126409609} diff --git a/test/resources/spark_event_logs/event_log_2 b/test/resources/spark_event_logs/event_log_2 deleted file mode 100644 index 35101daa4..000000000 --- a/test/resources/spark_event_logs/event_log_2 +++ /dev/null @@ -1,4 +0,0 @@ -{"Event":"SparkListenerLogStart","Spark Version":"1.4.1"} -{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"127.0.0.1","Port":54157},"Maximum Memory":515411804,"Timestamp":1475761114342} -{"Event":"SparkListenerEnvironmentUpdate","Spark Properties":{"spark.serializer":"org.apache.spark.serializer.KryoSerializer","spark.storage.memoryFraction":"0.3","spark.driver.memory":"2G","spark.executor.instances":"900","spark.executor.memory":"1g","spark.shuffle.memoryFraction":"0.5"},"JVM Information":{},"System Properties":{},"Classpath Entries":{}} -{"Event":"SparkListenerApplicationStart","App Name":"app","App ID":"application_1","Timestamp":1475761112578,"User":"foo","App Attempt ID":"1"} diff --git a/test/resources/test-init.sql b/test/resources/test-init.sql deleted file mode 100644 index f441fd7c4..000000000 --- a/test/resources/test-init.sql +++ /dev/null @@ -1,41 +0,0 @@ -insert into yarn_app_result - (id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) values - ('application_1458194917883_1453361','Email Overwriter','growth','misc_default',1460980616502,1460980723925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453361','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654676','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654676','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 100, 30, 20), - ('application_1458194917883_1453362','Email Overwriter','metrics','misc_default',1460980823925,1460980923925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453362','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://elephant.linkedin.com:8443/executor?execid=1654677&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654677','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654677&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654677','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 200, 40, 10); - -insert into yarn_app_heuristic_result(id,yarn_app_result_id,heuristic_class,heuristic_name,severity,score) values (137594512,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), (137594513,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), (137594516,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), (137594520,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), (137594523,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), (137594525,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), (137594530,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), (137594531,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), (137594534,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), (137594537,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), (137594540,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0), (137594612,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), (137594613,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), (137594616,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), (137594620,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), (137594623,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), (137594625,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), (137594630,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), (137594631,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), (137594634,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), (137594637,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), (137594640,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0); - -insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name,value,details) values (137594512,'Group A','1 tasks @ 4 MB avg','NULL'), (137594512,'Group B','1 tasks @ 79 MB avg','NULL'), (137594512,'Number of tasks','2','NULL'), (137594513,'Avg task CPU time (ms)','11510','NULL'), (137594513,'Avg task GC time (ms)','76','NULL'), (137594513,'Avg task runtime (ms)','11851','NULL'), (137594513,'Number of tasks','2','NULL'), (137594513,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), (137594516,'Average task input size','42 MB','NULL'), (137594516,'Average task runtime','11 sec','NULL'), (137594516,'Max task runtime','12 sec','NULL'), (137594516,'Min task runtime','11 sec','NULL'), (137594516,'Number of tasks','2','NULL'), (137594520,'Median task input size','42 MB','NULL'), (137594520,'Median task runtime','11 sec','NULL'), (137594520,'Median task speed','3 MB/s','NULL'), (137594520,'Number of tasks','2','NULL'), (137594523,'Avg output records per task','56687','NULL'), (137594523,'Avg spilled records per task','79913','NULL'), (137594523,'Number of tasks','2','NULL'), (137594523,'Ratio of spilled records to output records','1.4097111356119074','NULL'), (137594525,'Avg Physical Memory (MB)','522','NULL'), (137594525,'Avg task runtime','11 sec','NULL'), (137594525,'Avg Virtual Memory (MB)','3307','NULL'), (137594525,'Max Physical Memory (MB)','595','NULL'), (137594525,'Min Physical Memory (MB)','449','NULL'), (137594525,'Number of tasks','2','NULL'), (137594525,'Requested Container Memory','2 GB','NULL'), (137594530,'Group A','11 tasks @ 868 KB avg','NULL'), (137594530,'Group B','9 tasks @ 883 KB avg ','NULL'), (137594530,'Number of tasks','20','NULL'), (137594531,'Avg task CPU time (ms)','8912','NULL'), (137594531,'Avg task GC time (ms)','73','NULL'), (137594531,'Avg task runtime (ms)','11045','NULL'), (137594531,'Number of tasks','20','NULL'), (137594531,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), (137594534,'Average task runtime','11 sec','NULL'), (137594534,'Max task runtime','14 sec','NULL'), (137594534,'Min task runtime','8 sec','NULL'), (137594534,'Number of tasks','20','NULL'), (137594537,'Avg Physical Memory (MB)','416','NULL'), (137594537,'Avg task runtime','11 sec','NULL'), (137594537,'Avg Virtual Memory (MB)','3326','NULL'), (137594537,'Max Physical Memory (MB)','497','NULL'), (137594537,'Min Physical Memory (MB)','354','NULL'), (137594537,'Number of tasks','20','NULL'), (137594537,'Requested Container Memory','2 GB','NULL'), (137594540,'Average code runtime','1 sec','NULL'), (137594540,'Average shuffle time','9 sec (5.49x)','NULL'), (137594540,'Average sort time','(0.04x)','NULL'), (137594540,'Number of tasks','20','NULL'), (137594612,'Group A','1 tasks @ 4 MB avg','NULL'), (137594612,'Group B','1 tasks @ 79 MB avg','NULL'), (137594612,'Number of tasks','2','NULL'), (137594613,'Avg task CPU time (ms)','11510','NULL'), (137594613,'Avg task GC time (ms)','76','NULL'), (137594613,'Avg task runtime (ms)','11851','NULL'), (137594613,'Number of tasks','2','NULL'), (137594613,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), (137594616,'Average task input size','42 MB','NULL'), (137594616,'Average task runtime','11 sec','NULL'), (137594616,'Max task runtime','12 sec','NULL'), (137594616,'Min task runtime','11 sec','NULL'), (137594616,'Number of tasks','2','NULL'), (137594620,'Median task input size','42 MB','NULL'), (137594620,'Median task runtime','11 sec','NULL'), (137594620,'Median task speed','3 MB/s','NULL'), (137594620,'Number of tasks','2','NULL'), (137594623,'Avg output records per task','56687','NULL'), (137594623,'Avg spilled records per task','79913','NULL'), (137594623,'Number of tasks','2','NULL'), (137594623,'Ratio of spilled records to output records','1.4097111356119074','NULL'), (137594625,'Avg Physical Memory (MB)','522','NULL'), (137594625,'Avg task runtime','11 sec','NULL'), (137594625,'Avg Virtual Memory (MB)','3307','NULL'), (137594625,'Max Physical Memory (MB)','595','NULL'), (137594625,'Min Physical Memory (MB)','449','NULL'), (137594625,'Number of tasks','2','NULL'), (137594625,'Requested Container Memory','2 GB','NULL'), (137594630,'Group A','11 tasks @ 868 KB avg','NULL'), (137594630,'Group B','9 tasks @ 883 KB avg ','NULL'), (137594630,'Number of tasks','20','NULL'), (137594631,'Avg task CPU time (ms)','8912','NULL'), (137594631,'Avg task GC time (ms)','73','NULL'), (137594631,'Avg task runtime (ms)','11045','NULL'), (137594631,'Number of tasks','20','NULL'), (137594631,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), (137594634,'Average task runtime','11 sec','NULL'), (137594634,'Max task runtime','14 sec','NULL'), (137594634,'Min task runtime','8 sec','NULL'), (137594634,'Number of tasks','20','NULL'), (137594637,'Avg Physical Memory (MB)','416','NULL'), (137594637,'Avg task runtime','11 sec','NULL'), (137594637,'Avg Virtual Memory (MB)','3326','NULL'), (137594637,'Max Physical Memory (MB)','497','NULL'), (137594637,'Min Physical Memory (MB)','354','NULL'), (137594637,'Number of tasks','20','NULL'), (137594637,'Requested Container Memory','2 GB','NULL'), (137594640,'Average code runtime','1 sec','NULL'), (137594640,'Average shuffle time','9 sec (5.49x)','NULL'), (137594640,'Average sort time','(0.04x)','NULL'), (137594640,'Number of tasks','20','NULL'); - -INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES (10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); - -INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES (100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','mkumar1','2018-02-12 08:40:42','2018-02-12 08:40:43'); - -INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) -VALUES (100003,'azkaban',1,1,40.29456456163195,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL); - -INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES (1496,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003),(1497,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389',10003),(1498,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495',10003),(1499,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589',10003),(1500,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680',10003),(1501,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818',10003),(1502,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057',10003); - -INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES -(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1496,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'), -(1542,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416389&job=countByCountryFlow_countByCountry&attempt=0',100003,1497,'SUCCEEDED',23.334004991319443,3.6118166666666665,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), -(1543,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416495&job=countByCountryFlow_countByCountry&attempt=0',100003,1498,'SUCCEEDED',21.28552951388889,3.2940833333333335,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), -(1544,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416589&job=countByCountryFlow_countByCountry&attempt=0',100003,1499,'SUCCEEDED',21.630970052083335,3.9560833333333334,324713861757,'2018-02-14 06:29:45','2018-02-14 06:29:45'), -(1545,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416680&job=countByCountryFlow_countByCountry&attempt=0',100003,1500,'SUCCEEDED',22.328486328125,3.7285166666666667,324713861757,'2018-02-14 07:29:47','2018-02-14 07:29:48'), -(1546,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416818&job=countByCountryFlow_countByCountry&attempt=0',100003,1501,'SUCCEEDED',32.16945149739583,5.203783333333333,324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'), -(1547,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5417057&job=countByCountryFlow_countByCountry&attempt=0',100003,1502,'SUCCEEDED', 27.2955078125, 4.047583333333334, 324713861757,'2018-02-14 07:29:48','2018-02-14 07:29:48'); - -INSERT INTO job_suggested_param_set(fitness_job_execution_id, tuning_algorithm_id, param_set_state, is_param_set_default, fitness, is_param_set_best, are_constraints_violated, job_definition_id,id) VALUES -(1541,1,'FITNESS_COMPUTED',0,0.06987967161749142,0,0,100003,1541), -(1542,1,'FITNESS_COMPUTED',0,0.07715930864495756,0,0,100003,1542), -(1543,1,'FITNESS_COMPUTED',0,0.07038554856075895,0,0,100003,1543), -(1544,1,'FITNESS_COMPUTED',0,0.07152782795578526,0,0,100003,1544), -(1545,1,'FITNESS_COMPUTED',0,0.07383432757503201,0,0,100003,1545), -(1546,1,'FITNESS_COMPUTED',0,0.10637576523832741,0,0,100003,1546), -(1547,1,'FITNESS_COMPUTED',0,0.09025893809095505,0,0,100003,1547); - -INSERT INTO job_suggested_param_value(id, job_suggested_param_set_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES -(3209,1541,1,149.5239493606563,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3210,1541,2,1536,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3211,1541,3,10,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3212,1541,4,0.761466551875019,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3213,1541,5,2844.365182469904,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3214,1541,6,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3215,1541,7,2133.273886852428,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3216,1541,8,1152,'2018-02-14 05:30:42','2018-02-14 05:30:42'),(3217,1541,9,536870912,'2018-02-14 05:30:42','2018-02-14 05:30:42'), (3218,1542,1,162.7907384046847,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3219,1542,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3220,1542,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3221,1542,4,0.7632081784681852,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3222,1542,5,2809.5806453243313,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3223,1542,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3224,1542,7,2107.1854839932485,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3225,1542,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3226,1542,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3227,1543,1,124.05878355054111,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3228,1543,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3229,1543,3,12.521341191290857,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3230,1543,4,0.7622909149004323,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3231,1543,5,2041.562366831904,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3232,1543,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3233,1543,7,1531.171775123928,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3234,1543,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3235,1543,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3236,1544,1,149.51252503919468,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3237,1544,2,1536,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3238,1544,3,10,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3239,1544,4,0.7619961998308155,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3240,1544,5,2844.326081249364,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3241,1544,6,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3242,1544,7,2133.244560937023,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3243,1544,8,1152,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3244,1544,9,536870912,'2018-02-14 06:29:45','2018-02-14 06:29:45'),(3245,1545,1,159.88928672056016,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3246,1545,2,1536,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3247,1545,3,10,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3248,1545,4,0.770164839202443,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3249,1545,5,2863.372720073011,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3250,1545,6,536870912,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3251,1545,7,2147.5295400547584,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3252,1545,8,1152,'2018-02-14 07:29:47','2018-02-14 07:29:47'),(3253,1545,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3254,1546,1,201.64225529876035,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3255,1546,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3256,1546,3,10.29839988592941,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3257,1546,4,0.7635183100860585,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3258,1546,5,2789.189282499988,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3259,1546,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3260,1546,7,2091.891961874991,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3261,1546,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3262,1546,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3263,1547,1,149.52419594024295,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3264,1547,2,1536,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3265,1547,3,10,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3266,1547,4,0.7630834894363029,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3267,1547,5,2844.1716734703073,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3268,1547,6,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3269,1547,7,2133.1287551027303,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3270,1547,8,1152,'2018-02-14 07:29:48','2018-02-14 07:29:48'),(3271,1547,9,536870912,'2018-02-14 07:29:48','2018-02-14 07:29:48'); - -INSERT INTO backfill_info VALUES ('MAPREDUCE', 123456789); -INSERT INTO backfill_info VALUES ('SPARK', 123456779); diff --git a/test/resources/tunein-test1.sql b/test/resources/tunein-test1.sql deleted file mode 100644 index aee0f0d22..000000000 --- a/test/resources/tunein-test1.sql +++ /dev/null @@ -1,45 +0,0 @@ -INSERT INTO `flow_definition`(id, flow_def_id, flow_def_url) VALUES -(57,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images'), -(60,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall'); - - -INSERT INTO `job_definition`(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES -(51,'https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action',57,'score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','azkaban','dev_svc',parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss')), -(54,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry',60,'countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','azkaban','mkumar1',parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss')); - -INSERT INTO `tuning_job_definition`(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, -average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) VALUES -(51,'azkaban',1,1,458.4372180906033,61.92735384666667,5845242598595,200,200,parsedatetime('2018-01-13 17:27:09','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-13 17:28:34','dd-MM-yyyy hh:mm:ss'), NULL), -(54,'azkaban',1,1,0.09060929361979166,1.8950733333333332,532614133,200,200,parsedatetime('2018-01-22 11:41:12','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-22 15:49:33','dd-MM-yyyy hh:mm:ss'), NULL); - -INSERT INTO `flow_execution`(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES (846,'https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/executor?execid=5157830',57),(847,'https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/executor?execid=5158194',57),(848,'https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/executor?execid=5158533',57),(2035,'https://elephant.linkedin.com:8443/executor?execid=5356366','https://elephant.linkedin.com:8443/executor?execid=5356366',60),(2036,'https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/executor?execid=5356377',60),(2047,'https://elephant.linkedin.com:8443/executor?execid=5356853','https://elephant.linkedin.com:8443/executor?execid=5356853',60); - -INSERT INTO `job_execution`(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES -(1624,'https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0',51,847,'SUCCEEDED',196.53583333333333,43.996566666666666,3990998122525,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:08:44','dd-MM-yyyy hh:mm:ss')), -(1625,'https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0',51,846,'SUCCEEDED',169.09791666666666,34.86125,3990827471029,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:58:41','dd-MM-yyyy hh:mm:ss')), -(1626,'https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0',51,848,'SUCCEEDED',166.11400634765624,33.7234,3990138654674,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')), -(1719,'https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356366&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2035,'FAILED',0,0,0,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')), -(1720,'https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2036,'SUCCEEDED',0.07686197916666666,3.5777,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-09 05:52:42','dd-MM-yyyy hh:mm:ss')), -(1721,'https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356853&job=countByCountryFlowSmall_countByCountry&attempt=0',54,2047,'SUCCEEDED',0.21555555555555556,3.5456666666666665,540598828,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')); - -INSERT INTO `job_suggested_param_set`(fitness_job_execution_id, tuning_algorithm_id, param_set_state, is_param_set_default, fitness, is_param_set_best, are_constraints_violated, job_definition_id,id) VALUES -(1624,1,'FITNESS_COMPUTED',0,0.05287618226970775,0,0,51,1624), -(1625,1,'FITNESS_COMPUTED',0,0.04549620518409709,0,0,51,1625), -(1626,1,'FITNESS_COMPUTED',0,0.044701092268747876,0,0,51,1626), -(NULL,1,'CREATED',0,NULL,0,0,51,1627), -(NULL,1,'CREATED',0,NULL,0,0,51,1628), -(NULL,1,'CREATED',0,NULL,0,0,51,1629), -(1719,1,'FITNESS_COMPUTED',0,1.0960015760903588,0,0,54,1719), -(1720,1,'FITNESS_COMPUTED',0,0.15266389313494158,0,0,54,1720), -(1721,1,'FITNESS_COMPUTED',0,1.079813530812908,0,0,54,1721), -(2833,1,'FITNESS_COMPUTED',0,0.54365576171875,0,0,54,2833), -(NULL,1,'CREATED',0,NULL,0,0,54,2834), -(NULL,1,'CREATED',0,NULL,0,0,54,2835); - -INSERT INTO `job_suggested_param_value`(id, job_suggested_param_set_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES -(12569,1624,1,218.6447449212757,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')), -(12570,1624,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12571,1624,3,43.796210012462936,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12572,1624,4,0.7053483939633272,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12573,1624,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12574,1624,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12575,1624,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12576,1624,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12577,1624,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12578,1625,1,181.9245696813968,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12579,1625,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12580,1625,3,42.047012100221565,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12581,1625,4,0.7037704502574763,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12582,1625,5,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12583,1625,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12584,1625,7,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12585,1625,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12586,1625,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12587,1626,1,140.63900164342985,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12588,1626,2,1536,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12589,1626,3,51.426234701754865,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12590,1626,4,0.6951498772580411,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12591,1626,5,1976.5719588408601,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12592,1626,6,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12593,1626,7,1482.428969130645,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12594,1626,8,1152,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12595,1626,9,536870912,parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 03:03:05','dd-MM-yyyy hh:mm:ss')),(12596,1627,1,171.02605253760447,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12597,1627,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12598,1627,3,54.84590775607139,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12599,1627,4,0.7115250205281223,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12600,1627,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12601,1627,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12602,1627,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12603,1627,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12604,1627,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12605,1628,1,181.9245696813968,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12606,1628,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12607,1628,3,42.047012100221565,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12608,1628,4,0.7037704502574763,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12609,1628,5,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12610,1628,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12611,1628,7,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12612,1628,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12613,1628,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12614,1629,1,213.25485474773512,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12615,1629,2,1536,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12616,1629,3,31.525782197486357,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12617,1629,4,0.7476923282427653,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12618,1629,5,1581.3588284589678,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12619,1629,6,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12620,1629,7,1186.0191213442258,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12621,1629,8,1152,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(12622,1629,9,536870912,parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-15 05:59:21','dd-MM-yyyy hh:mm:ss')),(13330,1719,1,792.634660843181,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13331,1719,2,2492.7849019072983,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13332,1719,3,60.45853018964071,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13333,1719,4,0.7336072933432902,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13334,1719,5,2962.9708043614564,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13335,1719,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13336,1719,7,2222.2281032710926,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13337,1719,8,1869.5886764304737,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13338,1719,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13339,1720,1,872.5176554279782,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13340,1720,2,2535.6175654819463,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13341,1720,3,44.33267220160804,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13342,1720,4,0.7337286195207247,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13343,1720,5,3083.5815515791987,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13344,1720,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13345,1720,7,2312.686163684399,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13346,1720,8,1901.7131741114597,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13347,1720,9,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13348,1721,1,885.9516064178075,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13349,1721,2,2085.536516326114,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13350,1721,3,42.55444706800388,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13351,1721,4,0.7348330520625498,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13352,1721,5,3354.63139252968,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13353,1721,6,536870912,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13354,1721,7,2515.97354439726,parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:41','dd-MM-yyyy hh:mm:ss')),(13355,1721,8,1564.1523872445855,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(13356,1721,9,536870912,parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-01-23 04:06:42','dd-MM-yyyy hh:mm:ss')),(20462,2833,1,1459.6899166265118,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20463,2833,2,2421.8450860158327,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20464,2833,3,45.39309760195506,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20465,2833,4,0.7338175361303875,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20466,2833,5,2690.0010009930634,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20467,2833,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20468,2833,7,2017.5007507447976,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20469,2833,8,1816.3838145118746,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20470,2833,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20471,2834,1,678.4611819565604,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20472,2834,2,2233.783342819729,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20473,2834,3,67.46065454683335,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20474,2834,4,0.7337482196069645,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20475,2834,5,3068.4288351321584,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20476,2834,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20477,2834,7,2301.3216263491186,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20478,2834,8,1675.3375071147968,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20479,2834,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss')),(20480,2835,1,888.1718411828987,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20481,2835,2,2391.4206410271354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20482,2835,3,63.83042592878694,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20483,2835,4,0.7326134736198681,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20484,2835,5,2987.2156090895137,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20485,2835,6,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20486,2835,7,2240.4117068171354,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20487,2835,8,1793.5654807703515,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')),(20488,2835,9,536870912,parsedatetime('2018-02-13 18:02:55','dd-MM-yyyy hh:mm:ss'),parsedatetime('2018-02-13 18:02:56','dd-MM-yyyy hh:mm:ss')); - -INSERT INTO `yarn_app_result`(id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) VALUES ('application_1506645932520_14618965','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864742550,1515864947012,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618965','Pig',3,1809,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',104318976,16860147,12093),('application_1506645932520_14618978','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864747221,1515865052849,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618978','Pig',3,5835,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',349622272,53433482,24885),('application_1506645932520_14618986','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864749064,1515864871027,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618986','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',423936,75348,3168),('application_1506645932520_14618990','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864750944,1515864963817,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618990','Pig',2,408,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',33875968,5673563,42593),('application_1506645932520_14618994','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515864752042,1515864968073,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14618994','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5146403&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5146403','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',985088,149591,3109),('application_1506645932520_14819554','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987211575,1515987345216,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819554','Pig',2,994,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41111040,0,5629),('application_1506645932520_14819557','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987213364,1515987353613,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819557','Pig',4,6520,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',148128768,0,4803),('application_1506645932520_14819559','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987214957,1515987289457,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819559','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',159744,0,2335),('application_1506645932520_14819561','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987216374,1515987328839,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819561','Pig',2,298,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',12905472,0,2710),('application_1506645932520_14819564','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987217458,1515987328877,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819564','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',242688,0,3036),('application_1506645932520_14819566','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987218578,1515987337373,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819566','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',276480,0,2428),('application_1506645932520_14819567','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987219755,1515987345926,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819567','Pig',2,996,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',39914496,0,4385),('application_1506645932520_14819799','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987384869,1515987654239,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819799','Pig',3,3171,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',84836352,0,8879),('application_1506645932520_14819804','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987386490,1515987624024,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819804','Pig',3,3168,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',86373888,0,7496),('application_1506645932520_14819806','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987387715,1515987628906,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14819806','Pig',4,10113,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',204218880,0,12516),('application_1506645932520_14820502','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987672309,1515987731074,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820502','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',95232,0,4795),('application_1506645932520_14820601','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987743030,1515987809262,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820601','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',79872,0,4812),('application_1506645932520_14820710','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987821752,1515987866959,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820710','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',46080,0,5151),('application_1506645932520_14820781','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515987880705,1515988060797,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820781','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',2429952,0,30350),('application_1506645932520_14820903','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988082745,1515988166282,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820903','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1453056,0,4934),('application_1506645932520_14820904','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988083623,1515988172147,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820904','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',615936,0,5217),('application_1506645932520_14820960','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988182390,1515988255602,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820960','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',328704,0,5271),('application_1506645932520_14820995','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515988270096,1515988320392,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14820995','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5157830&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5157830','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',145920,0,4778),('application_1506645932520_14824675','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991076714,1515991231731,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824675','Pig',3,1518,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',51489792,0,3885),('application_1506645932520_14824676','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991079060,1515991237503,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824676','Pig',3,4896,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',158307840,0,5956),('application_1506645932520_14824677','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991080425,1515991187851,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824677','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',233472,0,2877),('application_1506645932520_14824680','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991082049,1515991240032,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824680','Pig',2,302,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',13275648,0,43306),('application_1506645932520_14824683','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991083758,1515991247488,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824683','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',281088,0,40714),('application_1506645932520_14824685','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991085047,1515991265419,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824685','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',287232,0,41607),('application_1506645932520_14824687','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991086759,1515991294536,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824687','Pig',3,1500,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41547264,0,70495),('application_1506645932520_14824996','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991326143,1515991641190,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824996','Pig',3,3213,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',92054016,0,6387),('application_1506645932520_14824999','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991327452,1515991670369,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14824999','Pig',3,3165,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',117229056,0,11406),('application_1506645932520_14825000','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991328750,1515991660916,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825000','Pig',4,10129,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',243571200,0,19257),('application_1506645932520_14825298','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991684212,1515991767524,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825298','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',127488,0,4634),('application_1506645932520_14825364','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991778730,1515991873235,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825364','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',124416,0,4678),('application_1506645932520_14825463','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991888162,1515991971005,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825463','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',56832,0,37362),('application_1506645932520_14825523','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515991982696,1515992198290,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825523','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',3177984,0,6575),('application_1506645932520_14825763','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992221397,1515992318476,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825763','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1463808,0,18682),('application_1506645932520_14825769','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992222425,1515992350395,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14825769','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',737280,0,4593),('application_1506645932520_14826095','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992373009,1515992451751,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14826095','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',347136,0,5224),('application_1506645932520_14826163','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515992466247,1515992538144,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14826163','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158194&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158194','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',198144,0,5388),('application_1506645932520_14828901','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994413751,1515994540123,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828901','Pig',3,1527,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',41195520,0,3799),('application_1506645932520_14828909','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994415531,1515994564943,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828909','Pig',4,6496,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',135762432,0,21059),('application_1506645932520_14828915','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994417787,1515994516440,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828915','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',192000,0,2372),('application_1506645932520_14828918','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994418955,1515994535247,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828918','Pig',2,300,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',11008512,0,3043),('application_1506645932520_14828920','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994420115,1515994548888,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828920','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',271872,0,2402),('application_1506645932520_14828927','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994421136,1515994524042,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828927','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',215040,0,3006),('application_1506645932520_14828931','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994422273,1515994554177,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14828931','Pig',2,990,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',40410624,0,17972),('application_1506645932520_14829374','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994598599,1515994826719,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829374','Pig',3,3234,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',76623663,0,13428),('application_1506645932520_14829375','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994599701,1515994811605,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829375','Pig',3,2643,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',73923525,0,12955),('application_1506645932520_14829377','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994601160,1515994871831,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829377','Pig',4,10093,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',227204472,0,12798),('application_1506645932520_14829662','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994884732,1515994942466,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829662','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',100554,0,5346),('application_1506645932520_14829738','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515994954814,1515995030508,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829738','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',102957,0,4962),('application_1506645932520_14829819','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995045481,1515995093836,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829819','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',58839,0,5399),('application_1506645932520_14829871','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995113082,1515995270705,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14829871','Pig',1,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',2724270,0,4927),('application_1506645932520_14830034','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995291854,1515995356948,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830034','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',1442952,0,4843),('application_1506645932520_14830035','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995293330,1515995363322,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830035','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',609561,0,4607),('application_1506645932520_14830183','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995379978,1515995475455,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830183','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',346434,0,35580),('application_1506645932520_14830345','PigLatin:generateImpressionAction.pig','dev_svc','sna_default',1515995489495,1515995541281,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_14830345','Pig',0,0,0,'azkaban','score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images','https://elephant.linkedin.com:8443/executor?execid=5158533&job=score-all-images_generate-impression-action&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5158533','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images&job=score-all-images_generate-impression-action','https://elephant.linkedin.com:8443/manager?project=nus-virals-autotuning-test&flow=score-all-images',169446,0,4860),('application_1506645932520_18304591','PigLatin:count_by_country_small.pig','mkumar1','sna_default',1517997086752,1517997308615,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_18304591','Pig',2,2,0,'azkaban','countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmallNew_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew','https://elephant.linkedin.com:8443/executor?execid=5356377&job=countByCountryFlowSmallNew_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5356377','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew',28334400,1248,7201),('application_1506645932520_18311651','PigLatin:count_by_country_small.pig','mkumar1','sna_default',1518001319781,1518001775270,'http://ltx1-holdemjh01.grid.linkedin.com:19888/jobhistory/job/job_1506645932520_18311651','Pig',3,6,0,'azkaban','countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5221700','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall','https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5221700','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall',794624,469908,242749); -INSERT INTO `yarn_app_heuristic_result` VALUES (1511830,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511831,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511832,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1511833,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511834,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511835,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511836,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511837,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511838,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511839,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511840,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511841,'application_1506645932520_14618986','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511842,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511843,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511844,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1511845,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511846,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511847,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511848,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511849,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511850,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511851,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511852,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511853,'application_1506645932520_14618994','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511866,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511867,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511868,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,408),(1511869,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511870,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511871,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511872,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511873,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511874,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511875,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511876,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511877,'application_1506645932520_14618990','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511878,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511879,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511880,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1809),(1511881,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511882,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511883,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511884,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511885,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511886,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511887,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511888,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511889,'application_1506645932520_14618965','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1511902,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1511903,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1511904,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,5835),(1511905,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1511906,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1511907,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1511908,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1511909,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1511910,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1511911,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1511912,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1511913,'application_1506645932520_14618978','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519428,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519429,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519430,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519431,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519432,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519433,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519434,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519435,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519436,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519437,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519438,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519439,'application_1506645932520_14819566','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519440,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519441,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519442,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519443,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519444,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519445,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519446,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519447,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519448,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519449,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519450,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519451,'application_1506645932520_14819564','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519452,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519453,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519454,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519455,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519456,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519457,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519458,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519459,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519460,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519461,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519462,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519463,'application_1506645932520_14819559','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519464,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519465,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519466,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,298),(1519467,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519468,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519469,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519470,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519471,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519472,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519473,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519474,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519475,'application_1506645932520_14819561','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519476,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519477,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519478,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,996),(1519479,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519480,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519481,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519482,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519483,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519484,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519485,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519486,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519487,'application_1506645932520_14819567','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519488,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519489,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519490,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,994),(1519491,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519492,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519493,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519494,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519495,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519496,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519497,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519498,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519499,'application_1506645932520_14819554','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519512,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519513,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519514,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,6520),(1519515,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519516,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519517,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519518,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519519,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519520,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519521,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519522,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519523,'application_1506645932520_14819557','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519524,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519525,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519526,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1503),(1519527,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519528,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519529,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519530,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519531,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519532,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1665),(1519533,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519534,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519535,'application_1506645932520_14819804','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519536,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519537,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519538,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519539,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519540,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519541,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519542,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519543,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519544,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519545,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519546,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519547,'application_1506645932520_14820502','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519548,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519549,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519550,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519551,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519552,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519553,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519554,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519555,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519556,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519557,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519558,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519559,'application_1506645932520_14820601','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519560,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519561,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519562,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519563,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519564,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519565,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519566,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519567,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519568,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519569,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519570,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519571,'application_1506645932520_14820710','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519572,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519573,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519574,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519575,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519576,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519577,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519578,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1519579,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519580,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519581,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519582,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519583,'application_1506645932520_14820781','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519584,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519585,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519586,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519587,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519588,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519589,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519590,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519591,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519592,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519593,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519594,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519595,'application_1506645932520_14820903','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519596,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519597,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519598,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519599,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519600,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519601,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519602,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519603,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519604,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519605,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519606,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519607,'application_1506645932520_14820904','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519608,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519609,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519610,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519611,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519612,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519613,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519614,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519615,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519616,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519617,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519618,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519619,'application_1506645932520_14820960','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519620,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519621,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519622,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519623,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519624,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519625,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519626,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519627,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519628,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519629,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519630,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519631,'application_1506645932520_14820995','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519632,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519633,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519634,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1500),(1519635,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519636,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519637,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519638,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519639,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519640,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1671),(1519641,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519642,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519643,'application_1506645932520_14819799','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519644,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519645,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519646,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519647,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519648,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519649,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519650,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519651,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519652,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519653,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519654,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519655,'application_1506645932520_14824677','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519656,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519657,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519658,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,302),(1519659,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519660,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519661,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519662,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519663,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519664,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519665,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519666,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519667,'application_1506645932520_14824680','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519668,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519669,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519670,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1518),(1519671,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519672,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519673,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519674,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519675,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519676,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519677,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519678,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519679,'application_1506645932520_14824675','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519680,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519681,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519682,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7116),(1519683,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519684,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519685,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519686,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519687,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519688,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1519689,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519690,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519691,'application_1506645932520_14819806','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519692,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519693,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519694,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519695,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519696,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519697,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519698,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519699,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519700,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519701,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519702,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519703,'application_1506645932520_14824685','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519704,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519705,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519706,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519707,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519708,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519709,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519710,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519711,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519712,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519713,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519714,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519715,'application_1506645932520_14824683','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519716,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519717,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519718,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1500),(1519719,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519720,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519721,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519722,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519723,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519724,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519725,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519726,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519727,'application_1506645932520_14824687','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519728,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519729,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519730,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,4896),(1519731,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519732,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519733,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519734,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519735,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519736,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519737,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519738,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519739,'application_1506645932520_14824676','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519740,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519741,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519742,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1527),(1519743,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519744,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519745,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519746,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519747,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519748,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1686),(1519749,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519750,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519751,'application_1506645932520_14824996','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519752,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519753,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519754,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519755,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519756,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519757,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519758,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519759,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519760,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519761,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519762,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519763,'application_1506645932520_14825298','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519764,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519765,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519766,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519767,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519768,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519769,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519770,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519771,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519772,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519773,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519774,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519775,'application_1506645932520_14825364','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519776,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519777,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519778,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519779,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519780,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519781,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519782,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519783,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519784,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519785,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519786,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519787,'application_1506645932520_14825463','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519788,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519789,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519790,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519791,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519792,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519793,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519794,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1519795,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519796,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519797,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519798,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519799,'application_1506645932520_14825523','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519800,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519801,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519802,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519803,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519804,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519805,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519806,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519807,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519808,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519809,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519810,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519811,'application_1506645932520_14825763','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519812,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519813,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519814,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519815,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519816,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519817,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519818,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519819,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519820,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519821,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519822,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519823,'application_1506645932520_14825769','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519824,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519825,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519826,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519827,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519828,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519829,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519830,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519831,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519832,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519833,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519834,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519835,'application_1506645932520_14826095','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519836,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519837,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519838,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519839,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519840,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519841,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519842,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519843,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519844,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519845,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519846,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519847,'application_1506645932520_14826163','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519848,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519849,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519850,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1509),(1519851,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519852,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519853,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519854,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519855,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519856,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1656),(1519857,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519858,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519859,'application_1506645932520_14824999','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519860,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519861,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519862,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519863,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519864,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519865,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519866,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519867,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519868,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519869,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519870,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519871,'application_1506645932520_14828927','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519872,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519873,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519874,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,300),(1519875,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519876,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519877,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519878,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519879,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519880,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519881,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519882,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519883,'application_1506645932520_14828918','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519884,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519885,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519886,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519887,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519888,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519889,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519890,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519891,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519892,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519893,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519894,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519895,'application_1506645932520_14828915','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519896,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519897,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519898,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1527),(1519899,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519900,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519901,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519902,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519903,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519904,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519905,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519906,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519907,'application_1506645932520_14828901','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519908,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519909,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519910,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519911,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519912,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519913,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519914,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519915,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519916,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519917,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519918,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519919,'application_1506645932520_14828920','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519920,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519921,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519922,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,990),(1519923,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519924,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519925,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519926,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519927,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519928,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519929,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519930,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519931,'application_1506645932520_14828931','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519932,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519933,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519934,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7132),(1519935,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519936,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519937,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519938,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519939,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519940,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1519941,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519942,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519943,'application_1506645932520_14825000','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519944,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519945,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519946,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,6496),(1519947,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519948,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519949,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519950,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519951,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519952,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519953,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519954,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519955,'application_1506645932520_14828909','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519956,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519957,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519958,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',2,996),(1519959,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519960,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519961,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519962,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519963,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519964,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1647),(1519965,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519966,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519967,'application_1506645932520_14829375','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519968,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519969,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519970,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519971,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519972,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519973,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519974,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519975,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519976,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519977,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519978,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519979,'application_1506645932520_14829662','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519980,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519981,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519982,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519983,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519984,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519985,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519986,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519987,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1519988,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1519989,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1519990,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1519991,'application_1506645932520_14829738','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1519992,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1519993,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1519994,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1519995,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1519996,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1519997,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1519998,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1519999,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520000,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520001,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520002,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520003,'application_1506645932520_14829819','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520004,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520005,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520006,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520007,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520008,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520009,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520010,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',1,0),(1520011,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520012,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520013,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520014,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520015,'application_1506645932520_14829871','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520016,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520017,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520018,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520019,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520020,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520021,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520022,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520023,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520024,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520025,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520026,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520027,'application_1506645932520_14830035','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520028,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520029,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520030,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520031,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520032,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520033,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520034,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520035,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520036,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520037,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520038,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520039,'application_1506645932520_14830034','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520040,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520041,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520042,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520043,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520044,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520045,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520046,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520047,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520048,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520049,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520050,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520051,'application_1506645932520_14830183','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520052,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520053,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520054,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1520055,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520056,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520057,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520058,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520059,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520060,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1520061,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520062,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520063,'application_1506645932520_14830345','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520064,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520065,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520066,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',3,1536),(1520067,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520068,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520069,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520070,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520071,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520072,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,1698),(1520073,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520074,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520075,'application_1506645932520_14829374','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1520076,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1520077,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1520078,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',4,7096),(1520079,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1520080,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1520081,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1520082,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1520083,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1520084,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',3,2997),(1520085,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(1520086,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1520087,'application_1506645932520_14829377','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1536115,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1536116,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1536117,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1536118,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1536119,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1536120,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(1536121,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1536122,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1536123,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1536124,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',2,2),(1536125,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1536126,'application_1506645932520_18304591','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0),(1536163,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperDataSkewHeuristic','Mapper Data Skew',0,0),(1536164,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(1536165,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(1536166,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(1536167,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(1536168,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',3,3),(1536169,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerDataSkewHeuristic','Reducer Data Skew',0,0),(1536170,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(1536171,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(1536172,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',3,3),(1536173,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0),(1536174,'application_1506645932520_18311651','com.linkedin.drelephant.mapreduce.heuristics.DistributedCacheLimitHeuristic','Distributed Cache Limit',0,0); -INSERT INTO `yarn_app_heuristic_result_details`(yarn_app_heuristic_result_id,name,value,details) VALUES (1511830,'Group A','1 tasks @ 9 MB avg',NULL),(1511830,'Group B','2 tasks @ 513 MB avg',NULL),(1511830,'Number of tasks','3',NULL),(1511831,'Avg task CPU time (ms)','55903',NULL),(1511831,'Avg task GC time (ms)','316',NULL),(1511831,'Avg task runtime (ms)','69313',NULL),(1511831,'Number of tasks','3',NULL),(1511831,'Task GC/CPU ratio','0.005652648337298535',NULL),(1511832,'Average task input size','345 MB',NULL),(1511832,'Average task runtime','1 min 9 sec',NULL),(1511832,'Max task runtime','1 min 42 sec',NULL),(1511832,'Min task runtime','9 sec',NULL),(1511832,'Number of tasks','3',NULL),(1511833,'Median task input size','511 MB',NULL),(1511833,'Median task runtime','1 min 36 sec',NULL),(1511833,'Median task speed','5 MB/s',NULL),(1511833,'Number of tasks','3',NULL),(1511833,'Total input size in MB','1035.5147895812988',NULL),(1511834,'Avg output records per task','1094880',NULL),(1511834,'Avg spilled records per task','0',NULL),(1511834,'Number of tasks','3',NULL),(1511834,'Ratio of spilled records to output records','0.0',NULL),(1511835,'Avg Physical Memory (MB)','656',NULL),(1511835,'Avg task runtime','1 min 9 sec',NULL),(1511835,'Avg Virtual Memory (MB)','2354',NULL),(1511835,'Max Physical Memory (MB)','743',NULL),(1511835,'Min Physical Memory (MB)','500',NULL),(1511835,'Number of tasks','3',NULL),(1511835,'Requested Container Memory','2 GB',NULL),(1511836,'Group A','0 tasks @ 0 bytes avg',NULL),(1511836,'Group B','0 tasks @ 0 bytes avg',NULL),(1511836,'Number of tasks','0',NULL),(1511837,'Avg task CPU time (ms)','0',NULL),(1511837,'Avg task GC time (ms)','0',NULL),(1511837,'Avg task runtime (ms)','0',NULL),(1511837,'Number of tasks','0',NULL),(1511837,'Task GC/CPU ratio','0.0',NULL),(1511838,'Average task runtime','0 sec',NULL),(1511838,'Max task runtime','0 sec',NULL),(1511838,'Min task runtime','0 sec',NULL),(1511838,'Number of tasks','0',NULL),(1511839,'Avg Physical Memory (MB)','0',NULL),(1511839,'Avg task runtime','0 sec',NULL),(1511839,'Avg Virtual Memory (MB)','0',NULL),(1511839,'Max Physical Memory (MB)','0',NULL),(1511839,'Min Physical Memory (MB)','0',NULL),(1511839,'Number of tasks','0',NULL),(1511839,'Requested Container Memory','2 GB',NULL),(1511840,'Average code runtime','0 sec',NULL),(1511840,'Average shuffle time','0 sec ',NULL),(1511840,'Average sort time','0 sec ',NULL),(1511840,'Number of tasks','0',NULL),(1511842,'Group A','1 tasks @ 157 MB avg',NULL),(1511842,'Group B','2 tasks @ 504 MB avg',NULL),(1511842,'Number of tasks','3',NULL),(1511843,'Avg task CPU time (ms)','148426',NULL),(1511843,'Avg task GC time (ms)','3774',NULL),(1511843,'Avg task runtime (ms)','161024',NULL),(1511843,'Number of tasks','3',NULL),(1511843,'Task GC/CPU ratio','0.025426812014067615',NULL),(1511844,'Average task input size','388 MB',NULL),(1511844,'Average task runtime','2 min 41 sec',NULL),(1511844,'Max task runtime','3 min 22 sec',NULL),(1511844,'Min task runtime','1 min 25 sec',NULL),(1511844,'Number of tasks','3',NULL),(1511845,'Median task input size','500 MB',NULL),(1511845,'Median task runtime','3 min 14 sec',NULL),(1511845,'Median task speed','2 MB/s',NULL),(1511845,'Number of tasks','3',NULL),(1511845,'Total input size in MB','1165.6459951400757',NULL),(1511846,'Avg output records per task','1157494',NULL),(1511846,'Avg spilled records per task','0',NULL),(1511846,'Number of tasks','3',NULL),(1511846,'Ratio of spilled records to output records','0.0',NULL),(1511847,'Avg Physical Memory (MB)','904',NULL),(1511847,'Avg task runtime','2 min 41 sec',NULL),(1511847,'Avg Virtual Memory (MB)','2388',NULL),(1511847,'Max Physical Memory (MB)','1109',NULL),(1511847,'Min Physical Memory (MB)','747',NULL),(1511847,'Number of tasks','3',NULL),(1511847,'Requested Container Memory','2 GB',NULL),(1511848,'Group A','0 tasks @ 0 bytes avg',NULL),(1511848,'Group B','0 tasks @ 0 bytes avg',NULL),(1511848,'Number of tasks','0',NULL),(1511849,'Avg task CPU time (ms)','0',NULL),(1511849,'Avg task GC time (ms)','0',NULL),(1511849,'Avg task runtime (ms)','0',NULL),(1511849,'Number of tasks','0',NULL),(1511849,'Task GC/CPU ratio','0.0',NULL),(1511850,'Average task runtime','0 sec',NULL),(1511850,'Max task runtime','0 sec',NULL),(1511850,'Min task runtime','0 sec',NULL),(1511850,'Number of tasks','0',NULL),(1511851,'Avg Physical Memory (MB)','0',NULL),(1511851,'Avg task runtime','0 sec',NULL),(1511851,'Avg Virtual Memory (MB)','0',NULL),(1511851,'Max Physical Memory (MB)','0',NULL),(1511851,'Min Physical Memory (MB)','0',NULL),(1511851,'Number of tasks','0',NULL),(1511851,'Requested Container Memory','2 GB',NULL),(1511852,'Average code runtime','0 sec',NULL),(1511852,'Average shuffle time','0 sec ',NULL),(1511852,'Average sort time','0 sec ',NULL),(1511852,'Number of tasks','0',NULL),(1511866,'Group A','105 tasks @ 302 MB avg',NULL),(1511866,'Group B','99 tasks @ 471 MB avg',NULL),(1511866,'Number of tasks','204',NULL),(1511867,'Avg task CPU time (ms)','69552',NULL),(1511867,'Avg task GC time (ms)','431',NULL),(1511867,'Avg task runtime (ms)','81565',NULL),(1511867,'Number of tasks','204',NULL),(1511867,'Task GC/CPU ratio','0.006196802392454567',NULL),(1511868,'Average task input size','384 MB',NULL),(1511868,'Average task runtime','1 min 21 sec',NULL),(1511868,'Max task runtime','2 min 41 sec',NULL),(1511868,'Min task runtime','41 sec',NULL),(1511868,'Number of tasks','204',NULL),(1511869,'Median task input size','377 MB',NULL),(1511869,'Median task runtime','1 min 19 sec',NULL),(1511869,'Median task speed','4 MB/s',NULL),(1511869,'Number of tasks','204',NULL),(1511869,'Total input size in MB','78404.13675880432',NULL),(1511870,'Avg output records per task','1411730',NULL),(1511870,'Avg spilled records per task','0',NULL),(1511870,'Number of tasks','204',NULL),(1511870,'Ratio of spilled records to output records','0.0',NULL),(1511871,'Avg Physical Memory (MB)','722',NULL),(1511871,'Avg task runtime','1 min 21 sec',NULL),(1511871,'Avg Virtual Memory (MB)','2358',NULL),(1511871,'Max Physical Memory (MB)','768',NULL),(1511871,'Min Physical Memory (MB)','669',NULL),(1511871,'Number of tasks','204',NULL),(1511871,'Requested Container Memory','2 GB',NULL),(1511872,'Group A','0 tasks @ 0 bytes avg',NULL),(1511872,'Group B','0 tasks @ 0 bytes avg',NULL),(1511872,'Number of tasks','0',NULL),(1511873,'Avg task CPU time (ms)','0',NULL),(1511873,'Avg task GC time (ms)','0',NULL),(1511873,'Avg task runtime (ms)','0',NULL),(1511873,'Number of tasks','0',NULL),(1511873,'Task GC/CPU ratio','0.0',NULL),(1511874,'Average task runtime','0 sec',NULL),(1511874,'Max task runtime','0 sec',NULL),(1511874,'Min task runtime','0 sec',NULL),(1511874,'Number of tasks','0',NULL),(1511875,'Avg Physical Memory (MB)','0',NULL),(1511875,'Avg task runtime','0 sec',NULL),(1511875,'Avg Virtual Memory (MB)','0',NULL),(1511875,'Max Physical Memory (MB)','0',NULL),(1511875,'Min Physical Memory (MB)','0',NULL),(1511875,'Number of tasks','0',NULL),(1511875,'Requested Container Memory','2 GB',NULL),(1511876,'Average code runtime','0 sec',NULL),(1511876,'Average shuffle time','0 sec ',NULL),(1511876,'Average sort time','0 sec ',NULL),(1511876,'Number of tasks','0',NULL),(1511878,'Group A','348 tasks @ 300 MB avg',NULL),(1511878,'Group B','255 tasks @ 438 MB avg',NULL),(1511878,'Number of tasks','603',NULL),(1511879,'Avg task CPU time (ms)','74298',NULL),(1511879,'Avg task GC time (ms)','762',NULL),(1511879,'Avg task runtime (ms)','84955',NULL),(1511879,'Number of tasks','603',NULL),(1511879,'Task GC/CPU ratio','0.010255996123718001',NULL),(1511880,'Average task input size','358 MB',NULL),(1511880,'Average task runtime','1 min 24 sec',NULL),(1511880,'Max task runtime','2 min 53 sec',NULL),(1511880,'Min task runtime','39 sec',NULL),(1511880,'Number of tasks','603',NULL),(1511881,'Median task input size','336 MB',NULL),(1511881,'Median task runtime','1 min 21 sec',NULL),(1511881,'Median task speed','4 MB/s',NULL),(1511881,'Number of tasks','603',NULL),(1511881,'Total input size in MB','216403.73294448853',NULL),(1511882,'Avg output records per task','327606',NULL),(1511882,'Avg spilled records per task','0',NULL),(1511882,'Number of tasks','603',NULL),(1511882,'Ratio of spilled records to output records','0.0',NULL),(1511883,'Avg Physical Memory (MB)','714',NULL),(1511883,'Avg task runtime','1 min 24 sec',NULL),(1511883,'Avg Virtual Memory (MB)','2352',NULL),(1511883,'Max Physical Memory (MB)','854',NULL),(1511883,'Min Physical Memory (MB)','650',NULL),(1511883,'Number of tasks','603',NULL),(1511883,'Requested Container Memory','2 GB',NULL),(1511884,'Group A','0 tasks @ 0 bytes avg',NULL),(1511884,'Group B','0 tasks @ 0 bytes avg',NULL),(1511884,'Number of tasks','0',NULL),(1511885,'Avg task CPU time (ms)','0',NULL),(1511885,'Avg task GC time (ms)','0',NULL),(1511885,'Avg task runtime (ms)','0',NULL),(1511885,'Number of tasks','0',NULL),(1511885,'Task GC/CPU ratio','0.0',NULL),(1511886,'Average task runtime','0 sec',NULL),(1511886,'Max task runtime','0 sec',NULL),(1511886,'Min task runtime','0 sec',NULL),(1511886,'Number of tasks','0',NULL),(1511887,'Avg Physical Memory (MB)','0',NULL),(1511887,'Avg task runtime','0 sec',NULL),(1511887,'Avg Virtual Memory (MB)','0',NULL),(1511887,'Max Physical Memory (MB)','0',NULL),(1511887,'Min Physical Memory (MB)','0',NULL),(1511887,'Number of tasks','0',NULL),(1511887,'Requested Container Memory','2 GB',NULL),(1511888,'Average code runtime','0 sec',NULL),(1511888,'Average shuffle time','0 sec ',NULL),(1511888,'Average sort time','0 sec ',NULL),(1511888,'Number of tasks','0',NULL),(1511902,'Group A','959 tasks @ 318 MB avg',NULL),(1511902,'Group B','986 tasks @ 467 MB avg',NULL),(1511902,'Number of tasks','1945',NULL),(1511903,'Avg task CPU time (ms)','76812',NULL),(1511903,'Avg task GC time (ms)','876',NULL),(1511903,'Avg task runtime (ms)','88276',NULL),(1511903,'Number of tasks','1945',NULL),(1511903,'Task GC/CPU ratio','0.011404468051866895',NULL),(1511904,'Average task input size','393 MB',NULL),(1511904,'Average task runtime','1 min 28 sec',NULL),(1511904,'Max task runtime','4 min 28 sec',NULL),(1511904,'Min task runtime','34 sec',NULL),(1511904,'Number of tasks','1945',NULL),(1511905,'Median task input size','396 MB',NULL),(1511905,'Median task runtime','1 min 24 sec',NULL),(1511905,'Median task speed','4 MB/s',NULL),(1511905,'Number of tasks','1945',NULL),(1511905,'Total input size in MB','766235.623585701',NULL),(1511906,'Avg output records per task','377268',NULL),(1511906,'Avg spilled records per task','0',NULL),(1511906,'Number of tasks','1945',NULL),(1511906,'Ratio of spilled records to output records','0.0',NULL),(1511907,'Avg Physical Memory (MB)','719',NULL),(1511907,'Avg task runtime','1 min 28 sec',NULL),(1511907,'Avg Virtual Memory (MB)','2354',NULL),(1511907,'Max Physical Memory (MB)','895',NULL),(1511907,'Min Physical Memory (MB)','675',NULL),(1511907,'Number of tasks','1945',NULL),(1511907,'Requested Container Memory','2 GB',NULL),(1511908,'Group A','0 tasks @ 0 bytes avg',NULL),(1511908,'Group B','0 tasks @ 0 bytes avg',NULL),(1511908,'Number of tasks','0',NULL),(1511909,'Avg task CPU time (ms)','0',NULL),(1511909,'Avg task GC time (ms)','0',NULL),(1511909,'Avg task runtime (ms)','0',NULL),(1511909,'Number of tasks','0',NULL),(1511909,'Task GC/CPU ratio','0.0',NULL),(1511910,'Average task runtime','0 sec',NULL),(1511910,'Max task runtime','0 sec',NULL),(1511910,'Min task runtime','0 sec',NULL),(1511910,'Number of tasks','0',NULL),(1511911,'Avg Physical Memory (MB)','0',NULL),(1511911,'Avg task runtime','0 sec',NULL),(1511911,'Avg Virtual Memory (MB)','0',NULL),(1511911,'Max Physical Memory (MB)','0',NULL),(1511911,'Min Physical Memory (MB)','0',NULL),(1511911,'Number of tasks','0',NULL),(1511911,'Requested Container Memory','2 GB',NULL),(1511912,'Average code runtime','0 sec',NULL),(1511912,'Average shuffle time','0 sec ',NULL),(1511912,'Average sort time','0 sec ',NULL),(1511912,'Number of tasks','0',NULL),(1519428,'Group A','1 tasks @ 297 MB avg',NULL),(1519428,'Group B','1 tasks @ 497 MB avg',NULL),(1519428,'Number of tasks','2',NULL),(1519429,'Avg task CPU time (ms)','160770',NULL),(1519429,'Avg task GC time (ms)','4433',NULL),(1519429,'Avg task runtime (ms)','90300',NULL),(1519429,'Number of tasks','2',NULL),(1519429,'Task GC/CPU ratio','0.027573552279654166',NULL),(1519430,'Average task input size','397 MB',NULL),(1519430,'Average task runtime','1 min 30 sec',NULL),(1519430,'Max task runtime','1 min 47 sec',NULL),(1519430,'Min task runtime','1 min 13 sec',NULL),(1519430,'Number of tasks','2',NULL),(1519431,'Median task input size','397 MB',NULL),(1519431,'Median task runtime','1 min 30 sec',NULL),(1519431,'Median task speed','4 MB/s',NULL),(1519431,'Number of tasks','2',NULL),(1519431,'Total input size in MB','794.1982917785645',NULL),(1519432,'Avg output records per task','1244514',NULL),(1519432,'Avg spilled records per task','0',NULL),(1519432,'Number of tasks','2',NULL),(1519432,'Ratio of spilled records to output records','0.0',NULL),(1519433,'Avg Physical Memory (MB)','766',NULL),(1519433,'Avg task runtime','1 min 30 sec',NULL),(1519433,'Avg Virtual Memory (MB)','2966',NULL),(1519433,'Max Physical Memory (MB)','826',NULL),(1519433,'Min Physical Memory (MB)','707',NULL),(1519433,'Number of tasks','2',NULL),(1519433,'Requested Container Memory','1 GB',NULL),(1519434,'Group A','0 tasks @ 0 bytes avg',NULL),(1519434,'Group B','0 tasks @ 0 bytes avg',NULL),(1519434,'Number of tasks','0',NULL),(1519435,'Avg task CPU time (ms)','0',NULL),(1519435,'Avg task GC time (ms)','0',NULL),(1519435,'Avg task runtime (ms)','0',NULL),(1519435,'Number of tasks','0',NULL),(1519435,'Task GC/CPU ratio','0.0',NULL),(1519436,'Average task runtime','0 sec',NULL),(1519436,'Max task runtime','0 sec',NULL),(1519436,'Min task runtime','0 sec',NULL),(1519436,'Number of tasks','0',NULL),(1519437,'Avg Physical Memory (MB)','0',NULL),(1519437,'Avg task runtime','0 sec',NULL),(1519437,'Avg Virtual Memory (MB)','0',NULL),(1519437,'Max Physical Memory (MB)','0',NULL),(1519437,'Min Physical Memory (MB)','0',NULL),(1519437,'Number of tasks','0',NULL),(1519437,'Requested Container Memory','1 GB',NULL),(1519438,'Average code runtime','0 sec',NULL),(1519438,'Average shuffle time','0 sec ',NULL),(1519438,'Average sort time','0 sec ',NULL),(1519438,'Number of tasks','0',NULL),(1519440,'Group A','1 tasks @ 304 MB avg',NULL),(1519440,'Group B','1 tasks @ 512 MB avg',NULL),(1519440,'Number of tasks','2',NULL),(1519441,'Avg task CPU time (ms)','117545',NULL),(1519441,'Avg task GC time (ms)','2388',NULL),(1519441,'Avg task runtime (ms)','79402',NULL),(1519441,'Number of tasks','2',NULL),(1519441,'Task GC/CPU ratio','0.020315623803649666',NULL),(1519442,'Average task input size','408 MB',NULL),(1519442,'Average task runtime','1 min 19 sec',NULL),(1519442,'Max task runtime','1 min 37 sec',NULL),(1519442,'Min task runtime','1 min 1 sec',NULL),(1519442,'Number of tasks','2',NULL),(1519443,'Median task input size','408 MB',NULL),(1519443,'Median task runtime','1 min 19 sec',NULL),(1519443,'Median task speed','5 MB/s',NULL),(1519443,'Number of tasks','2',NULL),(1519443,'Total input size in MB','817.0285835266113',NULL),(1519444,'Avg output records per task','1193587',NULL),(1519444,'Avg spilled records per task','0',NULL),(1519444,'Number of tasks','2',NULL),(1519444,'Ratio of spilled records to output records','0.0',NULL),(1519445,'Avg Physical Memory (MB)','742',NULL),(1519445,'Avg task runtime','1 min 19 sec',NULL),(1519445,'Avg Virtual Memory (MB)','2921',NULL),(1519445,'Max Physical Memory (MB)','807',NULL),(1519445,'Min Physical Memory (MB)','676',NULL),(1519445,'Number of tasks','2',NULL),(1519445,'Requested Container Memory','1 GB',NULL),(1519446,'Group A','0 tasks @ 0 bytes avg',NULL),(1519446,'Group B','0 tasks @ 0 bytes avg',NULL),(1519446,'Number of tasks','0',NULL),(1519447,'Avg task CPU time (ms)','0',NULL),(1519447,'Avg task GC time (ms)','0',NULL),(1519447,'Avg task runtime (ms)','0',NULL),(1519447,'Number of tasks','0',NULL),(1519447,'Task GC/CPU ratio','0.0',NULL),(1519448,'Average task runtime','0 sec',NULL),(1519448,'Max task runtime','0 sec',NULL),(1519448,'Min task runtime','0 sec',NULL),(1519448,'Number of tasks','0',NULL),(1519449,'Avg Physical Memory (MB)','0',NULL),(1519449,'Avg task runtime','0 sec',NULL),(1519449,'Avg Virtual Memory (MB)','0',NULL),(1519449,'Max Physical Memory (MB)','0',NULL),(1519449,'Min Physical Memory (MB)','0',NULL),(1519449,'Number of tasks','0',NULL),(1519449,'Requested Container Memory','1 GB',NULL),(1519450,'Average code runtime','0 sec',NULL),(1519450,'Average shuffle time','0 sec ',NULL),(1519450,'Average sort time','0 sec ',NULL),(1519450,'Number of tasks','0',NULL),(1519452,'Group A','1 tasks @ 271 MB avg',NULL),(1519452,'Group B','1 tasks @ 510 MB avg',NULL),(1519452,'Number of tasks','2',NULL),(1519453,'Avg task CPU time (ms)','56870',NULL),(1519453,'Avg task GC time (ms)','442',NULL),(1519453,'Avg task runtime (ms)','52394',NULL),(1519453,'Number of tasks','2',NULL),(1519453,'Task GC/CPU ratio','0.007772111834007386',NULL),(1519454,'Average task input size','391 MB',NULL),(1519454,'Average task runtime','52 sec',NULL),(1519454,'Max task runtime','1 min 5 sec',NULL),(1519454,'Min task runtime','39 sec',NULL),(1519454,'Number of tasks','2',NULL),(1519455,'Median task input size','391 MB',NULL),(1519455,'Median task runtime','52 sec',NULL),(1519455,'Median task speed','7 MB/s',NULL),(1519455,'Number of tasks','2',NULL),(1519455,'Total input size in MB','782.293210029602',NULL),(1519456,'Avg output records per task','1244941',NULL),(1519456,'Avg spilled records per task','0',NULL),(1519456,'Number of tasks','2',NULL),(1519456,'Ratio of spilled records to output records','0.0',NULL),(1519457,'Avg Physical Memory (MB)','623',NULL),(1519457,'Avg task runtime','52 sec',NULL),(1519457,'Avg Virtual Memory (MB)','2923',NULL),(1519457,'Max Physical Memory (MB)','624',NULL),(1519457,'Min Physical Memory (MB)','622',NULL),(1519457,'Number of tasks','2',NULL),(1519457,'Requested Container Memory','1 GB',NULL),(1519458,'Group A','0 tasks @ 0 bytes avg',NULL),(1519458,'Group B','0 tasks @ 0 bytes avg',NULL),(1519458,'Number of tasks','0',NULL),(1519459,'Avg task CPU time (ms)','0',NULL),(1519459,'Avg task GC time (ms)','0',NULL),(1519459,'Avg task runtime (ms)','0',NULL),(1519459,'Number of tasks','0',NULL),(1519459,'Task GC/CPU ratio','0.0',NULL),(1519460,'Average task runtime','0 sec',NULL),(1519460,'Max task runtime','0 sec',NULL),(1519460,'Min task runtime','0 sec',NULL),(1519460,'Number of tasks','0',NULL),(1519461,'Avg Physical Memory (MB)','0',NULL),(1519461,'Avg task runtime','0 sec',NULL),(1519461,'Avg Virtual Memory (MB)','0',NULL),(1519461,'Max Physical Memory (MB)','0',NULL),(1519461,'Min Physical Memory (MB)','0',NULL),(1519461,'Number of tasks','0',NULL),(1519461,'Requested Container Memory','1 GB',NULL),(1519462,'Average code runtime','0 sec',NULL),(1519462,'Average shuffle time','0 sec ',NULL),(1519462,'Average sort time','0 sec ',NULL),(1519462,'Number of tasks','0',NULL),(1519464,'Group A','49 tasks @ 305 MB avg',NULL),(1519464,'Group B','100 tasks @ 488 MB avg',NULL),(1519464,'Number of tasks','149',NULL),(1519465,'Avg task CPU time (ms)','65602',NULL),(1519465,'Avg task GC time (ms)','717',NULL),(1519465,'Avg task runtime (ms)','56899',NULL),(1519465,'Number of tasks','149',NULL),(1519465,'Task GC/CPU ratio','0.010929544830950277',NULL),(1519466,'Average task input size','427 MB',NULL),(1519466,'Average task runtime','56 sec',NULL),(1519466,'Max task runtime','1 min 43 sec',NULL),(1519466,'Min task runtime','28 sec',NULL),(1519466,'Number of tasks','149',NULL),(1519467,'Median task input size','474 MB',NULL),(1519467,'Median task runtime','56 sec',NULL),(1519467,'Median task speed','7 MB/s',NULL),(1519467,'Number of tasks','149',NULL),(1519467,'Total input size in MB','63770.37501811981',NULL),(1519468,'Avg output records per task','1560839',NULL),(1519468,'Avg spilled records per task','0',NULL),(1519468,'Number of tasks','149',NULL),(1519468,'Ratio of spilled records to output records','0.0',NULL),(1519469,'Avg Physical Memory (MB)','612',NULL),(1519469,'Avg task runtime','56 sec',NULL),(1519469,'Avg Virtual Memory (MB)','2928',NULL),(1519469,'Max Physical Memory (MB)','645',NULL),(1519469,'Min Physical Memory (MB)','572',NULL),(1519469,'Number of tasks','149',NULL),(1519469,'Requested Container Memory','1 GB',NULL),(1519470,'Group A','0 tasks @ 0 bytes avg',NULL),(1519470,'Group B','0 tasks @ 0 bytes avg',NULL),(1519470,'Number of tasks','0',NULL),(1519471,'Avg task CPU time (ms)','0',NULL),(1519471,'Avg task GC time (ms)','0',NULL),(1519471,'Avg task runtime (ms)','0',NULL),(1519471,'Number of tasks','0',NULL),(1519471,'Task GC/CPU ratio','0.0',NULL),(1519472,'Average task runtime','0 sec',NULL),(1519472,'Max task runtime','0 sec',NULL),(1519472,'Min task runtime','0 sec',NULL),(1519472,'Number of tasks','0',NULL),(1519473,'Avg Physical Memory (MB)','0',NULL),(1519473,'Avg task runtime','0 sec',NULL),(1519473,'Avg Virtual Memory (MB)','0',NULL),(1519473,'Max Physical Memory (MB)','0',NULL),(1519473,'Min Physical Memory (MB)','0',NULL),(1519473,'Number of tasks','0',NULL),(1519473,'Requested Container Memory','1 GB',NULL),(1519474,'Average code runtime','0 sec',NULL),(1519474,'Average shuffle time','0 sec ',NULL),(1519474,'Average sort time','0 sec ',NULL),(1519474,'Number of tasks','0',NULL),(1519476,'Group A','305 tasks @ 295 MB avg',NULL),(1519476,'Group B','193 tasks @ 453 MB avg',NULL),(1519476,'Number of tasks','498',NULL),(1519477,'Avg task CPU time (ms)','67224',NULL),(1519477,'Avg task GC time (ms)','978',NULL),(1519477,'Avg task runtime (ms)','52679',NULL),(1519477,'Number of tasks','498',NULL),(1519477,'Task GC/CPU ratio','0.014548375580149946',NULL),(1519478,'Average task input size','357 MB',NULL),(1519478,'Average task runtime','52 sec',NULL),(1519478,'Max task runtime','1 min 54 sec',NULL),(1519478,'Min task runtime','32 sec',NULL),(1519478,'Number of tasks','498',NULL),(1519479,'Median task input size','321 MB',NULL),(1519479,'Median task runtime','49 sec',NULL),(1519479,'Median task speed','7 MB/s',NULL),(1519479,'Number of tasks','498',NULL),(1519479,'Total input size in MB','177790.3564529419',NULL),(1519480,'Avg output records per task','306035',NULL),(1519480,'Avg spilled records per task','0',NULL),(1519480,'Number of tasks','498',NULL),(1519480,'Ratio of spilled records to output records','0.0',NULL),(1519481,'Avg Physical Memory (MB)','604',NULL),(1519481,'Avg task runtime','52 sec',NULL),(1519481,'Avg Virtual Memory (MB)','2927',NULL),(1519481,'Max Physical Memory (MB)','728',NULL),(1519481,'Min Physical Memory (MB)','565',NULL),(1519481,'Number of tasks','498',NULL),(1519481,'Requested Container Memory','1 GB',NULL),(1519482,'Group A','0 tasks @ 0 bytes avg',NULL),(1519482,'Group B','0 tasks @ 0 bytes avg',NULL),(1519482,'Number of tasks','0',NULL),(1519483,'Avg task CPU time (ms)','0',NULL),(1519483,'Avg task GC time (ms)','0',NULL),(1519483,'Avg task runtime (ms)','0',NULL),(1519483,'Number of tasks','0',NULL),(1519483,'Task GC/CPU ratio','0.0',NULL),(1519484,'Average task runtime','0 sec',NULL),(1519484,'Max task runtime','0 sec',NULL),(1519484,'Min task runtime','0 sec',NULL),(1519484,'Number of tasks','0',NULL),(1519485,'Avg Physical Memory (MB)','0',NULL),(1519485,'Avg task runtime','0 sec',NULL),(1519485,'Avg Virtual Memory (MB)','0',NULL),(1519485,'Max Physical Memory (MB)','0',NULL),(1519485,'Min Physical Memory (MB)','0',NULL),(1519485,'Number of tasks','0',NULL),(1519485,'Requested Container Memory','1 GB',NULL),(1519486,'Average code runtime','0 sec',NULL),(1519486,'Average shuffle time','0 sec ',NULL),(1519486,'Average sort time','0 sec ',NULL),(1519486,'Number of tasks','0',NULL),(1519488,'Group A','297 tasks @ 291 MB avg',NULL),(1519488,'Group B','200 tasks @ 458 MB avg',NULL),(1519488,'Number of tasks','497',NULL),(1519489,'Avg task CPU time (ms)','67218',NULL),(1519489,'Avg task GC time (ms)','994',NULL),(1519489,'Avg task runtime (ms)','54348',NULL),(1519489,'Number of tasks','497',NULL),(1519489,'Task GC/CPU ratio','0.014787705674075396',NULL),(1519490,'Average task input size','358 MB',NULL),(1519490,'Average task runtime','54 sec',NULL),(1519490,'Max task runtime','1 min 59 sec',NULL),(1519490,'Min task runtime','31 sec',NULL),(1519490,'Number of tasks','497',NULL),(1519491,'Median task input size','320 MB',NULL),(1519491,'Median task runtime','51 sec',NULL),(1519491,'Median task speed','6 MB/s',NULL),(1519491,'Number of tasks','497',NULL),(1519491,'Total input size in MB','178118.81388187408',NULL),(1519492,'Avg output records per task','309389',NULL),(1519492,'Avg spilled records per task','0',NULL),(1519492,'Number of tasks','497',NULL),(1519492,'Ratio of spilled records to output records','0.0',NULL),(1519493,'Avg Physical Memory (MB)','602',NULL),(1519493,'Avg task runtime','54 sec',NULL),(1519493,'Avg Virtual Memory (MB)','2924',NULL),(1519493,'Max Physical Memory (MB)','716',NULL),(1519493,'Min Physical Memory (MB)','568',NULL),(1519493,'Number of tasks','497',NULL),(1519493,'Requested Container Memory','1 GB',NULL),(1519494,'Group A','0 tasks @ 0 bytes avg',NULL),(1519494,'Group B','0 tasks @ 0 bytes avg',NULL),(1519494,'Number of tasks','0',NULL),(1519495,'Avg task CPU time (ms)','0',NULL),(1519495,'Avg task GC time (ms)','0',NULL),(1519495,'Avg task runtime (ms)','0',NULL),(1519495,'Number of tasks','0',NULL),(1519495,'Task GC/CPU ratio','0.0',NULL),(1519496,'Average task runtime','0 sec',NULL),(1519496,'Max task runtime','0 sec',NULL),(1519496,'Min task runtime','0 sec',NULL),(1519496,'Number of tasks','0',NULL),(1519497,'Avg Physical Memory (MB)','0',NULL),(1519497,'Avg task runtime','0 sec',NULL),(1519497,'Avg Virtual Memory (MB)','0',NULL),(1519497,'Max Physical Memory (MB)','0',NULL),(1519497,'Min Physical Memory (MB)','0',NULL),(1519497,'Number of tasks','0',NULL),(1519497,'Requested Container Memory','1 GB',NULL),(1519498,'Average code runtime','0 sec',NULL),(1519498,'Average shuffle time','0 sec ',NULL),(1519498,'Average sort time','0 sec ',NULL),(1519498,'Number of tasks','0',NULL),(1519512,'Group A','896 tasks @ 310 MB avg',NULL),(1519512,'Group B','734 tasks @ 448 MB avg',NULL),(1519512,'Number of tasks','1630',NULL),(1519513,'Avg task CPU time (ms)','72118',NULL),(1519513,'Avg task GC time (ms)','1457',NULL),(1519513,'Avg task runtime (ms)','59668',NULL),(1519513,'Number of tasks','1630',NULL),(1519513,'Task GC/CPU ratio','0.020203000637843534',NULL),(1519514,'Average task input size','372 MB',NULL),(1519514,'Average task runtime','59 sec',NULL),(1519514,'Max task runtime','2 min 8 sec',NULL),(1519514,'Min task runtime','30 sec',NULL),(1519514,'Number of tasks','1630',NULL),(1519515,'Median task input size','359 MB',NULL),(1519515,'Median task runtime','58 sec',NULL),(1519515,'Median task speed','6 MB/s',NULL),(1519515,'Number of tasks','1630',NULL),(1519515,'Total input size in MB','607747.2612476349',NULL),(1519516,'Avg output records per task','336482',NULL),(1519516,'Avg spilled records per task','0',NULL),(1519516,'Number of tasks','1630',NULL),(1519516,'Ratio of spilled records to output records','0.0',NULL),(1519517,'Avg Physical Memory (MB)','605',NULL),(1519517,'Avg task runtime','59 sec',NULL),(1519517,'Avg Virtual Memory (MB)','2927',NULL),(1519517,'Max Physical Memory (MB)','692',NULL),(1519517,'Min Physical Memory (MB)','562',NULL),(1519517,'Number of tasks','1630',NULL),(1519517,'Requested Container Memory','1 GB',NULL),(1519518,'Group A','0 tasks @ 0 bytes avg',NULL),(1519518,'Group B','0 tasks @ 0 bytes avg',NULL),(1519518,'Number of tasks','0',NULL),(1519519,'Avg task CPU time (ms)','0',NULL),(1519519,'Avg task GC time (ms)','0',NULL),(1519519,'Avg task runtime (ms)','0',NULL),(1519519,'Number of tasks','0',NULL),(1519519,'Task GC/CPU ratio','0.0',NULL),(1519520,'Average task runtime','0 sec',NULL),(1519520,'Max task runtime','0 sec',NULL),(1519520,'Min task runtime','0 sec',NULL),(1519520,'Number of tasks','0',NULL),(1519521,'Avg Physical Memory (MB)','0',NULL),(1519521,'Avg task runtime','0 sec',NULL),(1519521,'Avg Virtual Memory (MB)','0',NULL),(1519521,'Max Physical Memory (MB)','0',NULL),(1519521,'Min Physical Memory (MB)','0',NULL),(1519521,'Number of tasks','0',NULL),(1519521,'Requested Container Memory','1 GB',NULL),(1519522,'Average code runtime','0 sec',NULL),(1519522,'Average shuffle time','0 sec ',NULL),(1519522,'Average sort time','0 sec ',NULL),(1519522,'Number of tasks','0',NULL),(1519524,'Group A','1 tasks @ 381 MB avg',NULL),(1519524,'Group B','500 tasks @ 1 GB avg',NULL),(1519524,'Number of tasks','501',NULL),(1519525,'Avg task CPU time (ms)','126005',NULL),(1519525,'Avg task GC time (ms)','3854',NULL),(1519525,'Avg task runtime (ms)','105024',NULL),(1519525,'Number of tasks','501',NULL),(1519525,'Task GC/CPU ratio','0.0305860878536566',NULL),(1519526,'Average task input size','1 GB',NULL),(1519526,'Average task runtime','1 min 45 sec',NULL),(1519526,'Max task runtime','3 min 19 sec',NULL),(1519526,'Min task runtime','13 sec',NULL),(1519526,'Number of tasks','501',NULL),(1519527,'Median task input size','953 MB',NULL),(1519527,'Median task runtime','1 min 40 sec',NULL),(1519527,'Median task speed','10 MB/s',NULL),(1519527,'Number of tasks','501',NULL),(1519527,'Total input size in MB','528976.7128763199',NULL),(1519528,'Avg output records per task','757787',NULL),(1519528,'Avg spilled records per task','245456',NULL),(1519528,'Number of tasks','501',NULL),(1519528,'Ratio of spilled records to output records','0.3239116237443114',NULL),(1519529,'Avg Physical Memory (MB)','982',NULL),(1519529,'Avg task runtime','1 min 45 sec',NULL),(1519529,'Avg Virtual Memory (MB)','2923',NULL),(1519529,'Max Physical Memory (MB)','1096',NULL),(1519529,'Min Physical Memory (MB)','937',NULL),(1519529,'Number of tasks','501',NULL),(1519529,'Requested Container Memory','1 GB',NULL),(1519530,'Group A','280 tasks @ 7 MB avg',NULL),(1519530,'Group B','275 tasks @ 7 MB avg',NULL),(1519530,'Number of tasks','555',NULL),(1519531,'Avg task CPU time (ms)','8870',NULL),(1519531,'Avg task GC time (ms)','84',NULL),(1519531,'Avg task runtime (ms)','7490',NULL),(1519531,'Number of tasks','555',NULL),(1519531,'Task GC/CPU ratio','0.00947012401352875',NULL),(1519532,'Average task runtime','7 sec',NULL),(1519532,'Max task runtime','21 sec',NULL),(1519532,'Min task runtime','3 sec',NULL),(1519532,'Number of tasks','555',NULL),(1519533,'Avg Physical Memory (MB)','566',NULL),(1519533,'Avg task runtime','7 sec',NULL),(1519533,'Avg Virtual Memory (MB)','2944',NULL),(1519533,'Max Physical Memory (MB)','628',NULL),(1519533,'Min Physical Memory (MB)','509',NULL),(1519533,'Number of tasks','555',NULL),(1519533,'Requested Container Memory','1 GB',NULL),(1519534,'Average code runtime','',NULL),(1519534,'Average shuffle time','6 sec (27.00x)',NULL),(1519534,'Average sort time',' (1.25x)',NULL),(1519534,'Number of tasks','555',NULL),(1519536,'Group A','1 tasks @ 150 MB avg',NULL),(1519536,'Group B','1 tasks @ 154 MB avg',NULL),(1519536,'Number of tasks','2',NULL),(1519537,'Avg task CPU time (ms)','23865',NULL),(1519537,'Avg task GC time (ms)','307',NULL),(1519537,'Avg task runtime (ms)','18223',NULL),(1519537,'Number of tasks','2',NULL),(1519537,'Task GC/CPU ratio','0.01286402681751519',NULL),(1519538,'Average task input size','152 MB',NULL),(1519538,'Average task runtime','18 sec',NULL),(1519538,'Max task runtime','19 sec',NULL),(1519538,'Min task runtime','17 sec',NULL),(1519538,'Number of tasks','2',NULL),(1519539,'Median task input size','152 MB',NULL),(1519539,'Median task runtime','18 sec',NULL),(1519539,'Median task speed','8 MB/s',NULL),(1519539,'Number of tasks','2',NULL),(1519539,'Total input size in MB','305.83719635009766',NULL),(1519540,'Avg output records per task','2898219',NULL),(1519540,'Avg spilled records per task','5796438',NULL),(1519540,'Number of tasks','2',NULL),(1519540,'Ratio of spilled records to output records','2.0',NULL),(1519541,'Avg Physical Memory (MB)','920',NULL),(1519541,'Avg task runtime','18 sec',NULL),(1519541,'Avg Virtual Memory (MB)','2921',NULL),(1519541,'Max Physical Memory (MB)','921',NULL),(1519541,'Min Physical Memory (MB)','920',NULL),(1519541,'Number of tasks','2',NULL),(1519541,'Requested Container Memory','1 GB',NULL),(1519542,'Group A','0 tasks @ 0 bytes avg',NULL),(1519542,'Group B','1 tasks @ 187 MB avg',NULL),(1519542,'Number of tasks','1',NULL),(1519543,'Avg task CPU time (ms)','32280',NULL),(1519543,'Avg task GC time (ms)','299',NULL),(1519543,'Avg task runtime (ms)','26825',NULL),(1519543,'Number of tasks','1',NULL),(1519543,'Task GC/CPU ratio','0.00926270136307311',NULL),(1519544,'Average task runtime','26 sec',NULL),(1519544,'Max task runtime','26 sec',NULL),(1519544,'Min task runtime','26 sec',NULL),(1519544,'Number of tasks','1',NULL),(1519545,'Avg Physical Memory (MB)','761',NULL),(1519545,'Avg task runtime','26 sec',NULL),(1519545,'Avg Virtual Memory (MB)','2922',NULL),(1519545,'Max Physical Memory (MB)','761',NULL),(1519545,'Min Physical Memory (MB)','761',NULL),(1519545,'Number of tasks','1',NULL),(1519545,'Requested Container Memory','1 GB',NULL),(1519546,'Average code runtime','17 sec',NULL),(1519546,'Average shuffle time','5 sec (0.31x)',NULL),(1519546,'Average sort time','3 sec (0.19x)',NULL),(1519546,'Number of tasks','1',NULL),(1519548,'Group A','0 tasks @ 0 bytes avg',NULL),(1519548,'Group B','1 tasks @ 162 MB avg',NULL),(1519548,'Number of tasks','1',NULL),(1519549,'Avg task CPU time (ms)','42220',NULL),(1519549,'Avg task GC time (ms)','574',NULL),(1519549,'Avg task runtime (ms)','29318',NULL),(1519549,'Number of tasks','1',NULL),(1519549,'Task GC/CPU ratio','0.01359545239223117',NULL),(1519550,'Average task input size','162 MB',NULL),(1519550,'Average task runtime','29 sec',NULL),(1519550,'Max task runtime','29 sec',NULL),(1519550,'Min task runtime','29 sec',NULL),(1519550,'Number of tasks','1',NULL),(1519551,'Median task input size','162 MB',NULL),(1519551,'Median task runtime','29 sec',NULL),(1519551,'Median task speed','5 MB/s',NULL),(1519551,'Number of tasks','1',NULL),(1519551,'Total input size in MB','162.62904262542725',NULL),(1519552,'Avg output records per task','2909795',NULL),(1519552,'Avg spilled records per task','5819590',NULL),(1519552,'Number of tasks','1',NULL),(1519552,'Ratio of spilled records to output records','2.0',NULL),(1519553,'Avg Physical Memory (MB)','950',NULL),(1519553,'Avg task runtime','29 sec',NULL),(1519553,'Avg Virtual Memory (MB)','2910',NULL),(1519553,'Max Physical Memory (MB)','950',NULL),(1519553,'Min Physical Memory (MB)','950',NULL),(1519553,'Number of tasks','1',NULL),(1519553,'Requested Container Memory','1 GB',NULL),(1519554,'Group A','0 tasks @ 0 bytes avg',NULL),(1519554,'Group B','1 tasks @ 96 MB avg',NULL),(1519554,'Number of tasks','1',NULL),(1519555,'Avg task CPU time (ms)','27850',NULL),(1519555,'Avg task GC time (ms)','544',NULL),(1519555,'Avg task runtime (ms)','23846',NULL),(1519555,'Number of tasks','1',NULL),(1519555,'Task GC/CPU ratio','0.019533213644524237',NULL),(1519556,'Average task runtime','23 sec',NULL),(1519556,'Max task runtime','23 sec',NULL),(1519556,'Min task runtime','23 sec',NULL),(1519556,'Number of tasks','1',NULL),(1519557,'Avg Physical Memory (MB)','577',NULL),(1519557,'Avg task runtime','23 sec',NULL),(1519557,'Avg Virtual Memory (MB)','2930',NULL),(1519557,'Max Physical Memory (MB)','577',NULL),(1519557,'Min Physical Memory (MB)','577',NULL),(1519557,'Number of tasks','1',NULL),(1519557,'Requested Container Memory','1 GB',NULL),(1519558,'Average code runtime','15 sec',NULL),(1519558,'Average shuffle time','5 sec (0.37x)',NULL),(1519558,'Average sort time','2 sec (0.13x)',NULL),(1519558,'Number of tasks','1',NULL),(1519560,'Group A','0 tasks @ 0 bytes avg',NULL),(1519560,'Group B','1 tasks @ 157 MB avg',NULL),(1519560,'Number of tasks','1',NULL),(1519561,'Avg task CPU time (ms)','20500',NULL),(1519561,'Avg task GC time (ms)','309',NULL),(1519561,'Avg task runtime (ms)','16962',NULL),(1519561,'Number of tasks','1',NULL),(1519561,'Task GC/CPU ratio','0.015073170731707317',NULL),(1519562,'Average task input size','157 MB',NULL),(1519562,'Average task runtime','16 sec',NULL),(1519562,'Max task runtime','16 sec',NULL),(1519562,'Min task runtime','16 sec',NULL),(1519562,'Number of tasks','1',NULL),(1519563,'Median task input size','157 MB',NULL),(1519563,'Median task runtime','16 sec',NULL),(1519563,'Median task speed','9 MB/s',NULL),(1519563,'Number of tasks','1',NULL),(1519563,'Total input size in MB','157.62358570098877',NULL),(1519564,'Avg output records per task','2909795',NULL),(1519564,'Avg spilled records per task','5819590',NULL),(1519564,'Number of tasks','1',NULL),(1519564,'Ratio of spilled records to output records','2.0',NULL),(1519565,'Avg Physical Memory (MB)','901',NULL),(1519565,'Avg task runtime','16 sec',NULL),(1519565,'Avg Virtual Memory (MB)','2920',NULL),(1519565,'Max Physical Memory (MB)','901',NULL),(1519565,'Min Physical Memory (MB)','901',NULL),(1519565,'Number of tasks','1',NULL),(1519565,'Requested Container Memory','1 GB',NULL),(1519566,'Group A','0 tasks @ 0 bytes avg',NULL),(1519566,'Group B','1 tasks @ 93 MB avg',NULL),(1519566,'Number of tasks','1',NULL),(1519567,'Avg task CPU time (ms)','14500',NULL),(1519567,'Avg task GC time (ms)','115',NULL),(1519567,'Avg task runtime (ms)','14486',NULL),(1519567,'Number of tasks','1',NULL),(1519567,'Task GC/CPU ratio','0.007931034482758621',NULL),(1519568,'Average task runtime','14 sec',NULL),(1519568,'Max task runtime','14 sec',NULL),(1519568,'Min task runtime','14 sec',NULL),(1519568,'Number of tasks','1',NULL),(1519569,'Avg Physical Memory (MB)','551',NULL),(1519569,'Avg task runtime','14 sec',NULL),(1519569,'Avg Virtual Memory (MB)','2921',NULL),(1519569,'Max Physical Memory (MB)','551',NULL),(1519569,'Min Physical Memory (MB)','551',NULL),(1519569,'Number of tasks','1',NULL),(1519569,'Requested Container Memory','1 GB',NULL),(1519570,'Average code runtime','7 sec',NULL),(1519570,'Average shuffle time','5 sec (0.70x)',NULL),(1519570,'Average sort time','1 sec (0.23x)',NULL),(1519570,'Number of tasks','1',NULL),(1519572,'Group A','2 tasks @ 176 MB avg',NULL),(1519572,'Group B','31 tasks @ 511 MB avg',NULL),(1519572,'Number of tasks','33',NULL),(1519573,'Avg task CPU time (ms)','45107',NULL),(1519573,'Avg task GC time (ms)','574',NULL),(1519573,'Avg task runtime (ms)','32716',NULL),(1519573,'Number of tasks','33',NULL),(1519573,'Task GC/CPU ratio','0.012725297625645686',NULL),(1519574,'Average task input size','491 MB',NULL),(1519574,'Average task runtime','32 sec',NULL),(1519574,'Max task runtime','46 sec',NULL),(1519574,'Min task runtime','16 sec',NULL),(1519574,'Number of tasks','33',NULL),(1519575,'Median task input size','509 MB',NULL),(1519575,'Median task runtime','32 sec',NULL),(1519575,'Median task speed','15 MB/s',NULL),(1519575,'Number of tasks','33',NULL),(1519575,'Total input size in MB','16218.928217887878',NULL),(1519576,'Avg output records per task','5394385',NULL),(1519576,'Avg spilled records per task','10788771',NULL),(1519576,'Number of tasks','33',NULL),(1519576,'Ratio of spilled records to output records','2.0',NULL),(1519577,'Avg Physical Memory (MB)','930',NULL),(1519577,'Avg task runtime','32 sec',NULL),(1519577,'Avg Virtual Memory (MB)','2920',NULL),(1519577,'Max Physical Memory (MB)','947',NULL),(1519577,'Min Physical Memory (MB)','897',NULL),(1519577,'Number of tasks','33',NULL),(1519577,'Requested Container Memory','1 GB',NULL),(1519578,'Group A','17 tasks @ 127 MB avg',NULL),(1519578,'Group B','1 tasks @ 992 MB avg',NULL),(1519578,'Number of tasks','18',NULL),(1519579,'Avg task CPU time (ms)','39060',NULL),(1519579,'Avg task GC time (ms)','782',NULL),(1519579,'Avg task runtime (ms)','29503',NULL),(1519579,'Number of tasks','18',NULL),(1519579,'Task GC/CPU ratio','0.020020481310803893',NULL),(1519580,'Average task runtime','29 sec',NULL),(1519580,'Max task runtime','1 min 34 sec',NULL),(1519580,'Min task runtime','21 sec',NULL),(1519580,'Number of tasks','18',NULL),(1519581,'Avg Physical Memory (MB)','1288',NULL),(1519581,'Avg task runtime','29 sec',NULL),(1519581,'Avg Virtual Memory (MB)','2929',NULL),(1519581,'Max Physical Memory (MB)','1379',NULL),(1519581,'Min Physical Memory (MB)','1158',NULL),(1519581,'Number of tasks','18',NULL),(1519581,'Requested Container Memory','1 GB',NULL),(1519582,'Average code runtime','17 sec',NULL),(1519582,'Average shuffle time','10 sec (0.59x)',NULL),(1519582,'Average sort time','1 sec (0.08x)',NULL),(1519582,'Number of tasks','18',NULL),(1519584,'Group A','18 tasks @ 444 MB avg',NULL),(1519584,'Group B','9 tasks @ 478 MB avg',NULL),(1519584,'Number of tasks','27',NULL),(1519585,'Avg task CPU time (ms)','37574',NULL),(1519585,'Avg task GC time (ms)','857',NULL),(1519585,'Avg task runtime (ms)','28244',NULL),(1519585,'Number of tasks','27',NULL),(1519585,'Task GC/CPU ratio','0.022808324905519775',NULL),(1519586,'Average task input size','455 MB',NULL),(1519586,'Average task runtime','28 sec',NULL),(1519586,'Max task runtime','51 sec',NULL),(1519586,'Min task runtime','19 sec',NULL),(1519586,'Number of tasks','27',NULL),(1519587,'Median task input size','451 MB',NULL),(1519587,'Median task runtime','25 sec',NULL),(1519587,'Median task speed','17 MB/s',NULL),(1519587,'Number of tasks','27',NULL),(1519587,'Total input size in MB','12298.221353530884',NULL),(1519588,'Avg output records per task','4316983',NULL),(1519588,'Avg spilled records per task','2133848',NULL),(1519588,'Number of tasks','27',NULL),(1519588,'Ratio of spilled records to output records','0.4942915898368947',NULL),(1519589,'Avg Physical Memory (MB)','944',NULL),(1519589,'Avg task runtime','28 sec',NULL),(1519589,'Avg Virtual Memory (MB)','2924',NULL),(1519589,'Max Physical Memory (MB)','961',NULL),(1519589,'Min Physical Memory (MB)','924',NULL),(1519589,'Number of tasks','27',NULL),(1519589,'Requested Container Memory','1 GB',NULL),(1519590,'Group A','6 tasks @ 48 MB avg',NULL),(1519590,'Group B','7 tasks @ 48 MB avg',NULL),(1519590,'Number of tasks','13',NULL),(1519591,'Avg task CPU time (ms)','20741',NULL),(1519591,'Avg task GC time (ms)','168',NULL),(1519591,'Avg task runtime (ms)','15476',NULL),(1519591,'Number of tasks','13',NULL),(1519591,'Task GC/CPU ratio','0.00809989875126561',NULL),(1519592,'Average task runtime','15 sec',NULL),(1519592,'Max task runtime','19 sec',NULL),(1519592,'Min task runtime','14 sec',NULL),(1519592,'Number of tasks','13',NULL),(1519593,'Avg Physical Memory (MB)','605',NULL),(1519593,'Avg task runtime','15 sec',NULL),(1519593,'Avg Virtual Memory (MB)','2936',NULL),(1519593,'Max Physical Memory (MB)','652',NULL),(1519593,'Min Physical Memory (MB)','574',NULL),(1519593,'Number of tasks','13',NULL),(1519593,'Requested Container Memory','1 GB',NULL),(1519594,'Average code runtime','8 sec',NULL),(1519594,'Average shuffle time','5 sec (0.61x)',NULL),(1519594,'Average sort time','1 sec (0.17x)',NULL),(1519594,'Number of tasks','13',NULL),(1519596,'Group A','6 tasks @ 444 MB avg',NULL),(1519596,'Group B','3 tasks @ 478 MB avg',NULL),(1519596,'Number of tasks','9',NULL),(1519597,'Avg task CPU time (ms)','45667',NULL),(1519597,'Avg task GC time (ms)','644',NULL),(1519597,'Avg task runtime (ms)','33362',NULL),(1519597,'Number of tasks','9',NULL),(1519597,'Task GC/CPU ratio','0.014102086846081415',NULL),(1519598,'Average task input size','455 MB',NULL),(1519598,'Average task runtime','33 sec',NULL),(1519598,'Max task runtime','48 sec',NULL),(1519598,'Min task runtime','29 sec',NULL),(1519598,'Number of tasks','9',NULL),(1519599,'Median task input size','451 MB',NULL),(1519599,'Median task runtime','31 sec',NULL),(1519599,'Median task speed','13 MB/s',NULL),(1519599,'Number of tasks','9',NULL),(1519599,'Total input size in MB','4099.407117843628',NULL),(1519600,'Avg output records per task','4243305',NULL),(1519600,'Avg spilled records per task','4855608',NULL),(1519600,'Number of tasks','9',NULL),(1519600,'Ratio of spilled records to output records','1.1442985879719034',NULL),(1519601,'Avg Physical Memory (MB)','952',NULL),(1519601,'Avg task runtime','33 sec',NULL),(1519601,'Avg Virtual Memory (MB)','2927',NULL),(1519601,'Max Physical Memory (MB)','959',NULL),(1519601,'Min Physical Memory (MB)','943',NULL),(1519601,'Number of tasks','9',NULL),(1519601,'Requested Container Memory','1 GB',NULL),(1519602,'Group A','3 tasks @ 65 MB avg',NULL),(1519602,'Group B','2 tasks @ 65 MB avg',NULL),(1519602,'Number of tasks','5',NULL),(1519603,'Avg task CPU time (ms)','28524',NULL),(1519603,'Avg task GC time (ms)','756',NULL),(1519603,'Avg task runtime (ms)','21506',NULL),(1519603,'Number of tasks','5',NULL),(1519603,'Task GC/CPU ratio','0.026503996634413125',NULL),(1519604,'Average task runtime','21 sec',NULL),(1519604,'Max task runtime','26 sec',NULL),(1519604,'Min task runtime','18 sec',NULL),(1519604,'Number of tasks','5',NULL),(1519605,'Avg Physical Memory (MB)','963',NULL),(1519605,'Avg task runtime','21 sec',NULL),(1519605,'Avg Virtual Memory (MB)','2938',NULL),(1519605,'Max Physical Memory (MB)','1173',NULL),(1519605,'Min Physical Memory (MB)','891',NULL),(1519605,'Number of tasks','5',NULL),(1519605,'Requested Container Memory','1 GB',NULL),(1519606,'Average code runtime','12 sec',NULL),(1519606,'Average shuffle time','6 sec (0.50x)',NULL),(1519606,'Average sort time','2 sec (0.19x)',NULL),(1519606,'Number of tasks','5',NULL),(1519608,'Group A','1 tasks @ 157 MB avg',NULL),(1519608,'Group B','5 tasks @ 370 MB avg',NULL),(1519608,'Number of tasks','6',NULL),(1519609,'Avg task CPU time (ms)','28300',NULL),(1519609,'Avg task GC time (ms)','476',NULL),(1519609,'Avg task runtime (ms)','22000',NULL),(1519609,'Number of tasks','6',NULL),(1519609,'Task GC/CPU ratio','0.016819787985865725',NULL),(1519610,'Average task input size','334 MB',NULL),(1519610,'Average task runtime','22 sec',NULL),(1519610,'Max task runtime','27 sec',NULL),(1519610,'Min task runtime','17 sec',NULL),(1519610,'Number of tasks','6',NULL),(1519611,'Median task input size','355 MB',NULL),(1519611,'Median task runtime','20 sec',NULL),(1519611,'Median task speed','15 MB/s',NULL),(1519611,'Number of tasks','6',NULL),(1519611,'Total input size in MB','2008.3901090621948',NULL),(1519612,'Avg output records per task','4925411',NULL),(1519612,'Avg spilled records per task','9850822',NULL),(1519612,'Number of tasks','6',NULL),(1519612,'Ratio of spilled records to output records','2.0',NULL),(1519613,'Avg Physical Memory (MB)','927',NULL),(1519613,'Avg task runtime','22 sec',NULL),(1519613,'Avg Virtual Memory (MB)','2916',NULL),(1519613,'Max Physical Memory (MB)','943',NULL),(1519613,'Min Physical Memory (MB)','912',NULL),(1519613,'Number of tasks','6',NULL),(1519613,'Requested Container Memory','1 GB',NULL),(1519614,'Group A','1 tasks @ 85 MB avg',NULL),(1519614,'Group B','2 tasks @ 86 MB avg',NULL),(1519614,'Number of tasks','3',NULL),(1519615,'Avg task CPU time (ms)','40850',NULL),(1519615,'Avg task GC time (ms)','653',NULL),(1519615,'Avg task runtime (ms)','29500',NULL),(1519615,'Number of tasks','3',NULL),(1519615,'Task GC/CPU ratio','0.01598531211750306',NULL),(1519616,'Average task runtime','29 sec',NULL),(1519616,'Max task runtime','29 sec',NULL),(1519616,'Min task runtime','28 sec',NULL),(1519616,'Number of tasks','3',NULL),(1519617,'Avg Physical Memory (MB)','1272',NULL),(1519617,'Avg task runtime','29 sec',NULL),(1519617,'Avg Virtual Memory (MB)','2932',NULL),(1519617,'Max Physical Memory (MB)','1297',NULL),(1519617,'Min Physical Memory (MB)','1230',NULL),(1519617,'Number of tasks','3',NULL),(1519617,'Requested Container Memory','1 GB',NULL),(1519618,'Average code runtime','19 sec',NULL),(1519618,'Average shuffle time','5 sec (0.26x)',NULL),(1519618,'Average sort time','4 sec (0.22x)',NULL),(1519618,'Number of tasks','3',NULL),(1519620,'Group A','1 tasks @ 477 MB avg',NULL),(1519620,'Group B','2 tasks @ 496 MB avg',NULL),(1519620,'Number of tasks','3',NULL),(1519621,'Avg task CPU time (ms)','38393',NULL),(1519621,'Avg task GC time (ms)','484',NULL),(1519621,'Avg task runtime (ms)','27491',NULL),(1519621,'Number of tasks','3',NULL),(1519621,'Task GC/CPU ratio','0.012606464720131274',NULL),(1519622,'Average task input size','490 MB',NULL),(1519622,'Average task runtime','27 sec',NULL),(1519622,'Max task runtime','30 sec',NULL),(1519622,'Min task runtime','25 sec',NULL),(1519622,'Number of tasks','3',NULL),(1519623,'Median task input size','496 MB',NULL),(1519623,'Median task runtime','26 sec',NULL),(1519623,'Median task speed','18 MB/s',NULL),(1519623,'Number of tasks','3',NULL),(1519623,'Total input size in MB','1471.195945739746',NULL),(1519624,'Avg output records per task','8880890',NULL),(1519624,'Avg spilled records per task','635557',NULL),(1519624,'Number of tasks','3',NULL),(1519624,'Ratio of spilled records to output records','0.07156455897383562',NULL),(1519625,'Avg Physical Memory (MB)','943',NULL),(1519625,'Avg task runtime','27 sec',NULL),(1519625,'Avg Virtual Memory (MB)','2922',NULL),(1519625,'Max Physical Memory (MB)','953',NULL),(1519625,'Min Physical Memory (MB)','932',NULL),(1519625,'Number of tasks','3',NULL),(1519625,'Requested Container Memory','1 GB',NULL),(1519626,'Group A','1 tasks @ 15 MB avg',NULL),(1519626,'Group B','1 tasks @ 15 MB avg',NULL),(1519626,'Number of tasks','2',NULL),(1519627,'Avg task CPU time (ms)','7175',NULL),(1519627,'Avg task GC time (ms)','103',NULL),(1519627,'Avg task runtime (ms)','7740',NULL),(1519627,'Number of tasks','2',NULL),(1519627,'Task GC/CPU ratio','0.014355400696864112',NULL),(1519628,'Average task runtime','7 sec',NULL),(1519628,'Max task runtime','8 sec',NULL),(1519628,'Min task runtime','6 sec',NULL),(1519628,'Number of tasks','2',NULL),(1519629,'Avg Physical Memory (MB)','520',NULL),(1519629,'Avg task runtime','7 sec',NULL),(1519629,'Avg Virtual Memory (MB)','2928',NULL),(1519629,'Max Physical Memory (MB)','524',NULL),(1519629,'Min Physical Memory (MB)','517',NULL),(1519629,'Number of tasks','2',NULL),(1519629,'Requested Container Memory','1 GB',NULL),(1519630,'Average code runtime','2 sec',NULL),(1519630,'Average shuffle time','5 sec (2.49x)',NULL),(1519630,'Average sort time',' (0.19x)',NULL),(1519630,'Number of tasks','2',NULL),(1519632,'Group A','299 tasks @ 862 MB avg',NULL),(1519632,'Group B','201 tasks @ 1 GB avg',NULL),(1519632,'Number of tasks','500',NULL),(1519633,'Avg task CPU time (ms)','123768',NULL),(1519633,'Avg task GC time (ms)','3603',NULL),(1519633,'Avg task runtime (ms)','103449',NULL),(1519633,'Number of tasks','500',NULL),(1519633,'Task GC/CPU ratio','0.029110917199922436',NULL),(1519634,'Average task input size','1 GB',NULL),(1519634,'Average task runtime','1 min 43 sec',NULL),(1519634,'Max task runtime','3 min 53 sec',NULL),(1519634,'Min task runtime','23 sec',NULL),(1519634,'Number of tasks','500',NULL),(1519635,'Median task input size','950 MB',NULL),(1519635,'Median task runtime','1 min 40 sec',NULL),(1519635,'Median task speed','10 MB/s',NULL),(1519635,'Number of tasks','500',NULL),(1519635,'Total input size in MB','530836.3677902222',NULL),(1519636,'Avg output records per task','744680',NULL),(1519636,'Avg spilled records per task','238063',NULL),(1519636,'Number of tasks','500',NULL),(1519636,'Ratio of spilled records to output records','0.3196848153856348',NULL),(1519637,'Avg Physical Memory (MB)','980',NULL),(1519637,'Avg task runtime','1 min 43 sec',NULL),(1519637,'Avg Virtual Memory (MB)','2924',NULL),(1519637,'Max Physical Memory (MB)','1112',NULL),(1519637,'Min Physical Memory (MB)','947',NULL),(1519637,'Number of tasks','500',NULL),(1519637,'Requested Container Memory','1 GB',NULL),(1519638,'Group A','277 tasks @ 7 MB avg',NULL),(1519638,'Group B','280 tasks @ 7 MB avg',NULL),(1519638,'Number of tasks','557',NULL),(1519639,'Avg task CPU time (ms)','9354',NULL),(1519639,'Avg task GC time (ms)','85',NULL),(1519639,'Avg task runtime (ms)','7249',NULL),(1519639,'Number of tasks','557',NULL),(1519639,'Task GC/CPU ratio','0.009087021595039555',NULL),(1519640,'Average task runtime','7 sec',NULL),(1519640,'Max task runtime','17 sec',NULL),(1519640,'Min task runtime','3 sec',NULL),(1519640,'Number of tasks','557',NULL),(1519641,'Avg Physical Memory (MB)','565',NULL),(1519641,'Avg task runtime','7 sec',NULL),(1519641,'Avg Virtual Memory (MB)','2944',NULL),(1519641,'Max Physical Memory (MB)','610',NULL),(1519641,'Min Physical Memory (MB)','516',NULL),(1519641,'Number of tasks','557',NULL),(1519641,'Requested Container Memory','1 GB',NULL),(1519642,'Average code runtime','',NULL),(1519642,'Average shuffle time','6 sec (26.12x)',NULL),(1519642,'Average sort time',' (1.20x)',NULL),(1519642,'Number of tasks','557',NULL),(1519644,'Group A','1 tasks @ 266 MB avg',NULL),(1519644,'Group B','1 tasks @ 514 MB avg',NULL),(1519644,'Number of tasks','2',NULL),(1519645,'Avg task CPU time (ms)','70820',NULL),(1519645,'Avg task GC time (ms)','2151',NULL),(1519645,'Avg task runtime (ms)','76709',NULL),(1519645,'Number of tasks','2',NULL),(1519645,'Task GC/CPU ratio','0.030372776051962723',NULL),(1519646,'Average task input size','390 MB',NULL),(1519646,'Average task runtime','1 min 16 sec',NULL),(1519646,'Max task runtime','1 min 37 sec',NULL),(1519646,'Min task runtime','55 sec',NULL),(1519646,'Number of tasks','2',NULL),(1519647,'Median task input size','390 MB',NULL),(1519647,'Median task runtime','1 min 16 sec',NULL),(1519647,'Median task speed','5 MB/s',NULL),(1519647,'Number of tasks','2',NULL),(1519647,'Total input size in MB','780.7726430892944',NULL),(1519648,'Avg output records per task','1242835',NULL),(1519648,'Avg spilled records per task','0',NULL),(1519648,'Number of tasks','2',NULL),(1519648,'Ratio of spilled records to output records','0.0',NULL),(1519649,'Avg Physical Memory (MB)','637',NULL),(1519649,'Avg task runtime','1 min 16 sec',NULL),(1519649,'Avg Virtual Memory (MB)','2983',NULL),(1519649,'Max Physical Memory (MB)','654',NULL),(1519649,'Min Physical Memory (MB)','620',NULL),(1519649,'Number of tasks','2',NULL),(1519649,'Requested Container Memory','1 GB',NULL),(1519650,'Group A','0 tasks @ 0 bytes avg',NULL),(1519650,'Group B','0 tasks @ 0 bytes avg',NULL),(1519650,'Number of tasks','0',NULL),(1519651,'Avg task CPU time (ms)','0',NULL),(1519651,'Avg task GC time (ms)','0',NULL),(1519651,'Avg task runtime (ms)','0',NULL),(1519651,'Number of tasks','0',NULL),(1519651,'Task GC/CPU ratio','0.0',NULL),(1519652,'Average task runtime','0 sec',NULL),(1519652,'Max task runtime','0 sec',NULL),(1519652,'Min task runtime','0 sec',NULL),(1519652,'Number of tasks','0',NULL),(1519653,'Avg Physical Memory (MB)','0',NULL),(1519653,'Avg task runtime','0 sec',NULL),(1519653,'Avg Virtual Memory (MB)','0',NULL),(1519653,'Max Physical Memory (MB)','0',NULL),(1519653,'Min Physical Memory (MB)','0',NULL),(1519653,'Number of tasks','0',NULL),(1519653,'Requested Container Memory','1 GB',NULL),(1519654,'Average code runtime','0 sec',NULL),(1519654,'Average shuffle time','0 sec ',NULL),(1519654,'Average sort time','0 sec ',NULL),(1519654,'Number of tasks','0',NULL),(1519656,'Group A','53 tasks @ 306 MB avg',NULL),(1519656,'Group B','98 tasks @ 483 MB avg',NULL),(1519656,'Number of tasks','151',NULL),(1519657,'Avg task CPU time (ms)','64471',NULL),(1519657,'Avg task GC time (ms)','728',NULL),(1519657,'Avg task runtime (ms)','57720',NULL),(1519657,'Number of tasks','151',NULL),(1519657,'Task GC/CPU ratio','0.011291898683128848',NULL),(1519658,'Average task input size','421 MB',NULL),(1519658,'Average task runtime','57 sec',NULL),(1519658,'Max task runtime','1 min 44 sec',NULL),(1519658,'Min task runtime','28 sec',NULL),(1519658,'Number of tasks','151',NULL),(1519659,'Median task input size','469 MB',NULL),(1519659,'Median task runtime','55 sec',NULL),(1519659,'Median task speed','7 MB/s',NULL),(1519659,'Number of tasks','151',NULL),(1519659,'Total input size in MB','63655.82713317871',NULL),(1519660,'Avg output records per task','1537392',NULL),(1519660,'Avg spilled records per task','0',NULL),(1519660,'Number of tasks','151',NULL),(1519660,'Ratio of spilled records to output records','0.0',NULL),(1519661,'Avg Physical Memory (MB)','609',NULL),(1519661,'Avg task runtime','57 sec',NULL),(1519661,'Avg Virtual Memory (MB)','2927',NULL),(1519661,'Max Physical Memory (MB)','645',NULL),(1519661,'Min Physical Memory (MB)','576',NULL),(1519661,'Number of tasks','151',NULL),(1519661,'Requested Container Memory','1 GB',NULL),(1519662,'Group A','0 tasks @ 0 bytes avg',NULL),(1519662,'Group B','0 tasks @ 0 bytes avg',NULL),(1519662,'Number of tasks','0',NULL),(1519663,'Avg task CPU time (ms)','0',NULL),(1519663,'Avg task GC time (ms)','0',NULL),(1519663,'Avg task runtime (ms)','0',NULL),(1519663,'Number of tasks','0',NULL),(1519663,'Task GC/CPU ratio','0.0',NULL),(1519664,'Average task runtime','0 sec',NULL),(1519664,'Max task runtime','0 sec',NULL),(1519664,'Min task runtime','0 sec',NULL),(1519664,'Number of tasks','0',NULL),(1519665,'Avg Physical Memory (MB)','0',NULL),(1519665,'Avg task runtime','0 sec',NULL),(1519665,'Avg Virtual Memory (MB)','0',NULL),(1519665,'Max Physical Memory (MB)','0',NULL),(1519665,'Min Physical Memory (MB)','0',NULL),(1519665,'Number of tasks','0',NULL),(1519665,'Requested Container Memory','1 GB',NULL),(1519666,'Average code runtime','0 sec',NULL),(1519666,'Average shuffle time','0 sec ',NULL),(1519666,'Average sort time','0 sec ',NULL),(1519666,'Number of tasks','0',NULL),(1519668,'Group A','305 tasks @ 290 MB avg',NULL),(1519668,'Group B','201 tasks @ 453 MB avg',NULL),(1519668,'Number of tasks','506',NULL),(1519669,'Avg task CPU time (ms)','73100',NULL),(1519669,'Avg task GC time (ms)','1589',NULL),(1519669,'Avg task runtime (ms)','66727',NULL),(1519669,'Number of tasks','506',NULL),(1519669,'Task GC/CPU ratio','0.02173734610123119',NULL),(1519670,'Average task input size','355 MB',NULL),(1519670,'Average task runtime','1 min 6 sec',NULL),(1519670,'Max task runtime','2 min 23 sec',NULL),(1519670,'Min task runtime','31 sec',NULL),(1519670,'Number of tasks','506',NULL),(1519671,'Median task input size','320 MB',NULL),(1519671,'Median task runtime','1 min 2 sec',NULL),(1519671,'Median task speed','5 MB/s',NULL),(1519671,'Number of tasks','506',NULL),(1519671,'Total input size in MB','179708.45590114594',NULL),(1519672,'Avg output records per task','307333',NULL),(1519672,'Avg spilled records per task','0',NULL),(1519672,'Number of tasks','506',NULL),(1519672,'Ratio of spilled records to output records','0.0',NULL),(1519673,'Avg Physical Memory (MB)','598',NULL),(1519673,'Avg task runtime','1 min 6 sec',NULL),(1519673,'Avg Virtual Memory (MB)','2922',NULL),(1519673,'Max Physical Memory (MB)','644',NULL),(1519673,'Min Physical Memory (MB)','567',NULL),(1519673,'Number of tasks','506',NULL),(1519673,'Requested Container Memory','1 GB',NULL),(1519674,'Group A','0 tasks @ 0 bytes avg',NULL),(1519674,'Group B','0 tasks @ 0 bytes avg',NULL),(1519674,'Number of tasks','0',NULL),(1519675,'Avg task CPU time (ms)','0',NULL),(1519675,'Avg task GC time (ms)','0',NULL),(1519675,'Avg task runtime (ms)','0',NULL),(1519675,'Number of tasks','0',NULL),(1519675,'Task GC/CPU ratio','0.0',NULL),(1519676,'Average task runtime','0 sec',NULL),(1519676,'Max task runtime','0 sec',NULL),(1519676,'Min task runtime','0 sec',NULL),(1519676,'Number of tasks','0',NULL),(1519677,'Avg Physical Memory (MB)','0',NULL),(1519677,'Avg task runtime','0 sec',NULL),(1519677,'Avg Virtual Memory (MB)','0',NULL),(1519677,'Max Physical Memory (MB)','0',NULL),(1519677,'Min Physical Memory (MB)','0',NULL),(1519677,'Number of tasks','0',NULL),(1519677,'Requested Container Memory','1 GB',NULL),(1519678,'Average code runtime','0 sec',NULL),(1519678,'Average shuffle time','0 sec ',NULL),(1519678,'Average sort time','0 sec ',NULL),(1519678,'Number of tasks','0',NULL),(1519680,'Group A','932 tasks @ 780 MB avg',NULL),(1519680,'Group B','847 tasks @ 1 GB avg',NULL),(1519680,'Number of tasks','1779',NULL),(1519681,'Avg task CPU time (ms)','71616',NULL),(1519681,'Avg task GC time (ms)','2456',NULL),(1519681,'Avg task runtime (ms)','47598',NULL),(1519681,'Number of tasks','1779',NULL),(1519681,'Task GC/CPU ratio','0.03429401251117069',NULL),(1519682,'Average task input size','944 MB',NULL),(1519682,'Average task runtime','47 sec',NULL),(1519682,'Max task runtime','1 min 56 sec',NULL),(1519682,'Min task runtime','22 sec',NULL),(1519682,'Number of tasks','1779',NULL),(1519683,'Median task input size','924 MB',NULL),(1519683,'Median task runtime','45 sec',NULL),(1519683,'Median task speed','20 MB/s',NULL),(1519683,'Number of tasks','1779',NULL),(1519683,'Total input size in MB','1679594.0204076767',NULL),(1519684,'Avg output records per task','3494130',NULL),(1519684,'Avg spilled records per task','6988261',NULL),(1519684,'Number of tasks','1779',NULL),(1519684,'Ratio of spilled records to output records','2.0',NULL),(1519685,'Avg Physical Memory (MB)','989',NULL),(1519685,'Avg task runtime','47 sec',NULL),(1519685,'Avg Virtual Memory (MB)','2940',NULL),(1519685,'Max Physical Memory (MB)','1094',NULL),(1519685,'Min Physical Memory (MB)','915',NULL),(1519685,'Number of tasks','1779',NULL),(1519685,'Requested Container Memory','1 GB',NULL),(1519686,'Group A','525 tasks @ 500 MB avg',NULL),(1519686,'Group B','474 tasks @ 501 MB avg',NULL),(1519686,'Number of tasks','999',NULL),(1519687,'Avg task CPU time (ms)','69159',NULL),(1519687,'Avg task GC time (ms)','1680',NULL),(1519687,'Avg task runtime (ms)','49701',NULL),(1519687,'Number of tasks','999',NULL),(1519687,'Task GC/CPU ratio','0.024291849217021644',NULL),(1519688,'Average task runtime','49 sec',NULL),(1519688,'Max task runtime','1 min 43 sec',NULL),(1519688,'Min task runtime','36 sec',NULL),(1519688,'Number of tasks','999',NULL),(1519689,'Avg Physical Memory (MB)','1298',NULL),(1519689,'Avg task runtime','49 sec',NULL),(1519689,'Avg Virtual Memory (MB)','2960',NULL),(1519689,'Max Physical Memory (MB)','1468',NULL),(1519689,'Min Physical Memory (MB)','1038',NULL),(1519689,'Number of tasks','999',NULL),(1519689,'Requested Container Memory','1 GB',NULL),(1519690,'Average code runtime','29 sec',NULL),(1519690,'Average shuffle time','18 sec (0.64x)',NULL),(1519690,'Average sort time','2 sec (0.07x)',NULL),(1519690,'Number of tasks','999',NULL),(1519692,'Group A','1 tasks @ 291 MB avg',NULL),(1519692,'Group B','1 tasks @ 500 MB avg',NULL),(1519692,'Number of tasks','2',NULL),(1519693,'Avg task CPU time (ms)','128560',NULL),(1519693,'Avg task GC time (ms)','5568',NULL),(1519693,'Avg task runtime (ms)','93906',NULL),(1519693,'Number of tasks','2',NULL),(1519693,'Task GC/CPU ratio','0.0433105164903547',NULL),(1519694,'Average task input size','396 MB',NULL),(1519694,'Average task runtime','1 min 33 sec',NULL),(1519694,'Max task runtime','2 min 11 sec',NULL),(1519694,'Min task runtime','56 sec',NULL),(1519694,'Number of tasks','2',NULL),(1519695,'Median task input size','396 MB',NULL),(1519695,'Median task runtime','1 min 33 sec',NULL),(1519695,'Median task speed','4 MB/s',NULL),(1519695,'Number of tasks','2',NULL),(1519695,'Total input size in MB','792.533878326416',NULL),(1519696,'Avg output records per task','1243839',NULL),(1519696,'Avg spilled records per task','0',NULL),(1519696,'Number of tasks','2',NULL),(1519696,'Ratio of spilled records to output records','0.0',NULL),(1519697,'Avg Physical Memory (MB)','722',NULL),(1519697,'Avg task runtime','1 min 33 sec',NULL),(1519697,'Avg Virtual Memory (MB)','2921',NULL),(1519697,'Max Physical Memory (MB)','731',NULL),(1519697,'Min Physical Memory (MB)','712',NULL),(1519697,'Number of tasks','2',NULL),(1519697,'Requested Container Memory','1 GB',NULL),(1519698,'Group A','0 tasks @ 0 bytes avg',NULL),(1519698,'Group B','0 tasks @ 0 bytes avg',NULL),(1519698,'Number of tasks','0',NULL),(1519699,'Avg task CPU time (ms)','0',NULL),(1519699,'Avg task GC time (ms)','0',NULL),(1519699,'Avg task runtime (ms)','0',NULL),(1519699,'Number of tasks','0',NULL),(1519699,'Task GC/CPU ratio','0.0',NULL),(1519700,'Average task runtime','0 sec',NULL),(1519700,'Max task runtime','0 sec',NULL),(1519700,'Min task runtime','0 sec',NULL),(1519700,'Number of tasks','0',NULL),(1519701,'Avg Physical Memory (MB)','0',NULL),(1519701,'Avg task runtime','0 sec',NULL),(1519701,'Avg Virtual Memory (MB)','0',NULL),(1519701,'Max Physical Memory (MB)','0',NULL),(1519701,'Min Physical Memory (MB)','0',NULL),(1519701,'Number of tasks','0',NULL),(1519701,'Requested Container Memory','1 GB',NULL),(1519702,'Average code runtime','0 sec',NULL),(1519702,'Average shuffle time','0 sec ',NULL),(1519702,'Average sort time','0 sec ',NULL),(1519702,'Number of tasks','0',NULL),(1519704,'Group A','1 tasks @ 315 MB avg',NULL),(1519704,'Group B','1 tasks @ 513 MB avg',NULL),(1519704,'Number of tasks','2',NULL),(1519705,'Avg task CPU time (ms)','147230',NULL),(1519705,'Avg task GC time (ms)','3980',NULL),(1519705,'Avg task runtime (ms)','91773',NULL),(1519705,'Number of tasks','2',NULL),(1519705,'Task GC/CPU ratio','0.027032534130272363',NULL),(1519706,'Average task input size','414 MB',NULL),(1519706,'Average task runtime','1 min 31 sec',NULL),(1519706,'Max task runtime','1 min 52 sec',NULL),(1519706,'Min task runtime','1 min 11 sec',NULL),(1519706,'Number of tasks','2',NULL),(1519707,'Median task input size','414 MB',NULL),(1519707,'Median task runtime','1 min 31 sec',NULL),(1519707,'Median task speed','4 MB/s',NULL),(1519707,'Number of tasks','2',NULL),(1519707,'Total input size in MB','828.9662179946899',NULL),(1519708,'Avg output records per task','1205502',NULL),(1519708,'Avg spilled records per task','0',NULL),(1519708,'Number of tasks','2',NULL),(1519708,'Ratio of spilled records to output records','0.0',NULL),(1519709,'Avg Physical Memory (MB)','741',NULL),(1519709,'Avg task runtime','1 min 31 sec',NULL),(1519709,'Avg Virtual Memory (MB)','2920',NULL),(1519709,'Max Physical Memory (MB)','788',NULL),(1519709,'Min Physical Memory (MB)','694',NULL),(1519709,'Number of tasks','2',NULL),(1519709,'Requested Container Memory','1 GB',NULL),(1519710,'Group A','0 tasks @ 0 bytes avg',NULL),(1519710,'Group B','0 tasks @ 0 bytes avg',NULL),(1519710,'Number of tasks','0',NULL),(1519711,'Avg task CPU time (ms)','0',NULL),(1519711,'Avg task GC time (ms)','0',NULL),(1519711,'Avg task runtime (ms)','0',NULL),(1519711,'Number of tasks','0',NULL),(1519711,'Task GC/CPU ratio','0.0',NULL),(1519712,'Average task runtime','0 sec',NULL),(1519712,'Max task runtime','0 sec',NULL),(1519712,'Min task runtime','0 sec',NULL),(1519712,'Number of tasks','0',NULL),(1519713,'Avg Physical Memory (MB)','0',NULL),(1519713,'Avg task runtime','0 sec',NULL),(1519713,'Avg Virtual Memory (MB)','0',NULL),(1519713,'Max Physical Memory (MB)','0',NULL),(1519713,'Min Physical Memory (MB)','0',NULL),(1519713,'Number of tasks','0',NULL),(1519713,'Requested Container Memory','1 GB',NULL),(1519714,'Average code runtime','0 sec',NULL),(1519714,'Average shuffle time','0 sec ',NULL),(1519714,'Average sort time','0 sec ',NULL),(1519714,'Number of tasks','0',NULL),(1519716,'Group A','307 tasks @ 293 MB avg',NULL),(1519716,'Group B','193 tasks @ 448 MB avg',NULL),(1519716,'Number of tasks','500',NULL),(1519717,'Avg task CPU time (ms)','66281',NULL),(1519717,'Avg task GC time (ms)','1015',NULL),(1519717,'Avg task runtime (ms)','54589',NULL),(1519717,'Number of tasks','500',NULL),(1519717,'Task GC/CPU ratio','0.015313589113018814',NULL),(1519718,'Average task input size','353 MB',NULL),(1519718,'Average task runtime','54 sec',NULL),(1519718,'Max task runtime','2 min 4 sec',NULL),(1519718,'Min task runtime','32 sec',NULL),(1519718,'Number of tasks','500',NULL),(1519719,'Median task input size','319 MB',NULL),(1519719,'Median task runtime','51 sec',NULL),(1519719,'Median task speed','6 MB/s',NULL),(1519719,'Number of tasks','500',NULL),(1519719,'Total input size in MB','176676.36856079102',NULL),(1519720,'Avg output records per task','302754',NULL),(1519720,'Avg spilled records per task','0',NULL),(1519720,'Number of tasks','500',NULL),(1519720,'Ratio of spilled records to output records','0.0',NULL),(1519721,'Avg Physical Memory (MB)','602',NULL),(1519721,'Avg task runtime','54 sec',NULL),(1519721,'Avg Virtual Memory (MB)','2923',NULL),(1519721,'Max Physical Memory (MB)','674',NULL),(1519721,'Min Physical Memory (MB)','569',NULL),(1519721,'Number of tasks','500',NULL),(1519721,'Requested Container Memory','1 GB',NULL),(1519722,'Group A','0 tasks @ 0 bytes avg',NULL),(1519722,'Group B','0 tasks @ 0 bytes avg',NULL),(1519722,'Number of tasks','0',NULL),(1519723,'Avg task CPU time (ms)','0',NULL),(1519723,'Avg task GC time (ms)','0',NULL),(1519723,'Avg task runtime (ms)','0',NULL),(1519723,'Number of tasks','0',NULL),(1519723,'Task GC/CPU ratio','0.0',NULL),(1519724,'Average task runtime','0 sec',NULL),(1519724,'Max task runtime','0 sec',NULL),(1519724,'Min task runtime','0 sec',NULL),(1519724,'Number of tasks','0',NULL),(1519725,'Avg Physical Memory (MB)','0',NULL),(1519725,'Avg task runtime','0 sec',NULL),(1519725,'Avg Virtual Memory (MB)','0',NULL),(1519725,'Max Physical Memory (MB)','0',NULL),(1519725,'Min Physical Memory (MB)','0',NULL),(1519725,'Number of tasks','0',NULL),(1519725,'Requested Container Memory','1 GB',NULL),(1519726,'Average code runtime','0 sec',NULL),(1519726,'Average shuffle time','0 sec ',NULL),(1519726,'Average sort time','0 sec ',NULL),(1519726,'Number of tasks','0',NULL),(1519728,'Group A','897 tasks @ 309 MB avg',NULL),(1519728,'Group B','735 tasks @ 448 MB avg',NULL),(1519728,'Number of tasks','1632',NULL),(1519729,'Avg task CPU time (ms)','72014',NULL),(1519729,'Avg task GC time (ms)','1596',NULL),(1519729,'Avg task runtime (ms)','63650',NULL),(1519729,'Number of tasks','1632',NULL),(1519729,'Task GC/CPU ratio','0.022162357319410114',NULL),(1519730,'Average task input size','372 MB',NULL),(1519730,'Average task runtime','1 min 3 sec',NULL),(1519730,'Max task runtime','2 min 25 sec',NULL),(1519730,'Min task runtime','29 sec',NULL),(1519730,'Number of tasks','1632',NULL),(1519731,'Median task input size','358 MB',NULL),(1519731,'Median task runtime','59 sec',NULL),(1519731,'Median task speed','6 MB/s',NULL),(1519731,'Number of tasks','1632',NULL),(1519731,'Total input size in MB','607337.425579071',NULL),(1519732,'Avg output records per task','335777',NULL),(1519732,'Avg spilled records per task','0',NULL),(1519732,'Number of tasks','1632',NULL),(1519732,'Ratio of spilled records to output records','0.0',NULL),(1519733,'Avg Physical Memory (MB)','605',NULL),(1519733,'Avg task runtime','1 min 3 sec',NULL),(1519733,'Avg Virtual Memory (MB)','2925',NULL),(1519733,'Max Physical Memory (MB)','700',NULL),(1519733,'Min Physical Memory (MB)','568',NULL),(1519733,'Number of tasks','1632',NULL),(1519733,'Requested Container Memory','1 GB',NULL),(1519734,'Group A','0 tasks @ 0 bytes avg',NULL),(1519734,'Group B','0 tasks @ 0 bytes avg',NULL),(1519734,'Number of tasks','0',NULL),(1519735,'Avg task CPU time (ms)','0',NULL),(1519735,'Avg task GC time (ms)','0',NULL),(1519735,'Avg task runtime (ms)','0',NULL),(1519735,'Number of tasks','0',NULL),(1519735,'Task GC/CPU ratio','0.0',NULL),(1519736,'Average task runtime','0 sec',NULL),(1519736,'Max task runtime','0 sec',NULL),(1519736,'Min task runtime','0 sec',NULL),(1519736,'Number of tasks','0',NULL),(1519737,'Avg Physical Memory (MB)','0',NULL),(1519737,'Avg task runtime','0 sec',NULL),(1519737,'Avg Virtual Memory (MB)','0',NULL),(1519737,'Max Physical Memory (MB)','0',NULL),(1519737,'Min Physical Memory (MB)','0',NULL),(1519737,'Number of tasks','0',NULL),(1519737,'Requested Container Memory','1 GB',NULL),(1519738,'Average code runtime','0 sec',NULL),(1519738,'Average shuffle time','0 sec ',NULL),(1519738,'Average sort time','0 sec ',NULL),(1519738,'Number of tasks','0',NULL),(1519740,'Group A','307 tasks @ 858 MB avg',NULL),(1519740,'Group B','202 tasks @ 1 GB avg',NULL),(1519740,'Number of tasks','509',NULL),(1519741,'Avg task CPU time (ms)','126395',NULL),(1519741,'Avg task GC time (ms)','3820',NULL),(1519741,'Avg task runtime (ms)','107833',NULL),(1519741,'Number of tasks','509',NULL),(1519741,'Task GC/CPU ratio','0.030222714506111793',NULL),(1519742,'Average task input size','1 GB',NULL),(1519742,'Average task runtime','1 min 47 sec',NULL),(1519742,'Max task runtime','3 min 56 sec',NULL),(1519742,'Min task runtime','16 sec',NULL),(1519742,'Number of tasks','509',NULL),(1519743,'Median task input size','949 MB',NULL),(1519743,'Median task runtime','1 min 44 sec',NULL),(1519743,'Median task speed','10 MB/s',NULL),(1519743,'Number of tasks','509',NULL),(1519743,'Total input size in MB','535416.1670970917',NULL),(1519744,'Avg output records per task','736745',NULL),(1519744,'Avg spilled records per task','235806',NULL),(1519744,'Number of tasks','509',NULL),(1519744,'Ratio of spilled records to output records','0.3200645473294221',NULL),(1519745,'Avg Physical Memory (MB)','1039',NULL),(1519745,'Avg task runtime','1 min 47 sec',NULL),(1519745,'Avg Virtual Memory (MB)','2925',NULL),(1519745,'Max Physical Memory (MB)','1095',NULL),(1519745,'Min Physical Memory (MB)','811',NULL),(1519745,'Number of tasks','509',NULL),(1519745,'Requested Container Memory','1 GB',NULL),(1519746,'Group A','281 tasks @ 7 MB avg',NULL),(1519746,'Group B','281 tasks @ 7 MB avg',NULL),(1519746,'Number of tasks','562',NULL),(1519747,'Avg task CPU time (ms)','9259',NULL),(1519747,'Avg task GC time (ms)','92',NULL),(1519747,'Avg task runtime (ms)','9909',NULL),(1519747,'Number of tasks','562',NULL),(1519747,'Task GC/CPU ratio','0.009936278215790042',NULL),(1519748,'Average task runtime','9 sec',NULL),(1519748,'Max task runtime','1 min 6 sec',NULL),(1519748,'Min task runtime','3 sec',NULL),(1519748,'Number of tasks','562',NULL),(1519749,'Avg Physical Memory (MB)','567',NULL),(1519749,'Avg task runtime','9 sec',NULL),(1519749,'Avg Virtual Memory (MB)','2943',NULL),(1519749,'Max Physical Memory (MB)','618',NULL),(1519749,'Min Physical Memory (MB)','512',NULL),(1519749,'Number of tasks','562',NULL),(1519749,'Requested Container Memory','1 GB',NULL),(1519750,'Average code runtime','',NULL),(1519750,'Average shuffle time','9 sec (-31.16x)',NULL),(1519750,'Average sort time',' (-1.00x)',NULL),(1519750,'Number of tasks','562',NULL),(1519752,'Group A','1 tasks @ 150 MB avg',NULL),(1519752,'Group B','1 tasks @ 155 MB avg',NULL),(1519752,'Number of tasks','2',NULL),(1519753,'Avg task CPU time (ms)','24660',NULL),(1519753,'Avg task GC time (ms)','509',NULL),(1519753,'Avg task runtime (ms)','21495',NULL),(1519753,'Number of tasks','2',NULL),(1519753,'Task GC/CPU ratio','0.020640713706407136',NULL),(1519754,'Average task input size','153 MB',NULL),(1519754,'Average task runtime','21 sec',NULL),(1519754,'Max task runtime','25 sec',NULL),(1519754,'Min task runtime','17 sec',NULL),(1519754,'Number of tasks','2',NULL),(1519755,'Median task input size','153 MB',NULL),(1519755,'Median task runtime','21 sec',NULL),(1519755,'Median task speed','7 MB/s',NULL),(1519755,'Number of tasks','2',NULL),(1519755,'Total input size in MB','306.038969039917',NULL),(1519756,'Avg output records per task','2900081',NULL),(1519756,'Avg spilled records per task','5800162',NULL),(1519756,'Number of tasks','2',NULL),(1519756,'Ratio of spilled records to output records','2.0',NULL),(1519757,'Avg Physical Memory (MB)','993',NULL),(1519757,'Avg task runtime','21 sec',NULL),(1519757,'Avg Virtual Memory (MB)','2923',NULL),(1519757,'Max Physical Memory (MB)','996',NULL),(1519757,'Min Physical Memory (MB)','990',NULL),(1519757,'Number of tasks','2',NULL),(1519757,'Requested Container Memory','1 GB',NULL),(1519758,'Group A','0 tasks @ 0 bytes avg',NULL),(1519758,'Group B','1 tasks @ 187 MB avg',NULL),(1519758,'Number of tasks','1',NULL),(1519759,'Avg task CPU time (ms)','47200',NULL),(1519759,'Avg task GC time (ms)','1500',NULL),(1519759,'Avg task runtime (ms)','41974',NULL),(1519759,'Number of tasks','1',NULL),(1519759,'Task GC/CPU ratio','0.03177966101694915',NULL),(1519760,'Average task runtime','41 sec',NULL),(1519760,'Max task runtime','41 sec',NULL),(1519760,'Min task runtime','41 sec',NULL),(1519760,'Number of tasks','1',NULL),(1519761,'Avg Physical Memory (MB)','743',NULL),(1519761,'Avg task runtime','41 sec',NULL),(1519761,'Avg Virtual Memory (MB)','2914',NULL),(1519761,'Max Physical Memory (MB)','743',NULL),(1519761,'Min Physical Memory (MB)','743',NULL),(1519761,'Number of tasks','1',NULL),(1519761,'Requested Container Memory','1 GB',NULL),(1519762,'Average code runtime','27 sec',NULL),(1519762,'Average shuffle time','10 sec (0.37x)',NULL),(1519762,'Average sort time','4 sec (0.16x)',NULL),(1519762,'Number of tasks','1',NULL),(1519764,'Group A','0 tasks @ 0 bytes avg',NULL),(1519764,'Group B','1 tasks @ 163 MB avg',NULL),(1519764,'Number of tasks','1',NULL),(1519765,'Avg task CPU time (ms)','61890',NULL),(1519765,'Avg task GC time (ms)','5056',NULL),(1519765,'Avg task runtime (ms)','62645',NULL),(1519765,'Number of tasks','1',NULL),(1519765,'Task GC/CPU ratio','0.08169332687025367',NULL),(1519766,'Average task input size','163 MB',NULL),(1519766,'Average task runtime','1 min 2 sec',NULL),(1519766,'Max task runtime','1 min 2 sec',NULL),(1519766,'Min task runtime','1 min 2 sec',NULL),(1519766,'Number of tasks','1',NULL),(1519767,'Median task input size','163 MB',NULL),(1519767,'Median task runtime','1 min 2 sec',NULL),(1519767,'Median task speed','2 MB/s',NULL),(1519767,'Number of tasks','1',NULL),(1519767,'Total input size in MB','163.07244873046875',NULL),(1519768,'Avg output records per task','2917846',NULL),(1519768,'Avg spilled records per task','5835692',NULL),(1519768,'Number of tasks','1',NULL),(1519768,'Ratio of spilled records to output records','2.0',NULL),(1519769,'Avg Physical Memory (MB)','1017',NULL),(1519769,'Avg task runtime','1 min 2 sec',NULL),(1519769,'Avg Virtual Memory (MB)','2912',NULL),(1519769,'Max Physical Memory (MB)','1017',NULL),(1519769,'Min Physical Memory (MB)','1017',NULL),(1519769,'Number of tasks','1',NULL),(1519769,'Requested Container Memory','1 GB',NULL),(1519770,'Group A','0 tasks @ 0 bytes avg',NULL),(1519770,'Group B','1 tasks @ 96 MB avg',NULL),(1519770,'Number of tasks','1',NULL),(1519771,'Avg task CPU time (ms)','20850',NULL),(1519771,'Avg task GC time (ms)','105',NULL),(1519771,'Avg task runtime (ms)','19324',NULL),(1519771,'Number of tasks','1',NULL),(1519771,'Task GC/CPU ratio','0.005035971223021582',NULL),(1519772,'Average task runtime','19 sec',NULL),(1519772,'Max task runtime','19 sec',NULL),(1519772,'Min task runtime','19 sec',NULL),(1519772,'Number of tasks','1',NULL),(1519773,'Avg Physical Memory (MB)','573',NULL),(1519773,'Avg task runtime','19 sec',NULL),(1519773,'Avg Virtual Memory (MB)','2914',NULL),(1519773,'Max Physical Memory (MB)','573',NULL),(1519773,'Min Physical Memory (MB)','573',NULL),(1519773,'Number of tasks','1',NULL),(1519773,'Requested Container Memory','1 GB',NULL),(1519774,'Average code runtime','12 sec',NULL),(1519774,'Average shuffle time','5 sec (0.41x)',NULL),(1519774,'Average sort time','1 sec (0.14x)',NULL),(1519774,'Number of tasks','1',NULL),(1519776,'Group A','0 tasks @ 0 bytes avg',NULL),(1519776,'Group B','1 tasks @ 158 MB avg',NULL),(1519776,'Number of tasks','1',NULL),(1519777,'Avg task CPU time (ms)','19580',NULL),(1519777,'Avg task GC time (ms)','355',NULL),(1519777,'Avg task runtime (ms)','15753',NULL),(1519777,'Number of tasks','1',NULL),(1519777,'Task GC/CPU ratio','0.018130745658835545',NULL),(1519778,'Average task input size','158 MB',NULL),(1519778,'Average task runtime','15 sec',NULL),(1519778,'Max task runtime','15 sec',NULL),(1519778,'Min task runtime','15 sec',NULL),(1519778,'Number of tasks','1',NULL),(1519779,'Median task input size','158 MB',NULL),(1519779,'Median task runtime','15 sec',NULL),(1519779,'Median task speed','10 MB/s',NULL),(1519779,'Number of tasks','1',NULL),(1519779,'Total input size in MB','158.0611515045166',NULL),(1519780,'Avg output records per task','2917846',NULL),(1519780,'Avg spilled records per task','5835692',NULL),(1519780,'Number of tasks','1',NULL),(1519780,'Ratio of spilled records to output records','2.0',NULL),(1519781,'Avg Physical Memory (MB)','979',NULL),(1519781,'Avg task runtime','15 sec',NULL),(1519781,'Avg Virtual Memory (MB)','2914',NULL),(1519781,'Max Physical Memory (MB)','979',NULL),(1519781,'Min Physical Memory (MB)','979',NULL),(1519781,'Number of tasks','1',NULL),(1519781,'Requested Container Memory','1 GB',NULL),(1519782,'Group A','0 tasks @ 0 bytes avg',NULL),(1519782,'Group B','1 tasks @ 93 MB avg',NULL),(1519782,'Number of tasks','1',NULL),(1519783,'Avg task CPU time (ms)','20940',NULL),(1519783,'Avg task GC time (ms)','539',NULL),(1519783,'Avg task runtime (ms)','22238',NULL),(1519783,'Number of tasks','1',NULL),(1519783,'Task GC/CPU ratio','0.02574021012416428',NULL),(1519784,'Average task runtime','22 sec',NULL),(1519784,'Max task runtime','22 sec',NULL),(1519784,'Min task runtime','22 sec',NULL),(1519784,'Number of tasks','1',NULL),(1519785,'Avg Physical Memory (MB)','549',NULL),(1519785,'Avg task runtime','22 sec',NULL),(1519785,'Avg Virtual Memory (MB)','2922',NULL),(1519785,'Max Physical Memory (MB)','549',NULL),(1519785,'Min Physical Memory (MB)','549',NULL),(1519785,'Number of tasks','1',NULL),(1519785,'Requested Container Memory','1 GB',NULL),(1519786,'Average code runtime','11 sec',NULL),(1519786,'Average shuffle time','8 sec (0.71x)',NULL),(1519786,'Average sort time','2 sec (0.24x)',NULL),(1519786,'Number of tasks','1',NULL),(1519788,'Group A','2 tasks @ 176 MB avg',NULL),(1519788,'Group B','31 tasks @ 511 MB avg',NULL),(1519788,'Number of tasks','33',NULL),(1519789,'Avg task CPU time (ms)','53643',NULL),(1519789,'Avg task GC time (ms)','1419',NULL),(1519789,'Avg task runtime (ms)','44953',NULL),(1519789,'Number of tasks','33',NULL),(1519789,'Task GC/CPU ratio','0.02645265924724568',NULL),(1519790,'Average task input size','490 MB',NULL),(1519790,'Average task runtime','44 sec',NULL),(1519790,'Max task runtime','1 min 32 sec',NULL),(1519790,'Min task runtime','12 sec',NULL),(1519790,'Number of tasks','33',NULL),(1519791,'Median task input size','509 MB',NULL),(1519791,'Median task runtime','40 sec',NULL),(1519791,'Median task speed','11 MB/s',NULL),(1519791,'Number of tasks','33',NULL),(1519791,'Total input size in MB','16195.266300201416',NULL),(1519792,'Avg output records per task','5386511',NULL),(1519792,'Avg spilled records per task','10773022',NULL),(1519792,'Number of tasks','33',NULL),(1519792,'Ratio of spilled records to output records','2.0',NULL),(1519793,'Avg Physical Memory (MB)','971',NULL),(1519793,'Avg task runtime','44 sec',NULL),(1519793,'Avg Virtual Memory (MB)','2926',NULL),(1519793,'Max Physical Memory (MB)','1022',NULL),(1519793,'Min Physical Memory (MB)','770',NULL),(1519793,'Number of tasks','33',NULL),(1519793,'Requested Container Memory','1 GB',NULL),(1519794,'Group A','16 tasks @ 135 MB avg',NULL),(1519794,'Group B','1 tasks @ 995 MB avg',NULL),(1519794,'Number of tasks','17',NULL),(1519795,'Avg task CPU time (ms)','45495',NULL),(1519795,'Avg task GC time (ms)','1219',NULL),(1519795,'Avg task runtime (ms)','35958',NULL),(1519795,'Number of tasks','17',NULL),(1519795,'Task GC/CPU ratio','0.026794153203648752',NULL),(1519796,'Average task runtime','35 sec',NULL),(1519796,'Max task runtime','1 min 46 sec',NULL),(1519796,'Min task runtime','23 sec',NULL),(1519796,'Number of tasks','17',NULL),(1519797,'Avg Physical Memory (MB)','1329',NULL),(1519797,'Avg task runtime','35 sec',NULL),(1519797,'Avg Virtual Memory (MB)','2936',NULL),(1519797,'Max Physical Memory (MB)','1398',NULL),(1519797,'Min Physical Memory (MB)','1250',NULL),(1519797,'Number of tasks','17',NULL),(1519797,'Requested Container Memory','1 GB',NULL),(1519798,'Average code runtime','21 sec',NULL),(1519798,'Average shuffle time','12 sec (0.57x)',NULL),(1519798,'Average sort time','1 sec (0.07x)',NULL),(1519798,'Number of tasks','17',NULL),(1519800,'Group A','3 tasks @ 256 MB avg',NULL),(1519800,'Group B','24 tasks @ 480 MB avg',NULL),(1519800,'Number of tasks','27',NULL),(1519801,'Avg task CPU time (ms)','38015',NULL),(1519801,'Avg task GC time (ms)','890',NULL),(1519801,'Avg task runtime (ms)','27943',NULL),(1519801,'Number of tasks','27',NULL),(1519801,'Task GC/CPU ratio','0.023411811127186637',NULL),(1519802,'Average task input size','455 MB',NULL),(1519802,'Average task runtime','27 sec',NULL),(1519802,'Max task runtime','42 sec',NULL),(1519802,'Min task runtime','14 sec',NULL),(1519802,'Number of tasks','27',NULL),(1519803,'Median task input size','477 MB',NULL),(1519803,'Median task runtime','28 sec',NULL),(1519803,'Median task speed','17 MB/s',NULL),(1519803,'Number of tasks','27',NULL),(1519803,'Total input size in MB','12289.610097885132',NULL),(1519804,'Avg output records per task','4313924',NULL),(1519804,'Avg spilled records per task','2132270',NULL),(1519804,'Number of tasks','27',NULL),(1519804,'Ratio of spilled records to output records','0.49427629490420477',NULL),(1519805,'Avg Physical Memory (MB)','1014',NULL),(1519805,'Avg task runtime','27 sec',NULL),(1519805,'Avg Virtual Memory (MB)','2924',NULL),(1519805,'Max Physical Memory (MB)','1036',NULL),(1519805,'Min Physical Memory (MB)','799',NULL),(1519805,'Number of tasks','27',NULL),(1519805,'Requested Container Memory','1 GB',NULL),(1519806,'Group A','7 tasks @ 48 MB avg',NULL),(1519806,'Group B','6 tasks @ 48 MB avg',NULL),(1519806,'Number of tasks','13',NULL),(1519807,'Avg task CPU time (ms)','21024',NULL),(1519807,'Avg task GC time (ms)','286',NULL),(1519807,'Avg task runtime (ms)','16699',NULL),(1519807,'Number of tasks','13',NULL),(1519807,'Task GC/CPU ratio','0.013603500761035007',NULL),(1519808,'Average task runtime','16 sec',NULL),(1519808,'Max task runtime','28 sec',NULL),(1519808,'Min task runtime','13 sec',NULL),(1519808,'Number of tasks','13',NULL),(1519809,'Avg Physical Memory (MB)','605',NULL),(1519809,'Avg task runtime','16 sec',NULL),(1519809,'Avg Virtual Memory (MB)','2936',NULL),(1519809,'Max Physical Memory (MB)','623',NULL),(1519809,'Min Physical Memory (MB)','576',NULL),(1519809,'Number of tasks','13',NULL),(1519809,'Requested Container Memory','1 GB',NULL),(1519810,'Average code runtime','9 sec',NULL),(1519810,'Average shuffle time','5 sec (0.55x)',NULL),(1519810,'Average sort time','1 sec (0.16x)',NULL),(1519810,'Number of tasks','13',NULL),(1519812,'Group A','1 tasks @ 256 MB avg',NULL),(1519812,'Group B','8 tasks @ 480 MB avg',NULL),(1519812,'Number of tasks','9',NULL),(1519813,'Avg task CPU time (ms)','48771',NULL),(1519813,'Avg task GC time (ms)','1233',NULL),(1519813,'Avg task runtime (ms)','40227',NULL),(1519813,'Number of tasks','9',NULL),(1519813,'Task GC/CPU ratio','0.025281417235652335',NULL),(1519814,'Average task input size','455 MB',NULL),(1519814,'Average task runtime','40 sec',NULL),(1519814,'Max task runtime','1 min 22 sec',NULL),(1519814,'Min task runtime','23 sec',NULL),(1519814,'Number of tasks','9',NULL),(1519815,'Median task input size','477 MB',NULL),(1519815,'Median task runtime','36 sec',NULL),(1519815,'Median task speed','12 MB/s',NULL),(1519815,'Number of tasks','9',NULL),(1519815,'Total input size in MB','4096.536699295044',NULL),(1519816,'Avg output records per task','4240278',NULL),(1519816,'Avg spilled records per task','4851508',NULL),(1519816,'Number of tasks','9',NULL),(1519816,'Ratio of spilled records to output records','1.1441486724324312',NULL),(1519817,'Avg Physical Memory (MB)','1025',NULL),(1519817,'Avg task runtime','40 sec',NULL),(1519817,'Avg Virtual Memory (MB)','2917',NULL),(1519817,'Max Physical Memory (MB)','1040',NULL),(1519817,'Min Physical Memory (MB)','1003',NULL),(1519817,'Number of tasks','9',NULL),(1519817,'Requested Container Memory','1 GB',NULL),(1519818,'Group A','3 tasks @ 65 MB avg',NULL),(1519818,'Group B','2 tasks @ 65 MB avg',NULL),(1519818,'Number of tasks','5',NULL),(1519819,'Avg task CPU time (ms)','32478',NULL),(1519819,'Avg task GC time (ms)','1188',NULL),(1519819,'Avg task runtime (ms)','24655',NULL),(1519819,'Number of tasks','5',NULL),(1519819,'Task GC/CPU ratio','0.036578607057084794',NULL),(1519820,'Average task runtime','24 sec',NULL),(1519820,'Max task runtime','33 sec',NULL),(1519820,'Min task runtime','18 sec',NULL),(1519820,'Number of tasks','5',NULL),(1519821,'Avg Physical Memory (MB)','1140',NULL),(1519821,'Avg task runtime','24 sec',NULL),(1519821,'Avg Virtual Memory (MB)','2930',NULL),(1519821,'Max Physical Memory (MB)','1174',NULL),(1519821,'Min Physical Memory (MB)','1064',NULL),(1519821,'Number of tasks','5',NULL),(1519821,'Requested Container Memory','1 GB',NULL),(1519822,'Average code runtime','15 sec',NULL),(1519822,'Average shuffle time','6 sec (0.44x)',NULL),(1519822,'Average sort time','2 sec (0.18x)',NULL),(1519822,'Number of tasks','5',NULL),(1519824,'Group A','2 tasks @ 150 MB avg',NULL),(1519824,'Group B','4 tasks @ 426 MB avg',NULL),(1519824,'Number of tasks','6',NULL),(1519825,'Avg task CPU time (ms)','29083',NULL),(1519825,'Avg task GC time (ms)','614',NULL),(1519825,'Avg task runtime (ms)','22866',NULL),(1519825,'Number of tasks','6',NULL),(1519825,'Task GC/CPU ratio','0.02111198982223292',NULL),(1519826,'Average task input size','334 MB',NULL),(1519826,'Average task runtime','22 sec',NULL),(1519826,'Max task runtime','33 sec',NULL),(1519826,'Min task runtime','13 sec',NULL),(1519826,'Number of tasks','6',NULL),(1519827,'Median task input size','426 MB',NULL),(1519827,'Median task runtime','23 sec',NULL),(1519827,'Median task speed','14 MB/s',NULL),(1519827,'Number of tasks','6',NULL),(1519827,'Total input size in MB','2007.4812812805176',NULL),(1519828,'Avg output records per task','4923496',NULL),(1519828,'Avg spilled records per task','9506000',NULL),(1519828,'Number of tasks','6',NULL),(1519828,'Ratio of spilled records to output records','1.9307419971499926',NULL),(1519829,'Avg Physical Memory (MB)','1004',NULL),(1519829,'Avg task runtime','22 sec',NULL),(1519829,'Avg Virtual Memory (MB)','2918',NULL),(1519829,'Max Physical Memory (MB)','1021',NULL),(1519829,'Min Physical Memory (MB)','984',NULL),(1519829,'Number of tasks','6',NULL),(1519829,'Requested Container Memory','1 GB',NULL),(1519830,'Group A','1 tasks @ 84 MB avg',NULL),(1519830,'Group B','2 tasks @ 85 MB avg',NULL),(1519830,'Number of tasks','3',NULL),(1519831,'Avg task CPU time (ms)','43036',NULL),(1519831,'Avg task GC time (ms)','716',NULL),(1519831,'Avg task runtime (ms)','31501',NULL),(1519831,'Number of tasks','3',NULL),(1519831,'Task GC/CPU ratio','0.016637233943675063',NULL),(1519832,'Average task runtime','31 sec',NULL),(1519832,'Max task runtime','32 sec',NULL),(1519832,'Min task runtime','30 sec',NULL),(1519832,'Number of tasks','3',NULL),(1519833,'Avg Physical Memory (MB)','1293',NULL),(1519833,'Avg task runtime','31 sec',NULL),(1519833,'Avg Virtual Memory (MB)','2935',NULL),(1519833,'Max Physical Memory (MB)','1354',NULL),(1519833,'Min Physical Memory (MB)','1249',NULL),(1519833,'Number of tasks','3',NULL),(1519833,'Requested Container Memory','1 GB',NULL),(1519834,'Average code runtime','20 sec',NULL),(1519834,'Average shuffle time','6 sec (0.34x)',NULL),(1519834,'Average sort time','4 sec (0.22x)',NULL),(1519834,'Number of tasks','3',NULL),(1519836,'Group A','1 tasks @ 477 MB avg',NULL),(1519836,'Group B','2 tasks @ 496 MB avg',NULL),(1519836,'Number of tasks','3',NULL),(1519837,'Avg task CPU time (ms)','50053',NULL),(1519837,'Avg task GC time (ms)','1307',NULL),(1519837,'Avg task runtime (ms)','39273',NULL),(1519837,'Number of tasks','3',NULL),(1519837,'Task GC/CPU ratio','0.02611232093980381',NULL),(1519838,'Average task input size','490 MB',NULL),(1519838,'Average task runtime','39 sec',NULL),(1519838,'Max task runtime','50 sec',NULL),(1519838,'Min task runtime','25 sec',NULL),(1519838,'Number of tasks','3',NULL),(1519839,'Median task input size','495 MB',NULL),(1519839,'Median task runtime','41 sec',NULL),(1519839,'Median task speed','11 MB/s',NULL),(1519839,'Number of tasks','3',NULL),(1519839,'Total input size in MB','1470.1559200286865',NULL),(1519840,'Avg output records per task','8874376',NULL),(1519840,'Avg spilled records per task','635707',NULL),(1519840,'Number of tasks','3',NULL),(1519840,'Ratio of spilled records to output records','0.07163402650251867',NULL),(1519841,'Avg Physical Memory (MB)','1023',NULL),(1519841,'Avg task runtime','39 sec',NULL),(1519841,'Avg Virtual Memory (MB)','2917',NULL),(1519841,'Max Physical Memory (MB)','1031',NULL),(1519841,'Min Physical Memory (MB)','1018',NULL),(1519841,'Number of tasks','3',NULL),(1519841,'Requested Container Memory','1 GB',NULL),(1519842,'Group A','1 tasks @ 15 MB avg',NULL),(1519842,'Group B','1 tasks @ 15 MB avg',NULL),(1519842,'Number of tasks','2',NULL),(1519843,'Avg task CPU time (ms)','6380',NULL),(1519843,'Avg task GC time (ms)','56',NULL),(1519843,'Avg task runtime (ms)','7000',NULL),(1519843,'Number of tasks','2',NULL),(1519843,'Task GC/CPU ratio','0.00877742946708464',NULL),(1519844,'Average task runtime','7 sec',NULL),(1519844,'Max task runtime','7 sec',NULL),(1519844,'Min task runtime','6 sec',NULL),(1519844,'Number of tasks','2',NULL),(1519845,'Avg Physical Memory (MB)','520',NULL),(1519845,'Avg task runtime','7 sec',NULL),(1519845,'Avg Virtual Memory (MB)','2928',NULL),(1519845,'Max Physical Memory (MB)','522',NULL),(1519845,'Min Physical Memory (MB)','519',NULL),(1519845,'Number of tasks','2',NULL),(1519845,'Requested Container Memory','1 GB',NULL),(1519846,'Average code runtime','1 sec',NULL),(1519846,'Average shuffle time','4 sec (2.74x)',NULL),(1519846,'Average sort time',' (0.20x)',NULL),(1519846,'Number of tasks','2',NULL),(1519848,'Group A','1 tasks @ 403 MB avg',NULL),(1519848,'Group B','502 tasks @ 1 GB avg',NULL),(1519848,'Number of tasks','503',NULL),(1519849,'Avg task CPU time (ms)','124523',NULL),(1519849,'Avg task GC time (ms)','3644',NULL),(1519849,'Avg task runtime (ms)','106319',NULL),(1519849,'Number of tasks','503',NULL),(1519849,'Task GC/CPU ratio','0.029263670165350978',NULL),(1519850,'Average task input size','1 GB',NULL),(1519850,'Average task runtime','1 min 46 sec',NULL),(1519850,'Max task runtime','3 min 39 sec',NULL),(1519850,'Min task runtime','17 sec',NULL),(1519850,'Number of tasks','503',NULL),(1519851,'Median task input size','947 MB',NULL),(1519851,'Median task runtime','1 min 41 sec',NULL),(1519851,'Median task speed','10 MB/s',NULL),(1519851,'Number of tasks','503',NULL),(1519851,'Total input size in MB','525751.9063234329',NULL),(1519852,'Avg output records per task','750069',NULL),(1519852,'Avg spilled records per task','243854',NULL),(1519852,'Number of tasks','503',NULL),(1519852,'Ratio of spilled records to output records','0.32510910400171833',NULL),(1519853,'Avg Physical Memory (MB)','1045',NULL),(1519853,'Avg task runtime','1 min 46 sec',NULL),(1519853,'Avg Virtual Memory (MB)','2924',NULL),(1519853,'Max Physical Memory (MB)','1088',NULL),(1519853,'Min Physical Memory (MB)','814',NULL),(1519853,'Number of tasks','503',NULL),(1519853,'Requested Container Memory','1 GB',NULL),(1519854,'Group A','283 tasks @ 7 MB avg',NULL),(1519854,'Group B','269 tasks @ 7 MB avg',NULL),(1519854,'Number of tasks','552',NULL),(1519855,'Avg task CPU time (ms)','9793',NULL),(1519855,'Avg task GC time (ms)','100',NULL),(1519855,'Avg task runtime (ms)','42346',NULL),(1519855,'Number of tasks','552',NULL),(1519855,'Task GC/CPU ratio','0.010211375472276116',NULL),(1519856,'Average task runtime','42 sec',NULL),(1519856,'Max task runtime','1 min 43 sec',NULL),(1519856,'Min task runtime','7 sec',NULL),(1519856,'Number of tasks','552',NULL),(1519857,'Avg Physical Memory (MB)','577',NULL),(1519857,'Avg task runtime','42 sec',NULL),(1519857,'Avg Virtual Memory (MB)','2943',NULL),(1519857,'Max Physical Memory (MB)','623',NULL),(1519857,'Min Physical Memory (MB)','511',NULL),(1519857,'Number of tasks','552',NULL),(1519857,'Requested Container Memory','1 GB',NULL),(1519858,'Average code runtime','',NULL),(1519858,'Average shuffle time','41 sec (51.92x)',NULL),(1519858,'Average sort time',' (0.41x)',NULL),(1519858,'Number of tasks','552',NULL),(1519860,'Group A','1 tasks @ 280 MB avg',NULL),(1519860,'Group B','1 tasks @ 512 MB avg',NULL),(1519860,'Number of tasks','2',NULL),(1519861,'Avg task CPU time (ms)','102620',NULL),(1519861,'Avg task GC time (ms)','1747',NULL),(1519861,'Avg task runtime (ms)','70706',NULL),(1519861,'Number of tasks','2',NULL),(1519861,'Task GC/CPU ratio','0.017023971935295264',NULL),(1519862,'Average task input size','396 MB',NULL),(1519862,'Average task runtime','1 min 10 sec',NULL),(1519862,'Max task runtime','1 min 30 sec',NULL),(1519862,'Min task runtime','50 sec',NULL),(1519862,'Number of tasks','2',NULL),(1519863,'Median task input size','396 MB',NULL),(1519863,'Median task runtime','1 min 10 sec',NULL),(1519863,'Median task speed','5 MB/s',NULL),(1519863,'Number of tasks','2',NULL),(1519863,'Total input size in MB','792.506085395813',NULL),(1519864,'Avg output records per task','1245503',NULL),(1519864,'Avg spilled records per task','0',NULL),(1519864,'Number of tasks','2',NULL),(1519864,'Ratio of spilled records to output records','0.0',NULL),(1519865,'Avg Physical Memory (MB)','692',NULL),(1519865,'Avg task runtime','1 min 10 sec',NULL),(1519865,'Avg Virtual Memory (MB)','2929',NULL),(1519865,'Max Physical Memory (MB)','700',NULL),(1519865,'Min Physical Memory (MB)','684',NULL),(1519865,'Number of tasks','2',NULL),(1519865,'Requested Container Memory','1 GB',NULL),(1519866,'Group A','0 tasks @ 0 bytes avg',NULL),(1519866,'Group B','0 tasks @ 0 bytes avg',NULL),(1519866,'Number of tasks','0',NULL),(1519867,'Avg task CPU time (ms)','0',NULL),(1519867,'Avg task GC time (ms)','0',NULL),(1519867,'Avg task runtime (ms)','0',NULL),(1519867,'Number of tasks','0',NULL),(1519867,'Task GC/CPU ratio','0.0',NULL),(1519868,'Average task runtime','0 sec',NULL),(1519868,'Max task runtime','0 sec',NULL),(1519868,'Min task runtime','0 sec',NULL),(1519868,'Number of tasks','0',NULL),(1519869,'Avg Physical Memory (MB)','0',NULL),(1519869,'Avg task runtime','0 sec',NULL),(1519869,'Avg Virtual Memory (MB)','0',NULL),(1519869,'Max Physical Memory (MB)','0',NULL),(1519869,'Min Physical Memory (MB)','0',NULL),(1519869,'Number of tasks','0',NULL),(1519869,'Requested Container Memory','1 GB',NULL),(1519870,'Average code runtime','0 sec',NULL),(1519870,'Average shuffle time','0 sec ',NULL),(1519870,'Average sort time','0 sec ',NULL),(1519870,'Number of tasks','0',NULL),(1519872,'Group A','50 tasks @ 302 MB avg',NULL),(1519872,'Group B','100 tasks @ 483 MB avg',NULL),(1519872,'Number of tasks','150',NULL),(1519873,'Avg task CPU time (ms)','56132',NULL),(1519873,'Avg task GC time (ms)','306',NULL),(1519873,'Avg task runtime (ms)','48303',NULL),(1519873,'Number of tasks','150',NULL),(1519873,'Task GC/CPU ratio','0.005451435901090288',NULL),(1519874,'Average task input size','423 MB',NULL),(1519874,'Average task runtime','48 sec',NULL),(1519874,'Max task runtime','1 min 36 sec',NULL),(1519874,'Min task runtime','27 sec',NULL),(1519874,'Number of tasks','150',NULL),(1519875,'Median task input size','468 MB',NULL),(1519875,'Median task runtime','50 sec',NULL),(1519875,'Median task speed','9 MB/s',NULL),(1519875,'Number of tasks','150',NULL),(1519875,'Total input size in MB','63494.425676345825',NULL),(1519876,'Avg output records per task','1543736',NULL),(1519876,'Avg spilled records per task','0',NULL),(1519876,'Number of tasks','150',NULL),(1519876,'Ratio of spilled records to output records','0.0',NULL),(1519877,'Avg Physical Memory (MB)','607',NULL),(1519877,'Avg task runtime','48 sec',NULL),(1519877,'Avg Virtual Memory (MB)','2928',NULL),(1519877,'Max Physical Memory (MB)','645',NULL),(1519877,'Min Physical Memory (MB)','559',NULL),(1519877,'Number of tasks','150',NULL),(1519877,'Requested Container Memory','1 GB',NULL),(1519878,'Group A','0 tasks @ 0 bytes avg',NULL),(1519878,'Group B','0 tasks @ 0 bytes avg',NULL),(1519878,'Number of tasks','0',NULL),(1519879,'Avg task CPU time (ms)','0',NULL),(1519879,'Avg task GC time (ms)','0',NULL),(1519879,'Avg task runtime (ms)','0',NULL),(1519879,'Number of tasks','0',NULL),(1519879,'Task GC/CPU ratio','0.0',NULL),(1519880,'Average task runtime','0 sec',NULL),(1519880,'Max task runtime','0 sec',NULL),(1519880,'Min task runtime','0 sec',NULL),(1519880,'Number of tasks','0',NULL),(1519881,'Avg Physical Memory (MB)','0',NULL),(1519881,'Avg task runtime','0 sec',NULL),(1519881,'Avg Virtual Memory (MB)','0',NULL),(1519881,'Max Physical Memory (MB)','0',NULL),(1519881,'Min Physical Memory (MB)','0',NULL),(1519881,'Number of tasks','0',NULL),(1519881,'Requested Container Memory','1 GB',NULL),(1519882,'Average code runtime','0 sec',NULL),(1519882,'Average shuffle time','0 sec ',NULL),(1519882,'Average sort time','0 sec ',NULL),(1519882,'Number of tasks','0',NULL),(1519884,'Group A','1 tasks @ 264 MB avg',NULL),(1519884,'Group B','1 tasks @ 515 MB avg',NULL),(1519884,'Number of tasks','2',NULL),(1519885,'Avg task CPU time (ms)','66695',NULL),(1519885,'Avg task GC time (ms)','813',NULL),(1519885,'Avg task runtime (ms)','62888',NULL),(1519885,'Number of tasks','2',NULL),(1519885,'Task GC/CPU ratio','0.012189819326786117',NULL),(1519886,'Average task input size','390 MB',NULL),(1519886,'Average task runtime','1 min 2 sec',NULL),(1519886,'Max task runtime','1 min 28 sec',NULL),(1519886,'Min task runtime','37 sec',NULL),(1519886,'Number of tasks','2',NULL),(1519887,'Median task input size','390 MB',NULL),(1519887,'Median task runtime','1 min 2 sec',NULL),(1519887,'Median task speed','6 MB/s',NULL),(1519887,'Number of tasks','2',NULL),(1519887,'Total input size in MB','780.0830526351929',NULL),(1519888,'Avg output records per task','1242122',NULL),(1519888,'Avg spilled records per task','0',NULL),(1519888,'Number of tasks','2',NULL),(1519888,'Ratio of spilled records to output records','0.0',NULL),(1519889,'Avg Physical Memory (MB)','624',NULL),(1519889,'Avg task runtime','1 min 2 sec',NULL),(1519889,'Avg Virtual Memory (MB)','2966',NULL),(1519889,'Max Physical Memory (MB)','633',NULL),(1519889,'Min Physical Memory (MB)','614',NULL),(1519889,'Number of tasks','2',NULL),(1519889,'Requested Container Memory','1 GB',NULL),(1519890,'Group A','0 tasks @ 0 bytes avg',NULL),(1519890,'Group B','0 tasks @ 0 bytes avg',NULL),(1519890,'Number of tasks','0',NULL),(1519891,'Avg task CPU time (ms)','0',NULL),(1519891,'Avg task GC time (ms)','0',NULL),(1519891,'Avg task runtime (ms)','0',NULL),(1519891,'Number of tasks','0',NULL),(1519891,'Task GC/CPU ratio','0.0',NULL),(1519892,'Average task runtime','0 sec',NULL),(1519892,'Max task runtime','0 sec',NULL),(1519892,'Min task runtime','0 sec',NULL),(1519892,'Number of tasks','0',NULL),(1519893,'Avg Physical Memory (MB)','0',NULL),(1519893,'Avg task runtime','0 sec',NULL),(1519893,'Avg Virtual Memory (MB)','0',NULL),(1519893,'Max Physical Memory (MB)','0',NULL),(1519893,'Min Physical Memory (MB)','0',NULL),(1519893,'Number of tasks','0',NULL),(1519893,'Requested Container Memory','1 GB',NULL),(1519894,'Average code runtime','0 sec',NULL),(1519894,'Average shuffle time','0 sec ',NULL),(1519894,'Average sort time','0 sec ',NULL),(1519894,'Number of tasks','0',NULL),(1519896,'Group A','306 tasks @ 290 MB avg',NULL),(1519896,'Group B','203 tasks @ 454 MB avg',NULL),(1519896,'Number of tasks','509',NULL),(1519897,'Avg task CPU time (ms)','65394',NULL),(1519897,'Avg task GC time (ms)','909',NULL),(1519897,'Avg task runtime (ms)','53214',NULL),(1519897,'Number of tasks','509',NULL),(1519897,'Task GC/CPU ratio','0.013900357830993668',NULL),(1519898,'Average task input size','355 MB',NULL),(1519898,'Average task runtime','53 sec',NULL),(1519898,'Max task runtime','1 min 55 sec',NULL),(1519898,'Min task runtime','32 sec',NULL),(1519898,'Number of tasks','509',NULL),(1519899,'Median task input size','320 MB',NULL),(1519899,'Median task runtime','50 sec',NULL),(1519899,'Median task speed','7 MB/s',NULL),(1519899,'Number of tasks','509',NULL),(1519899,'Total input size in MB','181091.6296787262',NULL),(1519900,'Avg output records per task','308768',NULL),(1519900,'Avg spilled records per task','0',NULL),(1519900,'Number of tasks','509',NULL),(1519900,'Ratio of spilled records to output records','0.0',NULL),(1519901,'Avg Physical Memory (MB)','604',NULL),(1519901,'Avg task runtime','53 sec',NULL),(1519901,'Avg Virtual Memory (MB)','2924',NULL),(1519901,'Max Physical Memory (MB)','698',NULL),(1519901,'Min Physical Memory (MB)','569',NULL),(1519901,'Number of tasks','509',NULL),(1519901,'Requested Container Memory','1 GB',NULL),(1519902,'Group A','0 tasks @ 0 bytes avg',NULL),(1519902,'Group B','0 tasks @ 0 bytes avg',NULL),(1519902,'Number of tasks','0',NULL),(1519903,'Avg task CPU time (ms)','0',NULL),(1519903,'Avg task GC time (ms)','0',NULL),(1519903,'Avg task runtime (ms)','0',NULL),(1519903,'Number of tasks','0',NULL),(1519903,'Task GC/CPU ratio','0.0',NULL),(1519904,'Average task runtime','0 sec',NULL),(1519904,'Max task runtime','0 sec',NULL),(1519904,'Min task runtime','0 sec',NULL),(1519904,'Number of tasks','0',NULL),(1519905,'Avg Physical Memory (MB)','0',NULL),(1519905,'Avg task runtime','0 sec',NULL),(1519905,'Avg Virtual Memory (MB)','0',NULL),(1519905,'Max Physical Memory (MB)','0',NULL),(1519905,'Min Physical Memory (MB)','0',NULL),(1519905,'Number of tasks','0',NULL),(1519905,'Requested Container Memory','1 GB',NULL),(1519906,'Average code runtime','0 sec',NULL),(1519906,'Average shuffle time','0 sec ',NULL),(1519906,'Average sort time','0 sec ',NULL),(1519906,'Number of tasks','0',NULL),(1519908,'Group A','1 tasks @ 324 MB avg',NULL),(1519908,'Group B','1 tasks @ 507 MB avg',NULL),(1519908,'Number of tasks','2',NULL),(1519909,'Avg task CPU time (ms)','129925',NULL),(1519909,'Avg task GC time (ms)','3734',NULL),(1519909,'Avg task runtime (ms)','89082',NULL),(1519909,'Number of tasks','2',NULL),(1519909,'Task GC/CPU ratio','0.028739657494708484',NULL),(1519910,'Average task input size','415 MB',NULL),(1519910,'Average task runtime','1 min 29 sec',NULL),(1519910,'Max task runtime','1 min 59 sec',NULL),(1519910,'Min task runtime','58 sec',NULL),(1519910,'Number of tasks','2',NULL),(1519911,'Median task input size','415 MB',NULL),(1519911,'Median task runtime','1 min 29 sec',NULL),(1519911,'Median task speed','4 MB/s',NULL),(1519911,'Number of tasks','2',NULL),(1519911,'Total input size in MB','831.6270980834961',NULL),(1519912,'Avg output records per task','1211133',NULL),(1519912,'Avg spilled records per task','0',NULL),(1519912,'Number of tasks','2',NULL),(1519912,'Ratio of spilled records to output records','0.0',NULL),(1519913,'Avg Physical Memory (MB)','761',NULL),(1519913,'Avg task runtime','1 min 29 sec',NULL),(1519913,'Avg Virtual Memory (MB)','2926',NULL),(1519913,'Max Physical Memory (MB)','795',NULL),(1519913,'Min Physical Memory (MB)','726',NULL),(1519913,'Number of tasks','2',NULL),(1519913,'Requested Container Memory','1 GB',NULL),(1519914,'Group A','0 tasks @ 0 bytes avg',NULL),(1519914,'Group B','0 tasks @ 0 bytes avg',NULL),(1519914,'Number of tasks','0',NULL),(1519915,'Avg task CPU time (ms)','0',NULL),(1519915,'Avg task GC time (ms)','0',NULL),(1519915,'Avg task runtime (ms)','0',NULL),(1519915,'Number of tasks','0',NULL),(1519915,'Task GC/CPU ratio','0.0',NULL),(1519916,'Average task runtime','0 sec',NULL),(1519916,'Max task runtime','0 sec',NULL),(1519916,'Min task runtime','0 sec',NULL),(1519916,'Number of tasks','0',NULL),(1519917,'Avg Physical Memory (MB)','0',NULL),(1519917,'Avg task runtime','0 sec',NULL),(1519917,'Avg Virtual Memory (MB)','0',NULL),(1519917,'Max Physical Memory (MB)','0',NULL),(1519917,'Min Physical Memory (MB)','0',NULL),(1519917,'Number of tasks','0',NULL),(1519917,'Requested Container Memory','1 GB',NULL),(1519918,'Average code runtime','0 sec',NULL),(1519918,'Average shuffle time','0 sec ',NULL),(1519918,'Average sort time','0 sec ',NULL),(1519918,'Number of tasks','0',NULL),(1519920,'Group A','297 tasks @ 293 MB avg',NULL),(1519920,'Group B','198 tasks @ 448 MB avg',NULL),(1519920,'Number of tasks','495',NULL),(1519921,'Avg task CPU time (ms)','65955',NULL),(1519921,'Avg task GC time (ms)','971',NULL),(1519921,'Avg task runtime (ms)','53647',NULL),(1519921,'Number of tasks','495',NULL),(1519921,'Task GC/CPU ratio','0.014722159047835645',NULL),(1519922,'Average task input size','355 MB',NULL),(1519922,'Average task runtime','53 sec',NULL),(1519922,'Max task runtime','1 min 46 sec',NULL),(1519922,'Min task runtime','31 sec',NULL),(1519922,'Number of tasks','495',NULL),(1519923,'Median task input size','321 MB',NULL),(1519923,'Median task runtime','50 sec',NULL),(1519923,'Median task speed','6 MB/s',NULL),(1519923,'Number of tasks','495',NULL),(1519923,'Total input size in MB','175900.1569595337',NULL),(1519924,'Avg output records per task','304294',NULL),(1519924,'Avg spilled records per task','0',NULL),(1519924,'Number of tasks','495',NULL),(1519924,'Ratio of spilled records to output records','0.0',NULL),(1519925,'Avg Physical Memory (MB)','602',NULL),(1519925,'Avg task runtime','53 sec',NULL),(1519925,'Avg Virtual Memory (MB)','2927',NULL),(1519925,'Max Physical Memory (MB)','645',NULL),(1519925,'Min Physical Memory (MB)','569',NULL),(1519925,'Number of tasks','495',NULL),(1519925,'Requested Container Memory','1 GB',NULL),(1519926,'Group A','0 tasks @ 0 bytes avg',NULL),(1519926,'Group B','0 tasks @ 0 bytes avg',NULL),(1519926,'Number of tasks','0',NULL),(1519927,'Avg task CPU time (ms)','0',NULL),(1519927,'Avg task GC time (ms)','0',NULL),(1519927,'Avg task runtime (ms)','0',NULL),(1519927,'Number of tasks','0',NULL),(1519927,'Task GC/CPU ratio','0.0',NULL),(1519928,'Average task runtime','0 sec',NULL),(1519928,'Max task runtime','0 sec',NULL),(1519928,'Min task runtime','0 sec',NULL),(1519928,'Number of tasks','0',NULL),(1519929,'Avg Physical Memory (MB)','0',NULL),(1519929,'Avg task runtime','0 sec',NULL),(1519929,'Avg Virtual Memory (MB)','0',NULL),(1519929,'Max Physical Memory (MB)','0',NULL),(1519929,'Min Physical Memory (MB)','0',NULL),(1519929,'Number of tasks','0',NULL),(1519929,'Requested Container Memory','1 GB',NULL),(1519930,'Average code runtime','0 sec',NULL),(1519930,'Average shuffle time','0 sec ',NULL),(1519930,'Average sort time','0 sec ',NULL),(1519930,'Number of tasks','0',NULL),(1519932,'Group A','948 tasks @ 779 MB avg',NULL),(1519932,'Group B','835 tasks @ 1 GB avg',NULL),(1519932,'Number of tasks','1783',NULL),(1519933,'Avg task CPU time (ms)','71503',NULL),(1519933,'Avg task GC time (ms)','2697',NULL),(1519933,'Avg task runtime (ms)','47954',NULL),(1519933,'Number of tasks','1783',NULL),(1519933,'Task GC/CPU ratio','0.03771869711760346',NULL),(1519934,'Average task input size','941 MB',NULL),(1519934,'Average task runtime','47 sec',NULL),(1519934,'Max task runtime','1 min 54 sec',NULL),(1519934,'Min task runtime','20 sec',NULL),(1519934,'Number of tasks','1783',NULL),(1519935,'Median task input size','916 MB',NULL),(1519935,'Median task runtime','44 sec',NULL),(1519935,'Median task speed','21 MB/s',NULL),(1519935,'Number of tasks','1783',NULL),(1519935,'Total input size in MB','1678477.7600755692',NULL),(1519936,'Avg output records per task','3484134',NULL),(1519936,'Avg spilled records per task','6968269',NULL),(1519936,'Number of tasks','1783',NULL),(1519936,'Ratio of spilled records to output records','2.0',NULL),(1519937,'Avg Physical Memory (MB)','1055',NULL),(1519937,'Avg task runtime','47 sec',NULL),(1519937,'Avg Virtual Memory (MB)','2937',NULL),(1519937,'Max Physical Memory (MB)','1306',NULL),(1519937,'Min Physical Memory (MB)','778',NULL),(1519937,'Number of tasks','1783',NULL),(1519937,'Requested Container Memory','1 GB',NULL),(1519938,'Group A','520 tasks @ 500 MB avg',NULL),(1519938,'Group B','479 tasks @ 501 MB avg',NULL),(1519938,'Number of tasks','999',NULL),(1519939,'Avg task CPU time (ms)','69540',NULL),(1519939,'Avg task GC time (ms)','1728',NULL),(1519939,'Avg task runtime (ms)','74531',NULL),(1519939,'Number of tasks','999',NULL),(1519939,'Task GC/CPU ratio','0.024849007765314925',NULL),(1519940,'Average task runtime','1 min 14 sec',NULL),(1519940,'Max task runtime','3 min 1 sec',NULL),(1519940,'Min task runtime','39 sec',NULL),(1519940,'Number of tasks','999',NULL),(1519941,'Avg Physical Memory (MB)','1299',NULL),(1519941,'Avg task runtime','1 min 14 sec',NULL),(1519941,'Avg Virtual Memory (MB)','2962',NULL),(1519941,'Max Physical Memory (MB)','1479',NULL),(1519941,'Min Physical Memory (MB)','1088',NULL),(1519941,'Number of tasks','999',NULL),(1519941,'Requested Container Memory','1 GB',NULL),(1519942,'Average code runtime','29 sec',NULL),(1519942,'Average shuffle time','42 sec (1.45x)',NULL),(1519942,'Average sort time','2 sec (0.07x)',NULL),(1519942,'Number of tasks','999',NULL),(1519944,'Group A','889 tasks @ 310 MB avg',NULL),(1519944,'Group B','735 tasks @ 450 MB avg',NULL),(1519944,'Number of tasks','1624',NULL),(1519945,'Avg task CPU time (ms)','67810',NULL),(1519945,'Avg task GC time (ms)','1125',NULL),(1519945,'Avg task runtime (ms)','54926',NULL),(1519945,'Number of tasks','1624',NULL),(1519945,'Task GC/CPU ratio','0.016590473381507154',NULL),(1519946,'Average task input size','373 MB',NULL),(1519946,'Average task runtime','54 sec',NULL),(1519946,'Max task runtime','1 min 59 sec',NULL),(1519946,'Min task runtime','30 sec',NULL),(1519946,'Number of tasks','1624',NULL),(1519947,'Median task input size','361 MB',NULL),(1519947,'Median task runtime','52 sec',NULL),(1519947,'Median task speed','7 MB/s',NULL),(1519947,'Number of tasks','1624',NULL),(1519947,'Total input size in MB','606587.9218597412',NULL),(1519948,'Avg output records per task','337012',NULL),(1519948,'Avg spilled records per task','0',NULL),(1519948,'Number of tasks','1624',NULL),(1519948,'Ratio of spilled records to output records','0.0',NULL),(1519949,'Avg Physical Memory (MB)','607',NULL),(1519949,'Avg task runtime','54 sec',NULL),(1519949,'Avg Virtual Memory (MB)','2927',NULL),(1519949,'Max Physical Memory (MB)','674',NULL),(1519949,'Min Physical Memory (MB)','568',NULL),(1519949,'Number of tasks','1624',NULL),(1519949,'Requested Container Memory','1 GB',NULL),(1519950,'Group A','0 tasks @ 0 bytes avg',NULL),(1519950,'Group B','0 tasks @ 0 bytes avg',NULL),(1519950,'Number of tasks','0',NULL),(1519951,'Avg task CPU time (ms)','0',NULL),(1519951,'Avg task GC time (ms)','0',NULL),(1519951,'Avg task runtime (ms)','0',NULL),(1519951,'Number of tasks','0',NULL),(1519951,'Task GC/CPU ratio','0.0',NULL),(1519952,'Average task runtime','0 sec',NULL),(1519952,'Max task runtime','0 sec',NULL),(1519952,'Min task runtime','0 sec',NULL),(1519952,'Number of tasks','0',NULL),(1519953,'Avg Physical Memory (MB)','0',NULL),(1519953,'Avg task runtime','0 sec',NULL),(1519953,'Avg Virtual Memory (MB)','0',NULL),(1519953,'Max Physical Memory (MB)','0',NULL),(1519953,'Min Physical Memory (MB)','0',NULL),(1519953,'Number of tasks','0',NULL),(1519953,'Requested Container Memory','1 GB',NULL),(1519954,'Average code runtime','0 sec',NULL),(1519954,'Average shuffle time','0 sec ',NULL),(1519954,'Average sort time','0 sec ',NULL),(1519954,'Number of tasks','0',NULL),(1519956,'Group A','1 tasks @ 452 MB avg',NULL),(1519956,'Group B','497 tasks @ 1 GB avg',NULL),(1519956,'Number of tasks','498',NULL),(1519957,'Avg task CPU time (ms)','104899',NULL),(1519957,'Avg task GC time (ms)','1484',NULL),(1519957,'Avg task runtime (ms)','86266',NULL),(1519957,'Number of tasks','498',NULL),(1519957,'Task GC/CPU ratio','0.014146941343578108',NULL),(1519958,'Average task input size','1 GB',NULL),(1519958,'Average task runtime','1 min 26 sec',NULL),(1519958,'Max task runtime','2 min 46 sec',NULL),(1519958,'Min task runtime','16 sec',NULL),(1519958,'Number of tasks','498',NULL),(1519959,'Median task input size','949 MB',NULL),(1519959,'Median task runtime','1 min 21 sec',NULL),(1519959,'Median task speed','12 MB/s',NULL),(1519959,'Number of tasks','498',NULL),(1519959,'Total input size in MB','523536.60893917084',NULL),(1519960,'Avg output records per task','754187',NULL),(1519960,'Avg spilled records per task','244061',NULL),(1519960,'Number of tasks','498',NULL),(1519960,'Ratio of spilled records to output records','0.3236081854476542',NULL),(1519961,'Avg Physical Memory (MB)','891',NULL),(1519961,'Avg task runtime','1 min 26 sec',NULL),(1519961,'Avg Virtual Memory (MB)','2926',NULL),(1519961,'Max Physical Memory (MB)','965',NULL),(1519961,'Min Physical Memory (MB)','865',NULL),(1519961,'Number of tasks','498',NULL),(1519961,'Requested Container Memory','1 GB',NULL),(1519962,'Group A','276 tasks @ 7 MB avg',NULL),(1519962,'Group B','273 tasks @ 7 MB avg',NULL),(1519962,'Number of tasks','549',NULL),(1519963,'Avg task CPU time (ms)','9068',NULL),(1519963,'Avg task GC time (ms)','88',NULL),(1519963,'Avg task runtime (ms)','8130',NULL),(1519963,'Number of tasks','549',NULL),(1519963,'Task GC/CPU ratio','0.009704455227172474',NULL),(1519964,'Average task runtime','8 sec',NULL),(1519964,'Max task runtime','20 sec',NULL),(1519964,'Min task runtime','3 sec',NULL),(1519964,'Number of tasks','549',NULL),(1519965,'Avg Physical Memory (MB)','604',NULL),(1519965,'Avg task runtime','8 sec',NULL),(1519965,'Avg Virtual Memory (MB)','3285',NULL),(1519965,'Max Physical Memory (MB)','683',NULL),(1519965,'Min Physical Memory (MB)','507',NULL),(1519965,'Number of tasks','549',NULL),(1519965,'Requested Container Memory','1 GB',NULL),(1519966,'Average code runtime','',NULL),(1519966,'Average shuffle time','7 sec (16.20x)',NULL),(1519966,'Average sort time',' (0.71x)',NULL),(1519966,'Number of tasks','549',NULL),(1519968,'Group A','1 tasks @ 150 MB avg',NULL),(1519968,'Group B','1 tasks @ 155 MB avg',NULL),(1519968,'Number of tasks','2',NULL),(1519969,'Avg task CPU time (ms)','19210',NULL),(1519969,'Avg task GC time (ms)','164',NULL),(1519969,'Avg task runtime (ms)','16503',NULL),(1519969,'Number of tasks','2',NULL),(1519969,'Task GC/CPU ratio','0.008537220197813639',NULL),(1519970,'Average task input size','153 MB',NULL),(1519970,'Average task runtime','16 sec',NULL),(1519970,'Max task runtime','16 sec',NULL),(1519970,'Min task runtime','16 sec',NULL),(1519970,'Number of tasks','2',NULL),(1519971,'Median task input size','153 MB',NULL),(1519971,'Median task runtime','16 sec',NULL),(1519971,'Median task speed','9 MB/s',NULL),(1519971,'Number of tasks','2',NULL),(1519971,'Total input size in MB','306.22699069976807',NULL),(1519972,'Avg output records per task','2901809',NULL),(1519972,'Avg spilled records per task','5803619',NULL),(1519972,'Number of tasks','2',NULL),(1519972,'Ratio of spilled records to output records','2.0',NULL),(1519973,'Avg Physical Memory (MB)','848',NULL),(1519973,'Avg task runtime','16 sec',NULL),(1519973,'Avg Virtual Memory (MB)','2923',NULL),(1519973,'Max Physical Memory (MB)','852',NULL),(1519973,'Min Physical Memory (MB)','845',NULL),(1519973,'Number of tasks','2',NULL),(1519973,'Requested Container Memory','1 GB',NULL),(1519974,'Group A','0 tasks @ 0 bytes avg',NULL),(1519974,'Group B','1 tasks @ 188 MB avg',NULL),(1519974,'Number of tasks','1',NULL),(1519975,'Avg task CPU time (ms)','28760',NULL),(1519975,'Avg task GC time (ms)','174',NULL),(1519975,'Avg task runtime (ms)','26051',NULL),(1519975,'Number of tasks','1',NULL),(1519975,'Task GC/CPU ratio','0.0060500695410292075',NULL),(1519976,'Average task runtime','26 sec',NULL),(1519976,'Max task runtime','26 sec',NULL),(1519976,'Min task runtime','26 sec',NULL),(1519976,'Number of tasks','1',NULL),(1519977,'Avg Physical Memory (MB)','689',NULL),(1519977,'Avg task runtime','26 sec',NULL),(1519977,'Avg Virtual Memory (MB)','3255',NULL),(1519977,'Max Physical Memory (MB)','689',NULL),(1519977,'Min Physical Memory (MB)','689',NULL),(1519977,'Number of tasks','1',NULL),(1519977,'Requested Container Memory','1 GB',NULL),(1519978,'Average code runtime','16 sec',NULL),(1519978,'Average shuffle time','5 sec (0.33x)',NULL),(1519978,'Average sort time','3 sec (0.21x)',NULL),(1519978,'Number of tasks','1',NULL),(1519980,'Group A','0 tasks @ 0 bytes avg',NULL),(1519980,'Group B','1 tasks @ 163 MB avg',NULL),(1519980,'Number of tasks','1',NULL),(1519981,'Avg task CPU time (ms)','53920',NULL),(1519981,'Avg task GC time (ms)','631',NULL),(1519981,'Avg task runtime (ms)','40213',NULL),(1519981,'Number of tasks','1',NULL),(1519981,'Task GC/CPU ratio','0.011702522255192879',NULL),(1519982,'Average task input size','163 MB',NULL),(1519982,'Average task runtime','40 sec',NULL),(1519982,'Max task runtime','40 sec',NULL),(1519982,'Min task runtime','40 sec',NULL),(1519982,'Number of tasks','1',NULL),(1519983,'Median task input size','163 MB',NULL),(1519983,'Median task runtime','40 sec',NULL),(1519983,'Median task speed','4 MB/s',NULL),(1519983,'Number of tasks','1',NULL),(1519983,'Total input size in MB','163.50396060943604',NULL),(1519984,'Avg output records per task','2925684',NULL),(1519984,'Avg spilled records per task','5851368',NULL),(1519984,'Number of tasks','1',NULL),(1519984,'Ratio of spilled records to output records','2.0',NULL),(1519985,'Avg Physical Memory (MB)','856',NULL),(1519985,'Avg task runtime','40 sec',NULL),(1519985,'Avg Virtual Memory (MB)','2918',NULL),(1519985,'Max Physical Memory (MB)','856',NULL),(1519985,'Min Physical Memory (MB)','856',NULL),(1519985,'Number of tasks','1',NULL),(1519985,'Requested Container Memory','1 GB',NULL),(1519986,'Group A','0 tasks @ 0 bytes avg',NULL),(1519986,'Group B','1 tasks @ 97 MB avg',NULL),(1519986,'Number of tasks','1',NULL),(1519987,'Avg task CPU time (ms)','21470',NULL),(1519987,'Avg task GC time (ms)','304',NULL),(1519987,'Avg task runtime (ms)','21594',NULL),(1519987,'Number of tasks','1',NULL),(1519987,'Task GC/CPU ratio','0.01415929203539823',NULL),(1519988,'Average task runtime','21 sec',NULL),(1519988,'Max task runtime','21 sec',NULL),(1519988,'Min task runtime','21 sec',NULL),(1519988,'Number of tasks','1',NULL),(1519989,'Avg Physical Memory (MB)','687',NULL),(1519989,'Avg task runtime','21 sec',NULL),(1519989,'Avg Virtual Memory (MB)','3263',NULL),(1519989,'Max Physical Memory (MB)','687',NULL),(1519989,'Min Physical Memory (MB)','687',NULL),(1519989,'Number of tasks','1',NULL),(1519989,'Requested Container Memory','1 GB',NULL),(1519990,'Average code runtime','12 sec',NULL),(1519990,'Average shuffle time','7 sec (0.60x)',NULL),(1519990,'Average sort time','1 sec (0.15x)',NULL),(1519990,'Number of tasks','1',NULL),(1519992,'Group A','0 tasks @ 0 bytes avg',NULL),(1519992,'Group B','1 tasks @ 158 MB avg',NULL),(1519992,'Number of tasks','1',NULL),(1519993,'Avg task CPU time (ms)','22650',NULL),(1519993,'Avg task GC time (ms)','231',NULL),(1519993,'Avg task runtime (ms)','19696',NULL),(1519993,'Number of tasks','1',NULL),(1519993,'Task GC/CPU ratio','0.010198675496688741',NULL),(1519994,'Average task input size','158 MB',NULL),(1519994,'Average task runtime','19 sec',NULL),(1519994,'Max task runtime','19 sec',NULL),(1519994,'Min task runtime','19 sec',NULL),(1519994,'Number of tasks','1',NULL),(1519995,'Median task input size','158 MB',NULL),(1519995,'Median task runtime','19 sec',NULL),(1519995,'Median task speed','8 MB/s',NULL),(1519995,'Number of tasks','1',NULL),(1519995,'Total input size in MB','158.48675537109375',NULL),(1519996,'Avg output records per task','2925684',NULL),(1519996,'Avg spilled records per task','5851368',NULL),(1519996,'Number of tasks','1',NULL),(1519996,'Ratio of spilled records to output records','2.0',NULL),(1519997,'Avg Physical Memory (MB)','827',NULL),(1519997,'Avg task runtime','19 sec',NULL),(1519997,'Avg Virtual Memory (MB)','2924',NULL),(1519997,'Max Physical Memory (MB)','827',NULL),(1519997,'Min Physical Memory (MB)','827',NULL),(1519997,'Number of tasks','1',NULL),(1519997,'Requested Container Memory','1 GB',NULL),(1519998,'Group A','0 tasks @ 0 bytes avg',NULL),(1519998,'Group B','1 tasks @ 93 MB avg',NULL),(1519998,'Number of tasks','1',NULL),(1519999,'Avg task CPU time (ms)','15320',NULL),(1519999,'Avg task GC time (ms)','99',NULL),(1519999,'Avg task runtime (ms)','15367',NULL),(1519999,'Number of tasks','1',NULL),(1519999,'Task GC/CPU ratio','0.006462140992167102',NULL),(1520000,'Average task runtime','15 sec',NULL),(1520000,'Max task runtime','15 sec',NULL),(1520000,'Min task runtime','15 sec',NULL),(1520000,'Number of tasks','1',NULL),(1520001,'Avg Physical Memory (MB)','659',NULL),(1520001,'Avg task runtime','15 sec',NULL),(1520001,'Avg Virtual Memory (MB)','3270',NULL),(1520001,'Max Physical Memory (MB)','659',NULL),(1520001,'Min Physical Memory (MB)','659',NULL),(1520001,'Number of tasks','1',NULL),(1520001,'Requested Container Memory','1 GB',NULL),(1520002,'Average code runtime','7 sec',NULL),(1520002,'Average shuffle time','5 sec (0.71x)',NULL),(1520002,'Average sort time','1 sec (0.22x)',NULL),(1520002,'Number of tasks','1',NULL),(1520004,'Group A','2 tasks @ 176 MB avg',NULL),(1520004,'Group B','31 tasks @ 509 MB avg',NULL),(1520004,'Number of tasks','33',NULL),(1520005,'Avg task CPU time (ms)','46196',NULL),(1520005,'Avg task GC time (ms)','439',NULL),(1520005,'Avg task runtime (ms)','33085',NULL),(1520005,'Number of tasks','33',NULL),(1520005,'Task GC/CPU ratio','0.00950298727162525',NULL),(1520006,'Average task input size','489 MB',NULL),(1520006,'Average task runtime','33 sec',NULL),(1520006,'Max task runtime','49 sec',NULL),(1520006,'Min task runtime','11 sec',NULL),(1520006,'Number of tasks','33',NULL),(1520007,'Median task input size','507 MB',NULL),(1520007,'Median task runtime','32 sec',NULL),(1520007,'Median task speed','15 MB/s',NULL),(1520007,'Number of tasks','33',NULL),(1520007,'Total input size in MB','16157.14318561554',NULL),(1520008,'Avg output records per task','5373356',NULL),(1520008,'Avg spilled records per task','10746712',NULL),(1520008,'Number of tasks','33',NULL),(1520008,'Ratio of spilled records to output records','2.0',NULL),(1520009,'Avg Physical Memory (MB)','844',NULL),(1520009,'Avg task runtime','33 sec',NULL),(1520009,'Avg Virtual Memory (MB)','2923',NULL),(1520009,'Max Physical Memory (MB)','861',NULL),(1520009,'Min Physical Memory (MB)','816',NULL),(1520009,'Number of tasks','33',NULL),(1520009,'Requested Container Memory','1 GB',NULL),(1520010,'Group A','16 tasks @ 134 MB avg',NULL),(1520010,'Group B','1 tasks @ 993 MB avg',NULL),(1520010,'Number of tasks','17',NULL),(1520011,'Avg task CPU time (ms)','42275',NULL),(1520011,'Avg task GC time (ms)','727',NULL),(1520011,'Avg task runtime (ms)','32433',NULL),(1520011,'Number of tasks','17',NULL),(1520011,'Task GC/CPU ratio','0.01719692489651094',NULL),(1520012,'Average task runtime','32 sec',NULL),(1520012,'Max task runtime','1 min 35 sec',NULL),(1520012,'Min task runtime','23 sec',NULL),(1520012,'Number of tasks','17',NULL),(1520013,'Avg Physical Memory (MB)','1592',NULL),(1520013,'Avg task runtime','32 sec',NULL),(1520013,'Avg Virtual Memory (MB)','3271',NULL),(1520013,'Max Physical Memory (MB)','1710',NULL),(1520013,'Min Physical Memory (MB)','1407',NULL),(1520013,'Number of tasks','17',NULL),(1520013,'Requested Container Memory','1 GB',NULL),(1520014,'Average code runtime','19 sec',NULL),(1520014,'Average shuffle time','9 sec (0.50x)',NULL),(1520014,'Average sort time','2 sec (0.14x)',NULL),(1520014,'Number of tasks','17',NULL),(1520016,'Group A','1 tasks @ 260 MB avg',NULL),(1520016,'Group B','8 tasks @ 478 MB avg',NULL),(1520016,'Number of tasks','9',NULL),(1520017,'Avg task CPU time (ms)','41783',NULL),(1520017,'Avg task GC time (ms)','365',NULL),(1520017,'Avg task runtime (ms)','30677',NULL),(1520017,'Number of tasks','9',NULL),(1520017,'Task GC/CPU ratio','0.008735610176387526',NULL),(1520018,'Average task input size','454 MB',NULL),(1520018,'Average task runtime','30 sec',NULL),(1520018,'Max task runtime','35 sec',NULL),(1520018,'Min task runtime','20 sec',NULL),(1520018,'Number of tasks','9',NULL),(1520019,'Median task input size','476 MB',NULL),(1520019,'Median task runtime','31 sec',NULL),(1520019,'Median task speed','15 MB/s',NULL),(1520019,'Number of tasks','9',NULL),(1520019,'Total input size in MB','4087.891945838928',NULL),(1520020,'Avg output records per task','4231253',NULL),(1520020,'Avg spilled records per task','4840442',NULL),(1520020,'Number of tasks','9',NULL),(1520020,'Ratio of spilled records to output records','1.1439735330352832',NULL),(1520021,'Avg Physical Memory (MB)','855',NULL),(1520021,'Avg task runtime','30 sec',NULL),(1520021,'Avg Virtual Memory (MB)','2919',NULL),(1520021,'Max Physical Memory (MB)','874',NULL),(1520021,'Min Physical Memory (MB)','841',NULL),(1520021,'Number of tasks','9',NULL),(1520021,'Requested Container Memory','1 GB',NULL),(1520022,'Group A','2 tasks @ 65 MB avg',NULL),(1520022,'Group B','3 tasks @ 65 MB avg',NULL),(1520022,'Number of tasks','5',NULL),(1520023,'Avg task CPU time (ms)','26902',NULL),(1520023,'Avg task GC time (ms)','509',NULL),(1520023,'Avg task runtime (ms)','19608',NULL),(1520023,'Number of tasks','5',NULL),(1520023,'Task GC/CPU ratio','0.01892052635491785',NULL),(1520024,'Average task runtime','19 sec',NULL),(1520024,'Max task runtime','23 sec',NULL),(1520024,'Min task runtime','17 sec',NULL),(1520024,'Number of tasks','5',NULL),(1520025,'Avg Physical Memory (MB)','1041',NULL),(1520025,'Avg task runtime','19 sec',NULL),(1520025,'Avg Virtual Memory (MB)','3291',NULL),(1520025,'Max Physical Memory (MB)','1125',NULL),(1520025,'Min Physical Memory (MB)','981',NULL),(1520025,'Number of tasks','5',NULL),(1520025,'Requested Container Memory','1 GB',NULL),(1520026,'Average code runtime','11 sec',NULL),(1520026,'Average shuffle time','5 sec (0.53x)',NULL),(1520026,'Average sort time','2 sec (0.21x)',NULL),(1520026,'Number of tasks','5',NULL),(1520028,'Group A','3 tasks @ 260 MB avg',NULL),(1520028,'Group B','24 tasks @ 478 MB avg',NULL),(1520028,'Number of tasks','27',NULL),(1520029,'Avg task CPU time (ms)','34687',NULL),(1520029,'Avg task GC time (ms)','428',NULL),(1520029,'Avg task runtime (ms)','25760',NULL),(1520029,'Number of tasks','27',NULL),(1520029,'Task GC/CPU ratio','0.012338916596995993',NULL),(1520030,'Average task input size','454 MB',NULL),(1520030,'Average task runtime','25 sec',NULL),(1520030,'Max task runtime','36 sec',NULL),(1520030,'Min task runtime','13 sec',NULL),(1520030,'Number of tasks','27',NULL),(1520031,'Median task input size','476 MB',NULL),(1520031,'Median task runtime','24 sec',NULL),(1520031,'Median task speed','18 MB/s',NULL),(1520031,'Number of tasks','27',NULL),(1520031,'Total input size in MB','12263.675837516785',NULL),(1520032,'Avg output records per task','4304775',NULL),(1520032,'Avg spilled records per task','2127942',NULL),(1520032,'Number of tasks','27',NULL),(1520032,'Ratio of spilled records to output records','0.4943213459297361',NULL),(1520033,'Avg Physical Memory (MB)','854',NULL),(1520033,'Avg task runtime','25 sec',NULL),(1520033,'Avg Virtual Memory (MB)','2923',NULL),(1520033,'Max Physical Memory (MB)','872',NULL),(1520033,'Min Physical Memory (MB)','824',NULL),(1520033,'Number of tasks','27',NULL),(1520033,'Requested Container Memory','1 GB',NULL),(1520034,'Group A','8 tasks @ 48 MB avg',NULL),(1520034,'Group B','5 tasks @ 48 MB avg',NULL),(1520034,'Number of tasks','13',NULL),(1520035,'Avg task CPU time (ms)','20102',NULL),(1520035,'Avg task GC time (ms)','112',NULL),(1520035,'Avg task runtime (ms)','15768',NULL),(1520035,'Number of tasks','13',NULL),(1520035,'Task GC/CPU ratio','0.005571584916923689',NULL),(1520036,'Average task runtime','15 sec',NULL),(1520036,'Max task runtime','17 sec',NULL),(1520036,'Min task runtime','14 sec',NULL),(1520036,'Number of tasks','13',NULL),(1520037,'Avg Physical Memory (MB)','713',NULL),(1520037,'Avg task runtime','15 sec',NULL),(1520037,'Avg Virtual Memory (MB)','3282',NULL),(1520037,'Max Physical Memory (MB)','747',NULL),(1520037,'Min Physical Memory (MB)','681',NULL),(1520037,'Number of tasks','13',NULL),(1520037,'Requested Container Memory','1 GB',NULL),(1520038,'Average code runtime','8 sec',NULL),(1520038,'Average shuffle time','5 sec (0.69x)',NULL),(1520038,'Average sort time','1 sec (0.18x)',NULL),(1520038,'Number of tasks','13',NULL),(1520040,'Group A','2 tasks @ 150 MB avg',NULL),(1520040,'Group B','4 tasks @ 425 MB avg',NULL),(1520040,'Number of tasks','6',NULL),(1520041,'Avg task CPU time (ms)','25606',NULL),(1520041,'Avg task GC time (ms)','236',NULL),(1520041,'Avg task runtime (ms)','20435',NULL),(1520041,'Number of tasks','6',NULL),(1520041,'Task GC/CPU ratio','0.009216589861751152',NULL),(1520042,'Average task input size','334 MB',NULL),(1520042,'Average task runtime','20 sec',NULL),(1520042,'Max task runtime','24 sec',NULL),(1520042,'Min task runtime','12 sec',NULL),(1520042,'Number of tasks','6',NULL),(1520043,'Median task input size','425 MB',NULL),(1520043,'Median task runtime','22 sec',NULL),(1520043,'Median task speed','17 MB/s',NULL),(1520043,'Number of tasks','6',NULL),(1520043,'Total input size in MB','2004.069811820984',NULL),(1520044,'Avg output records per task','4915562',NULL),(1520044,'Avg spilled records per task','9831125',NULL),(1520044,'Number of tasks','6',NULL),(1520044,'Ratio of spilled records to output records','2.0',NULL),(1520045,'Avg Physical Memory (MB)','839',NULL),(1520045,'Avg task runtime','20 sec',NULL),(1520045,'Avg Virtual Memory (MB)','2915',NULL),(1520045,'Max Physical Memory (MB)','855',NULL),(1520045,'Min Physical Memory (MB)','821',NULL),(1520045,'Number of tasks','6',NULL),(1520045,'Requested Container Memory','1 GB',NULL),(1520046,'Group A','1 tasks @ 85 MB avg',NULL),(1520046,'Group B','2 tasks @ 86 MB avg',NULL),(1520046,'Number of tasks','3',NULL),(1520047,'Avg task CPU time (ms)','38256',NULL),(1520047,'Avg task GC time (ms)','454',NULL),(1520047,'Avg task runtime (ms)','27717',NULL),(1520047,'Number of tasks','3',NULL),(1520047,'Task GC/CPU ratio','0.011867419489753242',NULL),(1520048,'Average task runtime','27 sec',NULL),(1520048,'Max task runtime','28 sec',NULL),(1520048,'Min task runtime','26 sec',NULL),(1520048,'Number of tasks','3',NULL),(1520049,'Avg Physical Memory (MB)','1105',NULL),(1520049,'Avg task runtime','27 sec',NULL),(1520049,'Avg Virtual Memory (MB)','3276',NULL),(1520049,'Max Physical Memory (MB)','1176',NULL),(1520049,'Min Physical Memory (MB)','1057',NULL),(1520049,'Number of tasks','3',NULL),(1520049,'Requested Container Memory','1 GB',NULL),(1520050,'Average code runtime','18 sec',NULL),(1520050,'Average shuffle time','5 sec (0.28x)',NULL),(1520050,'Average sort time','4 sec (0.23x)',NULL),(1520050,'Number of tasks','3',NULL),(1520052,'Group A','1 tasks @ 476 MB avg',NULL),(1520052,'Group B','2 tasks @ 495 MB avg',NULL),(1520052,'Number of tasks','3',NULL),(1520053,'Avg task CPU time (ms)','39180',NULL),(1520053,'Avg task GC time (ms)','353',NULL),(1520053,'Avg task runtime (ms)','27557',NULL),(1520053,'Number of tasks','3',NULL),(1520053,'Task GC/CPU ratio','0.009009698825931597',NULL),(1520054,'Average task input size','489 MB',NULL),(1520054,'Average task runtime','27 sec',NULL),(1520054,'Max task runtime','29 sec',NULL),(1520054,'Min task runtime','26 sec',NULL),(1520054,'Number of tasks','3',NULL),(1520055,'Median task input size','495 MB',NULL),(1520055,'Median task runtime','27 sec',NULL),(1520055,'Median task speed','17 MB/s',NULL),(1520055,'Number of tasks','3',NULL),(1520055,'Total input size in MB','1467.140121459961',NULL),(1520056,'Avg output records per task','8855897',NULL),(1520056,'Avg spilled records per task','635697',NULL),(1520056,'Number of tasks','3',NULL),(1520056,'Ratio of spilled records to output records','0.07178233623003459',NULL),(1520057,'Avg Physical Memory (MB)','846',NULL),(1520057,'Avg task runtime','27 sec',NULL),(1520057,'Avg Virtual Memory (MB)','2917',NULL),(1520057,'Max Physical Memory (MB)','860',NULL),(1520057,'Min Physical Memory (MB)','834',NULL),(1520057,'Number of tasks','3',NULL),(1520057,'Requested Container Memory','1 GB',NULL),(1520058,'Group A','1 tasks @ 15 MB avg',NULL),(1520058,'Group B','1 tasks @ 15 MB avg',NULL),(1520058,'Number of tasks','2',NULL),(1520059,'Avg task CPU time (ms)','6420',NULL),(1520059,'Avg task GC time (ms)','59',NULL),(1520059,'Avg task runtime (ms)','11536',NULL),(1520059,'Number of tasks','2',NULL),(1520059,'Task GC/CPU ratio','0.009190031152647975',NULL),(1520060,'Average task runtime','11 sec',NULL),(1520060,'Max task runtime','11 sec',NULL),(1520060,'Min task runtime','11 sec',NULL),(1520060,'Number of tasks','2',NULL),(1520061,'Avg Physical Memory (MB)','617',NULL),(1520061,'Avg task runtime','11 sec',NULL),(1520061,'Avg Virtual Memory (MB)','3267',NULL),(1520061,'Max Physical Memory (MB)','617',NULL),(1520061,'Min Physical Memory (MB)','617',NULL),(1520061,'Number of tasks','2',NULL),(1520061,'Requested Container Memory','1 GB',NULL),(1520062,'Average code runtime','1 sec',NULL),(1520062,'Average shuffle time','9 sec (4.77x)',NULL),(1520062,'Average sort time',' (0.19x)',NULL),(1520062,'Number of tasks','2',NULL),(1520064,'Group A','308 tasks @ 859 MB avg',NULL),(1520064,'Group B','204 tasks @ 1 GB avg',NULL),(1520064,'Number of tasks','512',NULL),(1520065,'Avg task CPU time (ms)','105256',NULL),(1520065,'Avg task GC time (ms)','1649',NULL),(1520065,'Avg task runtime (ms)','87523',NULL),(1520065,'Number of tasks','512',NULL),(1520065,'Task GC/CPU ratio','0.015666565326442197',NULL),(1520066,'Average task input size','1 GB',NULL),(1520066,'Average task runtime','1 min 27 sec',NULL),(1520066,'Max task runtime','3 min 7 sec',NULL),(1520066,'Min task runtime','18 sec',NULL),(1520066,'Number of tasks','512',NULL),(1520067,'Median task input size','948 MB',NULL),(1520067,'Median task runtime','1 min 22 sec',NULL),(1520067,'Median task speed','12 MB/s',NULL),(1520067,'Number of tasks','512',NULL),(1520067,'Total input size in MB','539270.6440467834',NULL),(1520068,'Avg output records per task','736811',NULL),(1520068,'Avg spilled records per task','235320',NULL),(1520068,'Number of tasks','512',NULL),(1520068,'Ratio of spilled records to output records','0.3193772050486382',NULL),(1520069,'Avg Physical Memory (MB)','892',NULL),(1520069,'Avg task runtime','1 min 27 sec',NULL),(1520069,'Avg Virtual Memory (MB)','2924',NULL),(1520069,'Max Physical Memory (MB)','1001',NULL),(1520069,'Min Physical Memory (MB)','864',NULL),(1520069,'Number of tasks','512',NULL),(1520069,'Requested Container Memory','1 GB',NULL),(1520070,'Group A','291 tasks @ 7 MB avg',NULL),(1520070,'Group B','275 tasks @ 7 MB avg',NULL),(1520070,'Number of tasks','566',NULL),(1520071,'Avg task CPU time (ms)','9313',NULL),(1520071,'Avg task GC time (ms)','80',NULL),(1520071,'Avg task runtime (ms)','7792',NULL),(1520071,'Number of tasks','566',NULL),(1520071,'Task GC/CPU ratio','0.008590142811124236',NULL),(1520072,'Average task runtime','7 sec',NULL),(1520072,'Max task runtime','16 sec',NULL),(1520072,'Min task runtime','3 sec',NULL),(1520072,'Number of tasks','566',NULL),(1520073,'Avg Physical Memory (MB)','608',NULL),(1520073,'Avg task runtime','7 sec',NULL),(1520073,'Avg Virtual Memory (MB)','3286',NULL),(1520073,'Max Physical Memory (MB)','705',NULL),(1520073,'Min Physical Memory (MB)','508',NULL),(1520073,'Number of tasks','566',NULL),(1520073,'Requested Container Memory','1 GB',NULL),(1520074,'Average code runtime','',NULL),(1520074,'Average shuffle time','6 sec (8.14x)',NULL),(1520074,'Average sort time',' (0.38x)',NULL),(1520074,'Number of tasks','566',NULL),(1520076,'Group A','934 tasks @ 779 MB avg',NULL),(1520076,'Group B','840 tasks @ 1 GB avg',NULL),(1520076,'Number of tasks','1774',NULL),(1520077,'Avg task CPU time (ms)','64423',NULL),(1520077,'Avg task GC time (ms)','1714',NULL),(1520077,'Avg task runtime (ms)','46509',NULL),(1520077,'Number of tasks','1774',NULL),(1520077,'Task GC/CPU ratio','0.026605404901976003',NULL),(1520078,'Average task input size','944 MB',NULL),(1520078,'Average task runtime','46 sec',NULL),(1520078,'Max task runtime','2 min 23 sec',NULL),(1520078,'Min task runtime','23 sec',NULL),(1520078,'Number of tasks','1774',NULL),(1520079,'Median task input size','926 MB',NULL),(1520079,'Median task runtime','43 sec',NULL),(1520079,'Median task speed','21 MB/s',NULL),(1520079,'Number of tasks','1774',NULL),(1520079,'Total input size in MB','1676399.0118570328',NULL),(1520080,'Avg output records per task','3497298',NULL),(1520080,'Avg spilled records per task','6994597',NULL),(1520080,'Number of tasks','1774',NULL),(1520080,'Ratio of spilled records to output records','2.0',NULL),(1520081,'Avg Physical Memory (MB)','900',NULL),(1520081,'Avg task runtime','46 sec',NULL),(1520081,'Avg Virtual Memory (MB)','2936',NULL),(1520081,'Max Physical Memory (MB)','1000',NULL),(1520081,'Min Physical Memory (MB)','828',NULL),(1520081,'Number of tasks','1774',NULL),(1520081,'Requested Container Memory','1 GB',NULL),(1520082,'Group A','513 tasks @ 499 MB avg',NULL),(1520082,'Group B','486 tasks @ 500 MB avg',NULL),(1520082,'Number of tasks','999',NULL),(1520083,'Avg task CPU time (ms)','71535',NULL),(1520083,'Avg task GC time (ms)','1696',NULL),(1520083,'Avg task runtime (ms)','52042',NULL),(1520083,'Number of tasks','999',NULL),(1520083,'Task GC/CPU ratio','0.023708674075627316',NULL),(1520084,'Average task runtime','52 sec',NULL),(1520084,'Max task runtime','1 min 48 sec',NULL),(1520084,'Min task runtime','38 sec',NULL),(1520084,'Number of tasks','999',NULL),(1520085,'Avg Physical Memory (MB)','1612',NULL),(1520085,'Avg task runtime','52 sec',NULL),(1520085,'Avg Virtual Memory (MB)','3297',NULL),(1520085,'Max Physical Memory (MB)','1792',NULL),(1520085,'Min Physical Memory (MB)','1389',NULL),(1520085,'Number of tasks','999',NULL),(1520085,'Requested Container Memory','1 GB',NULL),(1520086,'Average code runtime','29 sec',NULL),(1520086,'Average shuffle time','15 sec (0.54x)',NULL),(1520086,'Average sort time','6 sec (0.22x)',NULL),(1520086,'Number of tasks','999',NULL),(1536115,'Group A','0 tasks @ 0 bytes avg',NULL),(1536115,'Group B','1 tasks @ 515 MB avg',NULL),(1536115,'Number of tasks','1',NULL),(1536116,'Avg task CPU time (ms)','109330',NULL),(1536116,'Avg task GC time (ms)','6065',NULL),(1536116,'Avg task runtime (ms)','102946',NULL),(1536116,'Number of tasks','1',NULL),(1536116,'Task GC/CPU ratio','0.05547425226378853',NULL),(1536117,'Average task input size','515 MB',NULL),(1536117,'Average task runtime','1 min 42 sec',NULL),(1536117,'Max task runtime','1 min 42 sec',NULL),(1536117,'Min task runtime','1 min 42 sec',NULL),(1536117,'Number of tasks','1',NULL),(1536118,'Median task input size','515 MB',NULL),(1536118,'Median task runtime','1 min 42 sec',NULL),(1536118,'Median task speed','5 MB/s',NULL),(1536118,'Number of tasks','1',NULL),(1536118,'Total input size in MB','515.5552177429199',NULL),(1536119,'Avg output records per task','977313',NULL),(1536119,'Avg spilled records per task','251',NULL),(1536119,'Number of tasks','1',NULL),(1536119,'Ratio of spilled records to output records','2.5682662565626365E-4',NULL),(1536120,'Avg Physical Memory (MB)','1860',NULL),(1536120,'Avg task runtime','1 min 42 sec',NULL),(1536120,'Avg Virtual Memory (MB)','3672',NULL),(1536120,'Max Physical Memory (MB)','1860',NULL),(1536120,'Min Physical Memory (MB)','1860',NULL),(1536120,'Number of tasks','1',NULL),(1536120,'Requested Container Memory','2 GB',NULL),(1536121,'Group A','0 tasks @ 0 bytes avg',NULL),(1536121,'Group B','1 tasks @ 1 KB avg',NULL),(1536121,'Number of tasks','1',NULL),(1536122,'Avg task CPU time (ms)','2700',NULL),(1536122,'Avg task GC time (ms)','109',NULL),(1536122,'Avg task runtime (ms)','8496',NULL),(1536122,'Number of tasks','1',NULL),(1536122,'Task GC/CPU ratio','0.04037037037037037',NULL),(1536123,'Average task runtime','8 sec',NULL),(1536123,'Max task runtime','8 sec',NULL),(1536123,'Min task runtime','8 sec',NULL),(1536123,'Number of tasks','1',NULL),(1536124,'Avg Physical Memory (MB)','398',NULL),(1536124,'Avg task runtime','8 sec',NULL),(1536124,'Avg Virtual Memory (MB)','4100',NULL),(1536124,'Max Physical Memory (MB)','398',NULL),(1536124,'Min Physical Memory (MB)','398',NULL),(1536124,'Number of tasks','1',NULL),(1536124,'Requested Container Memory','3 GB',NULL),(1536125,'Average code runtime','',NULL),(1536125,'Average shuffle time','7 sec (9.49x)',NULL),(1536125,'Average sort time',' (0.05x)',NULL),(1536125,'Number of tasks','1',NULL),(1536163,'Group A','0 tasks @ 0 bytes avg',NULL),(1536163,'Group B','1 tasks @ 515 MB avg',NULL),(1536163,'Number of tasks','1',NULL),(1536164,'Avg task CPU time (ms)','132240',NULL),(1536164,'Avg task GC time (ms)','6714',NULL),(1536164,'Avg task runtime (ms)','186804',NULL),(1536164,'Number of tasks','1',NULL),(1536164,'Task GC/CPU ratio','0.050771324863883846',NULL),(1536165,'Average task input size','515 MB',NULL),(1536165,'Average task runtime','3 min 6 sec',NULL),(1536165,'Max task runtime','3 min 6 sec',NULL),(1536165,'Min task runtime','3 min 6 sec',NULL),(1536165,'Number of tasks','1',NULL),(1536166,'Median task input size','515 MB',NULL),(1536166,'Median task runtime','3 min 6 sec',NULL),(1536166,'Median task speed','2 MB/s',NULL),(1536166,'Number of tasks','1',NULL),(1536166,'Total input size in MB','515.5552177429199',NULL),(1536167,'Avg output records per task','977313',NULL),(1536167,'Avg spilled records per task','251',NULL),(1536167,'Number of tasks','1',NULL),(1536167,'Ratio of spilled records to output records','2.5682662565626365E-4',NULL),(1536168,'Avg Physical Memory (MB)','968',NULL),(1536168,'Avg task runtime','3 min 6 sec',NULL),(1536168,'Avg Virtual Memory (MB)','2345',NULL),(1536168,'Max Physical Memory (MB)','968',NULL),(1536168,'Min Physical Memory (MB)','968',NULL),(1536168,'Number of tasks','1',NULL),(1536168,'Requested Container Memory','4 GB',NULL),(1536169,'Group A','0 tasks @ 0 bytes avg',NULL),(1536169,'Group B','1 tasks @ 1 KB avg',NULL),(1536169,'Number of tasks','1',NULL),(1536170,'Avg task CPU time (ms)','2830',NULL),(1536170,'Avg task GC time (ms)','73',NULL),(1536170,'Avg task runtime (ms)','8188',NULL),(1536170,'Number of tasks','1',NULL),(1536170,'Task GC/CPU ratio','0.025795053003533568',NULL),(1536171,'Average task runtime','8 sec',NULL),(1536171,'Max task runtime','8 sec',NULL),(1536171,'Min task runtime','8 sec',NULL),(1536171,'Number of tasks','1',NULL),(1536172,'Avg Physical Memory (MB)','301',NULL),(1536172,'Avg task runtime','8 sec',NULL),(1536172,'Avg Virtual Memory (MB)','2338',NULL),(1536172,'Max Physical Memory (MB)','301',NULL),(1536172,'Min Physical Memory (MB)','301',NULL),(1536172,'Number of tasks','1',NULL),(1536172,'Requested Container Memory','4 GB',NULL),(1536173,'Average code runtime','',NULL),(1536173,'Average shuffle time','7 sec (9.03x)',NULL),(1536173,'Average sort time',' (0.04x)',NULL),(1536173,'Number of tasks','1',NULL); diff --git a/test/rest/RestAPITest.java b/test/rest/RestAPITest.java deleted file mode 100644 index b3ad2d93e..000000000 --- a/test/rest/RestAPITest.java +++ /dev/null @@ -1,1021 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package rest; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; -import com.linkedin.drelephant.DrElephant; -import com.linkedin.drelephant.ElephantContext; -import com.linkedin.drelephant.tuning.BaselineComputeUtil; -import com.linkedin.drelephant.tuning.FitnessComputeUtil; -import com.linkedin.drelephant.util.Utils; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import models.JobDefinition; -import models.JobExecution; -import models.JobExecution.ExecutionState; -import models.JobSuggestedParamSet; -import models.TuningJobDefinition; -import models.JobSuggestedParamSet.ParamSetStatus; - -import models.TuningJobExecutionParamSet; -import org.apache.hadoop.conf.Configuration; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import play.Application; -import play.GlobalSettings; -import play.libs.WS; -import play.test.FakeApplication; - -import static common.DBTestUtil.*; -import static common.TestConstants.*; -import static org.junit.Assert.assertTrue; -import static play.test.Helpers.fakeApplication; -import static play.test.Helpers.running; -import static play.test.Helpers.testServer; - - -/** - *

- * Class aims to exercise all the rest end points exposed by Dr.Elephant - *

- *

- * A fake application connecting to an in-memory H2 DB is started inside
- * the test server which runs the test code. The global class is overridden
- * so that we don't have to go through the regular application start flow. - *

- */ -public class RestAPITest { - - private static final Logger logger = LoggerFactory.getLogger(RestAPITest.class); - private static FakeApplication fakeApp; - - @Before - public void setup() { - Map dbConn = new HashMap(); - dbConn.put(DB_DEFAULT_DRIVER_KEY, DB_DEFAULT_DRIVER_VALUE); - dbConn.put(DB_DEFAULT_URL_KEY, DB_DEFAULT_URL_VALUE); - dbConn.put(EVOLUTION_PLUGIN_KEY, EVOLUTION_PLUGIN_VALUE); - dbConn.put(APPLY_EVOLUTIONS_DEFAULT_KEY, APPLY_EVOLUTIONS_DEFAULT_VALUE); - - GlobalSettings gs = new GlobalSettings() { - @Override - public void onStart(Application app) { - logger.info("Starting FakeApplication"); - } - }; - - fakeApp = fakeApplication(dbConn, gs); - } - - /** - *

- * Rest API - Performs search by job ID - *
- * API provides information on the specific job - *

- *

- * Following assertions are made in the response json - *

    Job id
- *
    Job name
- *
    Job type
- *

- */ - @Test - public void testrestAppResult() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_APP_RESULT_PATH). - setQueryParameter("id", TEST_JOB_ID1).setQueryParameter("prioritize", "true"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - final JsonNode jsonResponse = response.asJson(); - assertTrue("Job id did not match", TEST_JOB_ID1.equals(jsonResponse.path("id").asText())); - assertTrue("Job name did not match", TEST_JOB_NAME.equals(jsonResponse.path("name").asText())); - assertTrue("Job type did not match", TEST_JOB_TYPE.equals(jsonResponse.path("jobType").asText())); - } - }); - } - - @Test - public void testRestGetCurrentRunParameters() { - Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - Boolean autoTuningEnabled = configuration.getBoolean(DrElephant.AUTO_TUNING_ENABLED, false); - org.junit.Assume.assumeTrue(autoTuningEnabled); - - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateAutoTuningTestData1(); - - JsonNode jsonNode = getTestGetCurrentRunParameterData(); - final WS.Response response = WS.url(BASE_URL + REST_GET_CURRENT_RUN_PARAMETERS). - post(jsonNode).get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - - final JsonNode jsonResponse = response.asJson(); - - logger.info("Output of getCurrentRunParameters "); - logger.info(jsonResponse.toString()); - - assertTrue("Get current run param output did not match", - jsonResponse.path("mapreduce.map.memory.mb").asDouble() > 0); - assertTrue("Get current run param output did not match", - jsonResponse.path("mapreduce.reduce.memory.mb").asDouble() > 0); - assertTrue("Get current run param output size did not match", jsonResponse.size() == 9); - - TuningJobExecutionParamSet tuningJobExecutionParamSet = TuningJobExecutionParamSet.find.select("*") - .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") - .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") - .where() - .eq(TuningJobExecutionParamSet.TABLE.jobExecution + "." + JobExecution.TABLE.jobExecId, - "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0") - .findUnique(); - - JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; - JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; - - jobExecution.executionState = ExecutionState.SUCCEEDED; - jobExecution.update(); - jobSuggestedParamSet.paramSetState = ParamSetStatus.EXECUTED; - jobSuggestedParamSet.update(); - - FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil(); - fitnessComputeUtil.updateFitness(); - - jobSuggestedParamSet = JobSuggestedParamSet.find.byId(jobSuggestedParamSet.id); - - assertTrue("Fitness not computed", jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)); - assertTrue("Fitness is non-positive", jobSuggestedParamSet.fitness > 0); - } - }); - } - - @Test - public void testRestGetCurrentRunParametersNewJob() { - Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - Boolean autoTuningEnabled = configuration.getBoolean(DrElephant.AUTO_TUNING_ENABLED, false); - org.junit.Assume.assumeTrue(autoTuningEnabled); - - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateAutoTuningTestData1(); - - JsonNode jsonNode = getTestGetCurrentRunParameterNewData(); - final WS.Response response = WS.url(BASE_URL + REST_GET_CURRENT_RUN_PARAMETERS). - post(jsonNode).get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - - final JsonNode jsonResponse = response.asJson(); - - assertTrue("Get current run param output did not match", - jsonResponse.path("mapreduce.map.memory.mb").asDouble() == 2048D); - assertTrue("Get current run param output did not match", - jsonResponse.path("mapreduce.reduce.memory.mb").asDouble() == 2048D); - assertTrue("Get current run param output size did not match", jsonResponse.size() == 2); - - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry") - .findUnique(); - - assertTrue("New Job Not created ", - tuningJobDefinition.job.jobName.equals("countByCountryFlowSmallNew_countByCountry")); - - BaselineComputeUtil baselineComputeUtil = new BaselineComputeUtil(); - baselineComputeUtil.computeBaseline(); - - tuningJobDefinition = TuningJobDefinition.find.select("*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry") - .findUnique(); - - assertTrue("Baseline not computed:averageResourceUsage ", tuningJobDefinition.averageResourceUsage > 0); - assertTrue("Baseline not computed:averageInputSizeInBytes ", tuningJobDefinition.averageInputSizeInBytes > 0); - assertTrue("Baseline not computed:averageExecutionTime ", tuningJobDefinition.averageExecutionTime > 0); - } - }); - } - - private JsonNode getTestGetCurrentRunParameterNewData() { - Map params = new HashMap(); - Map paramValueMap = new HashMap(); - paramValueMap.put("mapreduce.map.memory.mb", 2048D); - paramValueMap.put("mapreduce.reduce.memory.mb", 2048D); - Gson gson = new Gson(); - String jobParamsJson = gson.toJson(paramValueMap); - - params.put("projectName", "AzkabanHelloPigTest"); - params.put("flowDefId", - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew"); - params.put("jobDefId", - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry"); - params.put("flowExecId", "https://elephant.linkedin.com:8443/executor?execid=5221700"); - params.put("jobExecId", - "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmallNew_countByCountry&attempt=0"); - params.put("flowDefUrl", - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew"); - params.put("jobDefUrl", - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmallNew&job=countByCountryFlowSmallNew_countByCountry"); - params.put("flowExecUrl", "https://elephant.linkedin.com:8443/executor?execid=5221700"); - params.put("jobExecUrl", - "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmallNew_countByCountry&attempt=0"); - params.put("jobName", "countByCountryFlowSmallNew_countByCountry"); - params.put("defaultParams", jobParamsJson); - params.put("scheduler", "azkaban"); - params.put("client", "azkaban"); - params.put("autoTuningJobType", "PIG"); - params.put("optimizationMetric", "RESOURCE"); - params.put("userName", "mkumar1"); - params.put("isRetry", "false"); - params.put("skipExecutionForOptimization", "false"); - JsonNode jsonNode = new ObjectMapper().valueToTree(params); - return jsonNode; - } - - private JsonNode getTestGetCurrentRunParameterData() { - Map params = new HashMap(); - Map paramValueMap = new HashMap(); - paramValueMap.put("mapreduce.map.memory.mb", 2048D); - paramValueMap.put("mapreduce.reduce.memory.mb", 2048D); - Gson gson = new Gson(); - String jobParamsJson = gson.toJson(paramValueMap); - - params.put("projectName", "AzkabanHelloPigTest"); - params.put("flowDefId", - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall"); - params.put("jobDefId", - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry"); - params.put("flowExecId", "https://elephant.linkedin.com:8443/executor?execid=5221700"); - params.put("jobExecId", - "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0"); - params.put("flowDefUrl", - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall"); - params.put("jobDefUrl", - "https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlowSmall&job=countByCountryFlowSmall_countByCountry"); - params.put("flowExecUrl", "https://elephant.linkedin.com:8443/executor?execid=5221700"); - params.put("jobExecUrl", - "https://elephant.linkedin.com:8443/executor?execid=5221700&job=countByCountryFlowSmall_countByCountry&attempt=0"); - params.put("jobName", "countByCountryFlowSmall_countByCountry"); - params.put("defaultParams", jobParamsJson); - params.put("scheduler", "azkaban"); - params.put("client", "azkaban"); - params.put("autoTuningJobType", "PIG"); - params.put("optimizationMetric", "RESOURCE"); - params.put("userName", "mkumar1"); - params.put("isRetry", "false"); - params.put("skipExecutionForOptimization", "false"); - JsonNode jsonNode = new ObjectMapper().valueToTree(params); - return jsonNode; - } - - /** - *

- * Rest API - Performs search by job execution ID - *
- * API returns all jobs triggered by a particular Scheduler Job - *

- *

- * Following assertions are made in the response json - *

    Job id
- *
    Job execution id
- *

- */ - @Test - public void testrestJobExecResult() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_JOB_EXEC_RESULT_PATH). - setQueryParameter("id", TEST_JOB_EXEC_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - final JsonNode jsonResponse = response.asJson().get(0); - assertTrue("Job id did not match", TEST_JOB_ID1.equals(jsonResponse.path("id").asText())); - assertTrue("Job execution id did not match", TEST_JOB_EXEC_ID1.equals(jsonResponse.path("jobExecId").asText())); - } - }); - } - - /** - *

- * Rest API - Performs search by flow execution ID - *
- * API returns all jobs under a particular flow execution - *

- *

- * Following assertions are made in the response json - *

    Job id
- *
    Flow execution id
- *

- */ - @Test - public void testrestFlowExecResult() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_FLOW_EXEC_RESULT_PATH). - setQueryParameter("id", TEST_FLOW_EXEC_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - final JsonNode jsonResponse = response.asJson(); - assertTrue("Job id did not match", TEST_JOB_ID1.equals(jsonResponse.findValue("id").asText())); - assertTrue("Flow execution id did not match", - TEST_FLOW_EXEC_ID1.equals(jsonResponse.findValue("flowExecId").asText())); - } - }); - } - - /** - *

- * Rest API - Perform a generic search or search by filter criteria - *
- * Test verifies if all available flows are returned - *

- *

- * Following assertions are made in the response json - *

    First job id
- *
    Second job id
- *

- */ - @Test - public void testrestSearch() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_PATH). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - List jobList = response.asJson().findValuesAsText("id"); - assertTrue("Job id1 missing in list", jobList.contains(TEST_JOB_ID1)); - assertTrue("Job id2 missing in list", jobList.contains(TEST_JOB_ID2)); - } - }); - } - - /** - *

- * Rest API - Perform a search with additional params - *
- * Test verifies if specific flow is returned - *

- *

- * Following assertions are made in the response json - *

    No of jobs returned
- *
    Job id
- *
    Username
- *
    Job type
- *

- */ - @Test - public void testrestSearchWithUsernameAndJobType() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_PATH). - setQueryParameter("username", TEST_USERNAME). - setQueryParameter("", TEST_JOB_TYPE). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - JsonNode reponseJson = response.asJson(); - List jobList = reponseJson.findValuesAsText("id"); - assertTrue("More than one row returned", jobList.size() == 1); - assertTrue("Job id missing in response", TEST_JOB_ID1.equals(reponseJson.findValue("id").asText())); - assertTrue("Username incorrect", TEST_USERNAME.equals(reponseJson.findValue("username").asText())); - assertTrue("Job type incorrect", TEST_JOB_TYPE.equals(reponseJson.findValue("jobType").asText())); - } - }); - } - - /** - *

- * Rest API - Compares two flow executions by flow execution ID - *

- *

- * Following assertions are made in the response json - *

    Second job ID
- *

- */ - @Test - public void testrestCompare() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_COMPARE_PATH). - setQueryParameter("flow-exec-id1", TEST_FLOW_EXEC_ID1). - setQueryParameter("flow-exec-id2", TEST_FLOW_EXEC_ID2). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - assertTrue("Job id did not match", TEST_JOB_ID2.equals(response.asJson().findValue("id").asText())); - } - }); - } - - /** - *

- * Rest API - Provides data for plotting the flow history graph - *

- *

- * Following assertions are made in the response json - *

    First job execution ID
- *
    Second job execution ID
- *

- */ - @Test - public void testrestFlowGraphData() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_FLOW_GRAPH_DATA_PATH). - setQueryParameter("id", TEST_FLOW_DEF_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - List jobList = response.asJson().findValuesAsText("jobexecurl"); - assertTrue("Job exec url1 missing in list", jobList.contains(TEST_JOB_EXEC_ID1)); - assertTrue("Job exec url2 missing in list", jobList.contains(TEST_JOB_EXEC_ID2)); - } - }); - } - - /** - *

- * Rest API - Provides data for plotting the job history graph - *

- *

- * Following assertions are made in the response json - *

    First job id
- *
    Second job id
- *

- */ - @Test - public void testrestJobGraphData() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_JOB_GRAPH_DATA_PATH). - setQueryParameter("id", TEST_JOB_DEF_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - List jobList = response.asJson().findValuesAsText("stageid"); - assertTrue("Job id 1 missing in list", jobList.contains(TEST_JOB_ID1)); - assertTrue("Job id 2 missing in list", jobList.contains(TEST_JOB_ID2)); - } - }); - } - - /** - *

- * Rest API - Provides data for plotting the job history graph for time and resources - *

- */ - public void testrestJobMetricsGraphData() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_JOB_METRICS_GRAPH_DATA_PATH). - setQueryParameter("id", TEST_JOB_DEF_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - List jobList = response.asJson().findValuesAsText("stageid"); - assertTrue("Job id 1 missing in list", jobList.contains(TEST_JOB_ID1)); - assertTrue("Job id 2 missing in list", jobList.contains(TEST_JOB_ID2)); - } - }); - } - - /** - *

- * Rest API - Provides data for plotting the flow history graph for time and resources - *

- */ - @Test - public void testrestFlowMetricsGraphData() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_FLOW_METRICS_GRAPH_DATA_PATH). - setQueryParameter("id", TEST_FLOW_DEF_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - List jobList = response.asJson().findValuesAsText("jobexecurl"); - assertTrue("Job exec url1 missing in list", jobList.contains(TEST_JOB_EXEC_ID1)); - assertTrue("Job exec url2 missing in list", jobList.contains(TEST_JOB_EXEC_ID2)); - } - }); - } - - @Test - public void testRestUserResourceUsage() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_USER_RESOURCE_USAGE_PATH). - setQueryParameter("startTime", TEST_START_TIME1). - setQueryParameter("endTime", TEST_END_TIME1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator userResources = response.asJson().elements(); - while (userResources.hasNext()) { - JsonNode userResourceUsage = userResources.next(); - if (userResourceUsage.findValue("user").asText().equals("growth")) { - assertTrue("Wrong resourceusage for user growth", - userResourceUsage.findValue("resourceUsed").asDouble() == Utils.MBSecondsToGBHours(100)); - assertTrue("Wrong wastedResources for user growth", - userResourceUsage.findValue("resourceWasted").asDouble() == Utils.MBSecondsToGBHours(30)); - } else if (userResourceUsage.findValue("user").asText().equals("metrics")) { - assertTrue("Wrong resourceusage for user metrics", - userResourceUsage.findValue("resourceUsed").asDouble() == Utils.MBSecondsToGBHours(200)); - assertTrue("Wrong wastedResources for user metrics", - userResourceUsage.findValue("resourceWasted").asDouble() == Utils.MBSecondsToGBHours(40)); - } else { - assertTrue("Unexpected user" + userResourceUsage.findValue("user").asText(), false); - } - } - } - }); - } - - @Test - public void testRestUserResourceUsageBadInput() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_USER_RESOURCE_USAGE_PATH). - setQueryParameter("startTime", TEST_START_TIME1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - assertTrue("Invalid input test failed", response.getStatus() == 400); - } - }); - } - - @Test - public void testRestWorkflowForuser() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_WORKFLOW_SUMMARIES_PATH). - setQueryParameter("username", TEST_USERNAME). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator workflowSummaries = response.asJson().elements(); - while (workflowSummaries.hasNext()) { - JsonNode workflowSummary = workflowSummaries.next(); - Iterator workflowObjects = workflowSummary.elements(); - while (workflowObjects.hasNext()) { - JsonNode node = workflowObjects.next(); - Assert.assertEquals(node.findValue("username").asText(), "growth"); - Assert.assertEquals(node.findValue("starttime").asLong(), 1460980616502L); - Assert.assertEquals(node.findValue("finishtime").asLong(), 1460980723925L); - Assert.assertEquals(node.findValue("waittime").asLong(), 20); - Assert.assertEquals(node.findValue("resourceused").asLong(), 100); - Assert.assertEquals(node.findValue("resourcewasted").asLong(), 30); - Assert.assertEquals(node.findValue("severity").asText(), "None"); - Assert.assertEquals(node.findValue("queue").asText(), "misc_default"); - - Iterator jobs = node.findValue("jobsseverity").elements(); - while (jobs.hasNext()) { - JsonNode job = jobs.next(); - Assert.assertEquals(job.findValue("severity").asText(), "None"); - Assert.assertEquals(job.findValue("count").asInt(), 1); - } - } - } - } - }); - } - - @Test - public void testRestJobForUser() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_JOB_SUMMARIES_PATH). - setQueryParameter("username", TEST_USERNAME). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator jobSummaries = response.asJson().elements(); - while (jobSummaries.hasNext()) { - JsonNode jobSummary = jobSummaries.next(); - Iterator jobObjects = jobSummary.elements(); - while (jobObjects.hasNext()) { - JsonNode node = jobObjects.next(); - Assert.assertEquals(node.findValue("username").asText(), "growth"); - Assert.assertEquals(node.findValue("jobname").asText(), "overwriter-reminder2"); - Assert.assertEquals(node.findValue("jobtype").asText(), "HadoopJava"); - Assert.assertEquals(node.findValue("starttime").asLong(), 1460980616502L); - Assert.assertEquals(node.findValue("finishtime").asLong(), 1460980723925L); - Assert.assertEquals(node.findValue("waittime").asLong(), 20); - Assert.assertEquals(node.findValue("resourceused").asLong(), 100); - Assert.assertEquals(node.findValue("resourcewasted").asLong(), 30); - Assert.assertEquals(node.findValue("severity").asText(), "None"); - Assert.assertEquals(node.findValue("queue").asText(), "misc_default"); - - Iterator tasks = node.findValue("tasksseverity").elements(); - while (tasks.hasNext()) { - JsonNode job = tasks.next(); - Assert.assertEquals(job.findValue("severity").asText(), "None"); - Assert.assertEquals(job.findValue("count").asInt(), 1); - } - } - } - } - }); - } - - @Test - public void testRestApplicationForUser() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_APPLICATION_SUMMARIES_PATH). - setQueryParameter("username", TEST_USERNAME). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - - Iterator taskSummaries = response.asJson().elements(); - while (taskSummaries.hasNext()) { - JsonNode taskSummary = taskSummaries.next(); - Iterator jobObjects = taskSummary.elements(); - while (jobObjects.hasNext()) { - JsonNode node = jobObjects.next(); - Assert.assertEquals(node.findValue("username").asText(), "growth"); - Assert.assertEquals(node.findValue("starttime").asLong(), 1460980616502L); - Assert.assertEquals(node.findValue("finishtime").asLong(), 1460980723925L); - Assert.assertEquals(node.findValue("waittime").asLong(), 20); - Assert.assertEquals(node.findValue("resourceused").asLong(), 100); - Assert.assertEquals(node.findValue("resourcewasted").asLong(), 30); - Assert.assertEquals(node.findValue("severity").asText(), "None"); - Assert.assertEquals(node.findValue("queue").asText(), "misc_default"); - - Iterator heuristicsSummary = node.findValue("heuristicsummary").elements(); - - HashMap expectedHeuristics = new LinkedHashMap(); - expectedHeuristics.put("Mapper Skew", "None"); - expectedHeuristics.put("Mapper GC", "None"); - expectedHeuristics.put("Mapper Time", "None"); - expectedHeuristics.put("Mapper Speed", "None"); - expectedHeuristics.put("Mapper Spill", "None"); - expectedHeuristics.put("Mapper Memory", "None"); - expectedHeuristics.put("Reducer Skew", "None"); - expectedHeuristics.put("Reducer Time", "None"); - expectedHeuristics.put("Reducer GC", "None"); - expectedHeuristics.put("Reducer Memory", "None"); - expectedHeuristics.put("Shuffle & Sort", "None"); - - Iterator keyIterator = expectedHeuristics.keySet().iterator(); - while (heuristicsSummary.hasNext() && keyIterator.hasNext()) { - JsonNode job = heuristicsSummary.next(); - String key = keyIterator.next().toString(); - Assert.assertEquals(key, job.findValue("name").asText()); - Assert.assertEquals(expectedHeuristics.get(key), job.findValue("severity").asText()); - } - } - } - } - }); - } - - @Test - public void testRestWorkflowFromId() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_WORKFLOWS_PATH). - setQueryParameter("workflowid", TEST_FLOW_EXEC_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator workflows = response.asJson().elements(); - while (workflows.hasNext()) { - JsonNode node = workflows.next(); - Assert.assertEquals(node.findValue("username").asText(), "growth"); - Assert.assertEquals(node.findValue("starttime").asLong(), 1460980616502L); - Assert.assertEquals(node.findValue("finishtime").asLong(), 1460980723925L); - Assert.assertEquals(node.findValue("waittime").asLong(), 20); - Assert.assertEquals(node.findValue("resourceused").asLong(), 100); - Assert.assertEquals(node.findValue("resourcewasted").asLong(), 30); - Assert.assertEquals(node.findValue("severity").asText(), "None"); - Assert.assertEquals(node.findValue("queue").asText(), "misc_default"); - } - } - }); - } - - @Test - public void testRestWorkflowFromIdIsEmpty() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_WORKFLOWS_PATH). - setQueryParameter("workflowid", "this_is_a_random_id"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - JsonNode workflows = response.asJson(); - Assert.assertEquals(workflows.get("username"), null); - Assert.assertEquals(workflows.get("starttime"), null); - Assert.assertEquals(workflows.get("finishtime"), null); - Assert.assertEquals(workflows.get("waittime"), null); - Assert.assertEquals(workflows.get("resourceused"), null); - Assert.assertEquals(workflows.get("resourcewasted"), null); - Assert.assertEquals(workflows.get("severity"), null); - Assert.assertEquals(workflows.get("queue"), null); - } - }); - } - - @Test - public void testRestJobFromId() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_JOBS_PATH). - setQueryParameter("jobid", TEST_JOB_EXEC_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator jobs = response.asJson().elements(); - while (jobs.hasNext()) { - JsonNode node = jobs.next(); - Assert.assertEquals(node.findValue("username").asText(), "growth"); - Assert.assertEquals(node.findValue("starttime").asLong(), 1460980616502L); - Assert.assertEquals(node.findValue("finishtime").asLong(), 1460980723925L); - Assert.assertEquals(node.findValue("waittime").asLong(), 20); - Assert.assertEquals(node.findValue("resourceused").asLong(), 100); - Assert.assertEquals(node.findValue("resourcewasted").asLong(), 30); - Assert.assertEquals(node.findValue("severity").asText(), "None"); - Assert.assertEquals(node.findValue("queue").asText(), "misc_default"); - } - } - }); - } - - @Test - public void testRestJobFromIdIsEmpty() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_JOBS_PATH). - setQueryParameter("jobid", "this_is_a_random_job_id"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - JsonNode jobs = response.asJson(); - Assert.assertEquals(jobs.get("username"), null); - Assert.assertEquals(jobs.get("starttime"), null); - Assert.assertEquals(jobs.get("finishtime"), null); - Assert.assertEquals(jobs.get("waittime"), null); - Assert.assertEquals(jobs.get("resourceused"), null); - Assert.assertEquals(jobs.get("resourcewasted"), null); - Assert.assertEquals(jobs.get("severity"), null); - Assert.assertEquals(jobs.get("queue"), null); - } - }); - } - - @Test - public void testApplicationFromId() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_APPLICATIONS_PATH). - setQueryParameter("applicationid", TEST_JOB_ID1). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator applications = response.asJson().elements(); - while (applications.hasNext()) { - JsonNode node = applications.next(); - Assert.assertEquals(node.findValue("username").asText(), "growth"); - Assert.assertEquals(node.findValue("starttime").asLong(), 1460980616502L); - Assert.assertEquals(node.findValue("finishtime").asLong(), 1460980723925L); - Assert.assertEquals(node.findValue("waittime").asLong(), 20); - Assert.assertEquals(node.findValue("resourceused").asLong(), 100); - Assert.assertEquals(node.findValue("resourcewasted").asLong(), 30); - Assert.assertEquals(node.findValue("severity").asText(), "None"); - Assert.assertEquals(node.findValue("queue").asText(), "misc_default"); - Assert.assertEquals(node.findValue("trackingurl").asText(), - "http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453361"); - } - } - }); - } - - @Test - public void testApplicationFromIdIsEmpty() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_APPLICATIONS_PATH). - setQueryParameter("applicationid", "random_id"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - JsonNode applications = response.asJson(); - Assert.assertEquals(applications.get("username"), null); - Assert.assertEquals(applications.get("starttime"), null); - Assert.assertEquals(applications.get("finishtime"), null); - Assert.assertEquals(applications.get("waittime"), null); - Assert.assertEquals(applications.get("resourceused"), null); - Assert.assertEquals(applications.get("resourcewasted"), null); - Assert.assertEquals(applications.get("severity"), null); - Assert.assertEquals(applications.get("queue"), null); - } - }); - } - - @Test - public void testRestSearchDataParamUserQueue() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS). - setQueryParameter("username", "growth").setQueryParameter("queue-name", "misc_default"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator searchNode = response.asJson().elements(); - testRestSearchGeneric(searchNode); - } - }); - } - - @Test - public void testRestSearchDataParamTypeUser() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS). - setQueryParameter("username", "growth").setQueryParameter("job-type", "HadoopJava"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator searchNode = response.asJson().elements(); - testRestSearchGeneric(searchNode); - } - }); - } - - @Test - public void testRestSearchDataParamTimeUser() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS) - . - setQueryParameter("username", "growth") - .setQueryParameter("finishTimeBegin", "1460980723925") - .setQueryParameter("finishTimeEnd", "1460980723928") - . - get() - .get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator searchNode = response.asJson().elements(); - testRestSearchGeneric(searchNode); - } - }); - } - - @Test - public void testRestSearchOffsetNegative() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS). - setQueryParameter("username", "growth").setQueryParameter("offset", "-1"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator searchNode = response.asJson().elements(); - testRestSearchGeneric(searchNode); - } - }); - } - - @Test - public void testRestSearchLimitNegative() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS). - setQueryParameter("username", "growth").setQueryParameter("limit", "-1"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - JsonNode searchNode = response.asJson(); - Assert.assertTrue(searchNode.asText().toString().isEmpty()); - } - }); - } - - @Test - public void testRestSearchOffsetZero() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS). - setQueryParameter("username", "growth").setQueryParameter("offset", "0"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator searchNode = response.asJson().elements(); - testRestSearchGeneric(searchNode); - } - }); - } - - @Test - public void testRestSearchLimitZero() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS). - setQueryParameter("username", "growth").setQueryParameter("limit", "0"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - JsonNode searchNode = response.asJson(); - Assert.assertTrue(searchNode.asText().toString().isEmpty()); - } - }); - } - - @Test - public void tstRestSearchLimitOutOfLimit() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS). - setQueryParameter("username", "growth").setQueryParameter("limit", "1000"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator searchNode = response.asJson().elements(); - testRestSearchGeneric(searchNode); - } - }); - } - - @Test - public void testRestSearchOffsetOutofLimit() { - running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { - public void run() { - populateTestData(); - final WS.Response response = WS.url(BASE_URL + REST_SEARCH_RESULTS). - setQueryParameter("username", "growth").setQueryParameter("offset", "100"). - get().get(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS); - Iterator searchNode = response.asJson().elements(); - while (searchNode.hasNext()) { - JsonNode node = searchNode.next(); - JsonNode summaries = node.get("summaries"); - Assert.assertTrue(summaries.asText().toString().isEmpty()); - } - } - }); - } - - private void testRestSearchGeneric(Iterator searchNode) { - while (searchNode.hasNext()) { - JsonNode search = searchNode.next(); - Assert.assertEquals(search.findValue("start").asInt(), 0); - Assert.assertEquals(search.findValue("end").asInt(), 1); - Assert.assertEquals(search.findValue("total").asInt(), 1); - Assert.assertTrue(!search.findValue("summaries").isNull()); - Iterator iterator = search.findValue("summaries").elements(); - - while (iterator.hasNext()) { - JsonNode node = iterator.next(); - Assert.assertEquals(node.findValue("username").asText(), "growth"); - Assert.assertEquals(node.findValue("starttime").asLong(), 1460980616502L); - Assert.assertEquals(node.findValue("finishtime").asLong(), 1460980723925L); - Assert.assertEquals(node.findValue("waittime").asLong(), 20); - Assert.assertEquals(node.findValue("resourceused").asLong(), 100); - Assert.assertEquals(node.findValue("resourcewasted").asLong(), 30); - Assert.assertEquals(node.findValue("severity").asText(), "None"); - Assert.assertEquals(node.findValue("queue").asText(), "misc_default"); - - Iterator heuristicsSummary = node.findValue("heuristicsummary").elements(); - HashMap expectedHeuristics = new LinkedHashMap(); - expectedHeuristics.put("Mapper Skew", "None"); - expectedHeuristics.put("Mapper GC", "None"); - expectedHeuristics.put("Mapper Time", "None"); - expectedHeuristics.put("Mapper Speed", "None"); - expectedHeuristics.put("Mapper Spill", "None"); - expectedHeuristics.put("Mapper Memory", "None"); - expectedHeuristics.put("Reducer Skew", "None"); - expectedHeuristics.put("Reducer Time", "None"); - expectedHeuristics.put("Reducer GC", "None"); - expectedHeuristics.put("Reducer Memory", "None"); - expectedHeuristics.put("Shuffle & Sort", "None"); - - Iterator keyIterator = expectedHeuristics.keySet().iterator(); - while (heuristicsSummary.hasNext() && keyIterator.hasNext()) { - JsonNode job = heuristicsSummary.next(); - String key = keyIterator.next().toString(); - Assert.assertEquals(key, job.findValue("name").asText()); - Assert.assertEquals(expectedHeuristics.get(key), job.findValue("severity").asText()); - } - } - } - } - - private void populateTestData() { - try { - initDB(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - private void populateAutoTuningTestData1() { - try { - initAutoTuningDB1(); - } catch (Exception e) { - e.printStackTrace(); - } - } -}